opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merged in fbacall/galaxy-central-myexp-integration (pull request #45)

Browse Source
This commit is contained in:
Jeremy Goecks
2012-05-10 16:27:33 -04:00
parent 3155bf6039 b0accee722
commit 9c4d8cf2ea
274 changed files with 16355 additions and 7152 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.hgignore
+6 -1
View File
@@ -36,6 +36,7 @@ shed_tool_conf.xml
tool_data_table_conf.xml
tool_sheds_conf.xml
integrated_tool_panel.xml
openid_conf.xml
static/welcome.html.*
static/welcome.html
@@ -63,8 +64,12 @@ run_functional_tests.html
# Jars
tool-data/shared/jars/
# CSS build artifacts.
*/variables.less
static/june_2007_style/blue/base_sprites.less
# Misc
*.orig
.DS_Store
*.rej
*~
*~
community_wsgi.ini.sample
+5
View File
@@ -4,7 +4,12 @@
use = egg:Paste#http
port = 9009
# The address on which to listen. By default, only listen to localhost (the tool shed will not
# be accessible over the network). Use '0.0.0.0' to listen on all available network interfaces.
#host = 0.0.0.0
host = 127.0.0.1
use_threadpool = true
threadpool_workers = 10
contrib/universe_merger.py
+87
View File
@@ -0,0 +1,87 @@
#! /usr/bin/env python
'''
universe_merger.py
Created by Anne Pajon on 31 Jan 2012
Copyright (c) 2012 Cancer Research UK - Cambridge Research Institute.
This source file is licensed under the Academic Free License version
3.0 available at http://www.opensource.org/licenses/AFL-3.0.
Permission is hereby granted to reproduce, translate, adapt, alter,
transform, modify, or arrange this source file (the "Original Work");
to distribute or communicate copies of it under any license of your
choice that does not contradict the terms and conditions; to perform
or display the Original Work publicly.
THE ORIGINAL WORK IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS
AND WITHOUT WARRANTY, EITHER EXPRESS OR IMPLIED, INCLUDING, WITHOUT
LIMITATION, THE WARRANTIES OF NON-INFRINGEMENT, MERCHANTABILITY OR
FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY OF
THE ORIGINAL WORK IS WITH YOU.
Script for merging specific local Galaxy config universe_wsgi.ini.cri with default Galaxy universe_wsgi.ini.sample
'''
import ConfigParser
import sys
import optparse
import logging
def main():
# logging configuration
logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
# get the options
parser = optparse.OptionParser()
parser.add_option("-s", "--sample", dest="sample", action="store", help="path to Galaxy universe_wsgi.ini.sample file")
parser.add_option("-c", "--config", dest="config", action="store", help="path to your own universe_wsgi.ini file")
parser.add_option("-o", "--output", dest="output", action="store", help="path to the new merged universe_wsgi.ini.new file")
(options, args) = parser.parse_args()
for option in ['sample', 'config']:
if getattr(options, option) == None:
print "Please supply a --%s parameter.\n" % (option)
parser.print_help()
sys.exit()
config_sample = ConfigParser.RawConfigParser()
config_sample.read(options.sample)
config_sample_content = open(options.sample, 'r').read()
config = ConfigParser.RawConfigParser()
config.read(options.config)
logging.info("Merging your own config file %s into the sample one %s." % (options.config, options.sample))
logging.info("---------- DIFFERENCE ANALYSIS BEGIN ----------")
for section in config.sections():
if not config_sample.has_section(section):
logging.warning("-MISSING- section [%s] not found in sample file. It will be ignored." % section)
else:
for (name, value) in config.items(section):
if not config_sample.has_option(section, name):
if not "#%s" % name in config_sample_content:
logging.warning("-MISSING- section [%s] option '%s' not found in sample file. It will be ignored." % (section, name))
else:
logging.info("-notset- section [%s] option '%s' not set in sample file. It will be added." % (section, name))
config_sample.set(section, name, value)
else:
if not config_sample.get(section, name) == value:
logging.info("- diff - section [%s] option '%s' has different value ('%s':'%s'). It will be modified." % (section, name, config_sample.get(section, name), value))
config_sample.set(section, name, value)
logging.info("---------- DIFFERENCE ANALYSIS END ----------")
if options.output:
outputfile = open(options.output, 'w')
config_sample.write(outputfile)
outputfile.close()
else:
#print "----------"
#config_sample.write(sys.stdout)
#print "----------"
logging.info("use -o OUTPUT to write the merged configuration into a file.")
logging.info("read Galaxy universe_wsgi.ini.sample for detailed information.")
if __name__ == '__main__':
main()
datatypes_conf.xml.sample
+11 -4
View File
@@ -10,6 +10,7 @@
<display file="ucsc/bam.xml" />
<display file="ensembl/ensembl_bam.xml" />
<display file="igv/bam.xml" />
<display file="igb/bam.xml" />
</datatype>
<datatype extension="bed" type="galaxy.datatypes.interval:Bed" display_in_upload="true">
<converter file="bed_to_gff_converter.xml" target_datatype="gff"/>
@@ -19,6 +20,7 @@
<converter file="bed_to_summary_tree_converter.xml" target_datatype="summary_tree"/>
<!-- <display file="ucsc/interval_as_bed.xml" /> -->
<display file="genetrack.xml" />
<display file="igb/bed.xml" />
</datatype>
<datatype extension="bedgraph" type="galaxy.datatypes.interval:BedGraph" display_in_upload="true" />
<datatype extension="bedstrict" type="galaxy.datatypes.interval:BedStrict" />
@@ -31,9 +33,11 @@
</datatype>
<datatype extension="bigbed" type="galaxy.datatypes.binary:BigBed" mimetype="application/octet-stream" display_in_upload="true">
<display file="ucsc/bigbed.xml" />
<display file="igb/bb.xml" />
</datatype>
<datatype extension="bigwig" type="galaxy.datatypes.binary:BigWig" mimetype="application/octet-stream" display_in_upload="true">
<display file="ucsc/bigwig.xml" />
<display file="igb/bigwig.xml" />
</datatype>
<!-- MSI added Datatypes -->
<datatype extension="csv" type="galaxy.datatypes.tabular:Tabular" subclass="True" display_in_upload="true" /> <!-- FIXME: csv is 'tabular'ized data, but not 'tab-delimited'; the class used here is intended for 'tab-delimited' -->
@@ -81,7 +85,11 @@
<datatype extension="gff3" type="galaxy.datatypes.interval:Gff3" display_in_upload="true"/>
<datatype extension="gif" type="galaxy.datatypes.images:Gif" mimetype="image/gif"/>
<datatype extension="gmaj.zip" type="galaxy.datatypes.images:Gmaj" mimetype="application/zip"/>
<datatype extension="gtf" type="galaxy.datatypes.interval:Gtf" display_in_upload="true"/>
<datatype extension="gtf" type="galaxy.datatypes.interval:Gtf" display_in_upload="true">
<converter file="gff_to_bgzip_converter.xml" target_datatype="bgzip"/>
<converter file="gff_to_tabix_converter.xml" target_datatype="tabix" depends_on="bgzip"/>
<converter file="gff_to_summary_tree_converter.xml" target_datatype="summary_tree"/>
</datatype>
<datatype extension="h5" type="galaxy.datatypes.binary:Binary" mimetype="application/octet-stream" subclass="True" />
<datatype extension="html" type="galaxy.datatypes.images:Html" mimetype="text/html"/>
<datatype extension="interval" type="galaxy.datatypes.interval:Interval" display_in_upload="true">
@@ -170,13 +178,13 @@
<display file="igv/vcf.xml" />
<display file="rviewer/vcf.xml" inherit="True"/>
</datatype>
<datatype extension="bcf" type="galaxy.datatypes.binary:Binary" subclass="True"/>
<datatype extension="wsf" type="galaxy.datatypes.wsf:SnpFile" display_in_upload="true"/>
<datatype extension="bcf" type="galaxy.datatypes.binary:Binary" subclass="True" display_in_upload="True"/>
<datatype extension="velvet" type="galaxy.datatypes.assembly:Velvet" display_in_upload="false"/>
<datatype extension="wig" type="galaxy.datatypes.interval:Wiggle" display_in_upload="true">
<converter file="wig_to_bigwig_converter.xml" target_datatype="bigwig"/>
<converter file="wiggle_to_simple_converter.xml" target_datatype="interval"/>
<!-- <display file="gbrowse/gbrowse_wig.xml" /> -->
<display file="igb/wig.xml" />
</datatype>
<datatype extension="summary_tree" type="galaxy.datatypes.binary:Binary" subclass="True" />
<datatype extension="interval_index" type="galaxy.datatypes.binary:Binary" subclass="True" />
@@ -268,7 +276,6 @@
<sniffer type="galaxy.datatypes.images:Ppm"/>
<sniffer type="galaxy.datatypes.images:Psd"/>
<sniffer type="galaxy.datatypes.images:Xbm"/>
<sniffer type="galaxy.datatypes.images:Xpm"/>
<sniffer type="galaxy.datatypes.images:Rgb"/>
<sniffer type="galaxy.datatypes.images:Pbm"/>
<sniffer type="galaxy.datatypes.images:Pgm"/>
display_applications/igb/bam.xml
+58
View File
@@ -0,0 +1,58 @@
<display id="igb_bam" version="0.0.0" name="display in IGB">
<link id="Local" name="Local">
<url>http://localhost:7085/UnibrowControl?version=${bam_file.dbkey}&amp;feature_url_0=${bam_file.url}&amp;sym_name_0=${thenicename}&amp;sym_method_0=${bam_file.url}&amp;query_url=${bam_file.url}&amp;server_url=galaxy</url>
<param type="data" name="bam_file_for_name" viewable="False"/>
<param type="template" name="thenicename" viewable="False" strip="True">
#import re
#set nm=$bam_file_for_name.name
${re.sub('\W',"_",nm)}
</param>
<param type="data" name="bai_file" url="${thenicename}_${DATASET_HASH}.bam.bai" metadata="bam_index" strip_https="True" />
<param type="data" name="bam_file" url="${thenicename}_${DATASET_HASH}.bam" strip_https="True" />
</link>
<link id="Web" name="Web">
<url>$jnlp.url</url>
<param type="data" name="bam_file_for_name" viewable="False"/>
<param type="template" name="thenicename" viewable="False" strip="True">
#import re
#set nm=$bam_file_for_name.name
${re.sub('\W',"_",nm)}
</param>
<param type="data" name="bai_file" url="${thenicename}_${DATASET_HASH}.bam.bai" metadata="bam_index" strip_https="True" />
<param type="data" name="bam_file" url="${thenicename}_${DATASET_HASH}.bam" strip_https="True" />
<param type="template" name="jnlp" url="${thenicename}_${DATASET_HASH}.jnlp" viewable="True" strip_https="True" mimetype="application/x-java-jnlp-file">
&lt;?xml version=&quot;1.0&quot; encoding=&quot;utf-8&quot;?&gt;
&lt;jnlp spec=&quot;6.0+&quot; version=&quot;&quot; codebase=&quot;http://www.bioviz.org/igb/releases/current/&quot; &gt;
&lt;information&gt;
&lt;title&gt;IGB&lt;/title&gt;
&lt;vendor&gt;Genoviz Project&lt;/vendor&gt;
&lt;homepage href=&quot;http://genoviz.sourceforge.net/&quot; /&gt;
&lt;description&gt;Integrated Genome Browser&lt;/description&gt;
&lt;icon href=&quot;igb.gif&quot;/&gt;
&lt;offline-allowed/&gt;
&lt;shortcut online=&quot;true&quot;&gt;
&lt;desktop/&gt;
&lt;menu submenu=&quot;IGB&quot; /&gt;
&lt;/shortcut&gt;
&lt;/information&gt;
&lt;security&gt;
&lt;all-permissions/&gt;
&lt;/security&gt;
&lt;resources&gt;
&lt;java version=&quot;1.6+&quot; initial-heap-size=&quot;32m&quot; max-heap-size=&quot;1024m&quot;/&gt;
&lt;jar href=&quot;igb_exe.jar&quot; main=&quot;true&quot; /&gt;
&lt;jar href=&quot;igb-i18n.jar&quot; /&gt;
&lt;property name=&quot;apple.laf.useScreenMenuBar&quot; value=&quot;true&quot; /&gt;
&lt;property name=&quot;http.agent&quot; value=&quot;IGB (Webstart)&quot; /&gt;
&lt;/resources&gt;
&lt;application-desc main-class=&quot;com.affymetrix.main.Main&quot;&gt;
&lt;argument&gt;-prefs&lt;/argument&gt;
&lt;argument&gt;http://www.bioviz.org/igb/releases/current/igb_prefs.xml&lt;/argument&gt;
&lt;argument&gt;-href&lt;/argument&gt;
&lt;argument&gt;http://localhost:7085/UnibrowControl?version=${bam_file.dbkey}&amp;feature_url_0=${bam_file.url}&amp;sym_name_0=${thenicename}&amp;sym_method_0=${bam_file.url}&amp;query_url=${bam_file.url}&amp;server_url=galaxy &lt;/argument&gt;
&lt;/application-desc&gt;
&lt;/jnlp&gt;
</param>
</link>
</display>
display_applications/igb/bb.xml
+58
View File
@@ -0,0 +1,58 @@
<display id="igb_bb" version="1.0.0" name="display in IGB">
<link id="Local" name="Local">
<url>http://localhost:7085/UnibrowControl?version=${bigbed_file.dbkey}&amp;loadresidues=false&amp;feature_url_0=${bigbed_file.url}&amp;sym_name_0=${thenicename}&amp;sym_method_0=${bigbed_file.url}&amp;query_url=${bigbed_file.url}&amp;server_url=galaxy</url>
<param type="data" name="bigbed_file_for_name" viewable="False"/>
<param type="template" name="thenicename" viewable="False" strip="True">
#import re
#set nm=$bigbed_file_for_name.name
${re.sub('\W',"_",nm)}
</param>
<param type="data" name="bigbed_file" url="${thenicename}_${DATASET_HASH}.bigbed" strip_https="True" />
</link>
<link id="Web" name="Web">
<url>$jnlp.url</url>
<param type="data" name="bigbed_file_for_name" viewable="False"/>
<param type="template" name="thenicename" viewable="False" strip="True">
#import re
#set nm=$bigbed_file_for_name.name
${re.sub('\W',"_",nm)}
</param>
<param type="data" name="bigbed_file" url="${thenicename}_${DATASET_HASH}.bigbed" strip_https="True" />
<param type="template" name="jnlp" url="${thenicename}_${DATASET_HASH}.jnlp" viewable="True" strip_https="True" mimetype="application/x-java-jnlp-file">
&lt;?xml version=&quot;1.0&quot; encoding=&quot;utf-8&quot;?&gt;
&lt;jnlp spec=&quot;6.0+&quot; version=&quot;&quot; codebase=&quot;http://www.bioviz.org/igb/releases/current/&quot; &gt;
&lt;information&gt;
&lt;title&gt;IGB&lt;/title&gt;
&lt;vendor&gt;Genoviz Project&lt;/vendor&gt;
&lt;homepage href=&quot;http://genoviz.sourceforge.net/&quot; /&gt;
&lt;description&gt;Integrated Genome Browser&lt;/description&gt;
&lt;icon href=&quot;igb.gif&quot;/&gt;
&lt;offline-allowed/&gt;
&lt;shortcut online=&quot;true&quot;&gt;
&lt;desktop/&gt;
&lt;menu submenu=&quot;IGB&quot; /&gt;
&lt;/shortcut&gt;
&lt;/information&gt;
&lt;security&gt;
&lt;all-permissions/&gt;
&lt;/security&gt;
&lt;resources&gt;
&lt;java version=&quot;1.6+&quot; initial-heap-size=&quot;32m&quot; max-heap-size=&quot;1024m&quot;/&gt;
&lt;jar href=&quot;igb_exe.jar&quot; main=&quot;true&quot; /&gt;
&lt;jar href=&quot;igb-i18n.jar&quot; /&gt;
&lt;property name=&quot;apple.laf.useScreenMenuBar&quot; value=&quot;true&quot; /&gt;
&lt;property name=&quot;http.agent&quot; value=&quot;IGB (Webstart)&quot; /&gt;
&lt;/resources&gt;
&lt;application-desc main-class=&quot;com.affymetrix.main.Main&quot;&gt;
&lt;argument&gt;-prefs&lt;/argument&gt;
&lt;argument&gt;http://www.bioviz.org/igb/releases/current/igb_prefs.xml&lt;/argument&gt;
&lt;argument&gt;-href&lt;/argument&gt;
&lt;argument&gt;http://localhost:7085/UnibrowControl?version=${bigbed_file.dbkey}&amp;loadresidues=false&amp;feature_url_0=${bigbed_file.url}&amp;sym_name_0=${thenicename}&amp;sym_method_0=${bigbed_file.url}&amp;query_url=${bigbed_file.url}&amp;server_url=galaxy &lt;/argument&gt;
&lt;/application-desc&gt;
&lt;/jnlp&gt;
</param>
</link>
</display>
display_applications/igb/bed.xml
+73
View File
@@ -0,0 +1,73 @@
<display id="igb_bed" version="1.0.0" name="display in IGB">
<link id="Local" name="Local">
<url>http://localhost:7085/UnibrowControl?version=${bed_file.dbkey}&amp;${position}&amp;loadresidues=false&amp;feature_url_0=${bed_file.url}&amp;sym_name_0=${thenicename}&amp;sym_method_0=${bed_file.url}&amp;query_url=${bed_file.url}&amp;server_url=galaxy</url>
<param type="data" name="bed_file_for_name" viewable="False"/>
<param type="template" name="thenicename" viewable="False" strip="True">
#import re
#set nm=$bed_file_for_name.name
${re.sub('\W',"_",nm)}
</param>
<param type="data" name="bed_file" url="${thenicename}_${DATASET_HASH}.bed" strip_https="True" />
<param type="template" name="position" strip="True" >
#set chrom, start, end = $bed_file.datatype.get_estimated_display_viewport( $bed_file )
#if $chrom is not None:
seqid=${chrom}&amp;start=${start}&amp;end=${int(end) + 1}
#else:
seqid=&amp;start=&amp;end=
#end if
</param>
</link>
<link id="Web" name="Web">
<url>$jnlp.url</url>
<param type="data" name="bed_file_for_name" viewable="False"/>
<param type="template" name="thenicename" viewable="False" strip="True">
#import re
#set nm=$bed_file_for_name.name
${re.sub('\W',"_",nm)}
</param>
<param type="data" name="bed_file" url="${thenicename}_${DATASET_HASH}.bed" strip_https="True" />
<param type="template" name="position" strip="True" >
#set chrom, start, end = $bed_file.datatype.get_estimated_display_viewport( $bed_file )
#if $chrom is not None:
seqid=${chrom}&amp;start=${start}&amp;end=${int(end) + 1}
#else:
seqid=&amp;start=&amp;end=
#end if
</param>
<param type="template" name="jnlp" url="${thenicename}_${DATASET_HASH}.jnlp" viewable="True" strip_https="True" mimetype="application/x-java-jnlp-file">
&lt;?xml version=&quot;1.0&quot; encoding=&quot;utf-8&quot;?&gt;
&lt;jnlp spec=&quot;6.0+&quot; version=&quot;&quot; codebase=&quot;http://www.bioviz.org/igb/releases/current/&quot; &gt;
&lt;information&gt;
&lt;title&gt;IGB&lt;/title&gt;
&lt;vendor&gt;Genoviz Project&lt;/vendor&gt;
&lt;homepage href=&quot;http://genoviz.sourceforge.net/&quot; /&gt;
&lt;description&gt;Integrated Genome Browser&lt;/description&gt;
&lt;icon href=&quot;igb.gif&quot;/&gt;
&lt;offline-allowed/&gt;
&lt;shortcut online=&quot;true&quot;&gt;
&lt;desktop/&gt;
&lt;menu submenu=&quot;IGB&quot; /&gt;
&lt;/shortcut&gt;
&lt;/information&gt;
&lt;security&gt;
&lt;all-permissions/&gt;
&lt;/security&gt;
&lt;resources&gt;
&lt;java version=&quot;1.6+&quot; initial-heap-size=&quot;32m&quot; max-heap-size=&quot;1024m&quot;/&gt;
&lt;jar href=&quot;igb_exe.jar&quot; main=&quot;true&quot; /&gt;
&lt;jar href=&quot;igb-i18n.jar&quot; /&gt;
&lt;property name=&quot;apple.laf.useScreenMenuBar&quot; value=&quot;true&quot; /&gt;
&lt;property name=&quot;http.agent&quot; value=&quot;IGB (Webstart)&quot; /&gt;
&lt;/resources&gt;
&lt;application-desc main-class=&quot;com.affymetrix.main.Main&quot;&gt;
&lt;argument&gt;-prefs&lt;/argument&gt;
&lt;argument&gt;http://www.bioviz.org/igb/releases/current/igb_prefs.xml&lt;/argument&gt;
&lt;argument&gt;-href&lt;/argument&gt;
&lt;argument&gt;http://localhost:7085/UnibrowControl?version=${bed_file.dbkey}&amp;${position}&amp;loadresidues=false&amp;feature_url_0=${bed_file.url}&amp;sym_name_0=${thenicename}&amp;sym_method_0=${bed_file.url}&amp;query_url=${bed_file.url}&amp;server_url=galaxy &lt;/argument&gt;
&lt;/application-desc&gt;
&lt;/jnlp&gt;
</param>
</link>
</display>
display_applications/igb/bigwig.xml
+65
View File
@@ -0,0 +1,65 @@
<display id="igb_bigwig" version="1.0.0" name="display in IGB">
<link id="Local" name="Local">
<url>http://localhost:7085/UnibrowControl?version=${bigwig_file.dbkey}&amp;loadresidues=false&amp;feature_url_0=${bigwig_file.url}&amp;sym_name_0=${thenicename}&amp;sym_method_0=${bigwig_file.url}&amp;query_url=${bigwig_file.url}&amp;server_url=galaxy</url>
<param type="data" name="bigwig_file_for_name" viewable="False"/>
<param type="template" name="thenicename" viewable="False" strip="True">
#import re
#set nm=$bigwig_file_for_name.name
${re.sub('\W',"_",nm)}
</param>
<param type="data" name="bigwig_file" url="${thenicename}_${DATASET_HASH}.bigwig" strip_https="True" />
<!--<param type="template" name="position" strip="True" >
#set chrom, start, end = $bigwig_file.datatype.get_estimated_display_viewport( $bigwig_file )
#if $chrom is not None:
seqid=${chrom}&amp;start=${start}&amp;end=${int(end) + 1}
#else:
seqid=&amp;start=&amp;end=
#end if
</param>-->
</link>
<link id="Web" name="Web">
<url>$jnlp.url</url>
<param type="data" name="bigwig_file_for_name" viewable="False"/>
<param type="template" name="thenicename" viewable="False" strip="True">
#import re
#set nm=$bigwig_file_for_name.name
${re.sub('\W',"_",nm)}
</param>
<param type="data" name="bigwig_file" url="${thenicename}_${DATASET_HASH}.bigwig" strip_https="True" />
<param type="template" name="jnlp" url="${thenicename}_${DATASET_HASH}.jnlp" viewable="True" strip_https="True" mimetype="application/x-java-jnlp-file">
&lt;?xml version=&quot;1.0&quot; encoding=&quot;utf-8&quot;?&gt;
&lt;jnlp spec=&quot;6.0+&quot; version=&quot;&quot; codebase=&quot;http://www.bioviz.org/igb/releases/current/&quot; &gt;
&lt;information&gt;
&lt;title&gt;IGB&lt;/title&gt;
&lt;vendor&gt;Genoviz Project&lt;/vendor&gt;
&lt;homepage href=&quot;http://genoviz.sourceforge.net/&quot; /&gt;
&lt;description&gt;Integrated Genome Browser&lt;/description&gt;
&lt;icon href=&quot;igb.gif&quot;/&gt;
&lt;offline-allowed/&gt;
&lt;shortcut online=&quot;true&quot;&gt;
&lt;desktop/&gt;
&lt;menu submenu=&quot;IGB&quot; /&gt;
&lt;/shortcut&gt;
&lt;/information&gt;
&lt;security&gt;
&lt;all-permissions/&gt;
&lt;/security&gt;
&lt;resources&gt;
&lt;java version=&quot;1.6+&quot; initial-heap-size=&quot;32m&quot; max-heap-size=&quot;1024m&quot;/&gt;
&lt;jar href=&quot;igb_exe.jar&quot; main=&quot;true&quot; /&gt;
&lt;jar href=&quot;igb-i18n.jar&quot; /&gt;
&lt;property name=&quot;apple.laf.useScreenMenuBar&quot; value=&quot;true&quot; /&gt;
&lt;property name=&quot;http.agent&quot; value=&quot;IGB (Webstart)&quot; /&gt;
&lt;/resources&gt;
&lt;application-desc main-class=&quot;com.affymetrix.main.Main&quot;&gt;
&lt;argument&gt;-prefs&lt;/argument&gt;
&lt;argument&gt;http://www.bioviz.org/igb/releases/current/igb_prefs.xml&lt;/argument&gt;
&lt;argument&gt;-href&lt;/argument&gt;
&lt;argument&gt;http://localhost:7085/UnibrowControl?version=${bigwig_file.dbkey}&amp;loadresidues=false&amp;feature_url_0=${bigwig_file.url}&amp;sym_name_0=${thenicename}&amp;sym_method_0=${bigwig_file.url}&amp;query_url=${bigwig_file.url}&amp;server_url=galaxy &lt;/argument&gt;
&lt;/application-desc&gt;
&lt;/jnlp&gt;
</param>
</link>
</display>
display_applications/igb/wig.xml
+74
View File
@@ -0,0 +1,74 @@
<display id="igb_wig" version="1.0.0" name="display in IGB">
<link id="Local" name="Local">
<url>http://localhost:7085/UnibrowControl?version=${wig_file.dbkey}&amp;${position}&amp;loadresidues=false&amp;feature_url_0=${wig_file.url}&amp;sym_name_0=${thenicename}&amp;sym_method_0=${wig_file.url}&amp;query_url=${wig_file.url}&amp;server_url=galaxy</url>
<param type="data" name="wig_file_for_name" viewable="False"/>
<param type="template" name="thenicename" viewable="False" strip="True">
#import re
#set nm=$wig_file_for_name.name
${re.sub('\W',"_",nm)}
</param>
<param type="data" name="wig_file" url="${thenicename}_${DATASET_HASH}.wig" strip_https="True" />
<param type="template" name="position" strip="True" >
#set chrom, start, end = $wig_file.datatype.get_estimated_display_viewport( $wig_file )
#if $chrom is not None:
seqid=${chrom}&amp;start=${start}&amp;end=${int(end) + 1}
#else:
seqid=&amp;start=&amp;end=
#end if
</param>
</link>
<link id="Web" name="Web">
<url>$jnlp.url</url>
<param type="data" name="wig_file_for_name" viewable="False"/>
<param type="template" name="thenicename" viewable="False" strip="True">
#import re
#set nm=$wig_file_for_name.name
${re.sub('\W',"_",nm)}
</param>
<param type="data" name="wig_file" url="${thenicename}_${DATASET_HASH}.wig" strip_https="True" />
<param type="template" name="position" strip="True" >
#set chrom, start, end = $wig_file.datatype.get_estimated_display_viewport( $wig_file )
#if $chrom is not None:
seqid=${chrom}&amp;start=${start}&amp;end=${int(end) + 1}
#else:
seqid=&amp;start=&amp;end=
#end if
</param>
<param type="template" name="jnlp" url="${thenicename}_${DATASET_HASH}.jnlp" viewable="True" strip_https="True" mimetype="application/x-java-jnlp-file">
&lt;?xml version=&quot;1.0&quot; encoding=&quot;utf-8&quot;?&gt;
&lt;jnlp spec=&quot;6.0+&quot; version=&quot;&quot; codebase=&quot;http://www.bioviz.org/igb/releases/current/&quot; &gt;
&lt;information&gt;
&lt;title&gt;IGB&lt;/title&gt;
&lt;vendor&gt;Genoviz Project&lt;/vendor&gt;
&lt;homepage href=&quot;http://genoviz.sourceforge.net/&quot; /&gt;
&lt;description&gt;Integrated Genome Browser&lt;/description&gt;
&lt;icon href=&quot;igb.gif&quot;/&gt;
&lt;offline-allowed/&gt;
&lt;shortcut online=&quot;true&quot;&gt;
&lt;desktop/&gt;
&lt;menu submenu=&quot;IGB&quot; /&gt;
&lt;/shortcut&gt;
&lt;/information&gt;
&lt;security&gt;
&lt;all-permissions/&gt;
&lt;/security&gt;
&lt;resources&gt;
&lt;java version=&quot;1.6+&quot; initial-heap-size=&quot;32m&quot; max-heap-size=&quot;1024m&quot;/&gt;
&lt;jar href=&quot;igb_exe.jar&quot; main=&quot;true&quot; /&gt;
&lt;jar href=&quot;igb-i18n.jar&quot; /&gt;
&lt;property name=&quot;apple.laf.useScreenMenuBar&quot; value=&quot;true&quot; /&gt;
&lt;property name=&quot;http.agent&quot; value=&quot;IGB (Webstart)&quot; /&gt;
&lt;/resources&gt;
&lt;application-desc main-class=&quot;com.affymetrix.main.Main&quot;&gt;
&lt;argument&gt;-prefs&lt;/argument&gt;
&lt;argument&gt;http://www.bioviz.org/igb/releases/current/igb_prefs.xml&lt;/argument&gt;
&lt;argument&gt;-href&lt;/argument&gt;
&lt;argument&gt;http://localhost:7085/UnibrowControl?version=${wig_file.dbkey}&amp;${position}&amp;loadresidues=false&amp;feature_url_0=${wig_file.url}&amp;sym_name_0=${thenicename}&amp;sym_method_0=${wig_file.url}&amp;query_url=${wig_file.url}&amp;server_url=galaxy &lt;/argument&gt;
&lt;/application-desc&gt;
&lt;/jnlp&gt;
</param>
</link>
</display>
dist-eggs.ini
+12 -35
View File
@@ -7,105 +7,82 @@
;
[hosts]
py2.4-linux-i686-ucs2 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs2/bin/python2.4
py2.4-linux-i686-ucs4 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs4/bin/python2.4
py2.5-linux-i686-ucs2 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs2/bin/python2.5
py2.5-linux-i686-ucs4 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs4/bin/python2.5
py2.6-linux-i686-ucs2 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs2/bin/python2.6
py2.6-linux-i686-ucs4 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs4/bin/python2.6
py2.7-linux-i686-ucs2 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs2/bin/python2.7
py2.7-linux-i686-ucs4 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs4/bin/python2.7
py2.4-linux-x86_64-ucs2 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs2/bin/python2.4
py2.4-linux-x86_64-ucs4 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.4
py2.5-linux-x86_64-ucs2 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs2/bin/python2.5
py2.5-linux-x86_64-ucs4 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.5
py2.6-linux-x86_64-ucs2 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs2/bin/python2.6
py2.6-linux-x86_64-ucs4 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.6
py2.7-linux-x86_64-ucs2 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs2/bin/python2.7
py2.7-linux-x86_64-ucs4 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.7
py2.4-macosx-10.3-fat-ucs2 = weyerbacher.bx.psu.edu /usr/local/bin/python2.4
py2.5-macosx-10.3-fat-ucs2 = weyerbacher.bx.psu.edu /usr/local/bin/python2.5
py2.6-macosx-10.3-fat-ucs2 = weyerbacher.bx.psu.edu /usr/local/bin/python2.6
py2.7-macosx-10.3-fat-ucs2 = weyerbacher.bx.psu.edu /usr/local/bin/python2.7
py2.5-macosx-10.3-fat-ucs2 = weyerbacher.bx.psu.edu /Library/Frameworks/Python.framework/Versions/2.5/bin/python2.5
py2.6-macosx-10.3-fat-ucs2 = weyerbacher.bx.psu.edu /Library/Frameworks/Python.framework/Versions/2.6/bin/python2.6
py2.7-macosx-10.3-fat-ucs2 = weyerbacher.bx.psu.edu /Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7
py2.6-macosx-10.6-universal-ucs2 = lion.bx.psu.edu /usr/bin/python2.6
py2.7-macosx-10.6-intel-ucs2 = lion.bx.psu.edu /usr/local/bin/python2.7
py2.4-solaris-2.10-i86pc_32-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_32-ucs2/bin/python2.4
py2.5-solaris-2.10-i86pc_32-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_32-ucs2/bin/python2.5
py2.6-solaris-2.10-i86pc_32-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_32-ucs2/bin/python2.6
py2.7-solaris-2.10-i86pc_32-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_32-ucs2/bin/python2.7
py2.4-solaris-2.10-i86pc_64-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_64-ucs2/bin/python2.4
py2.5-solaris-2.10-i86pc_64-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_64-ucs2/bin/python2.5
py2.6-solaris-2.10-i86pc_64-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_64-ucs2/bin/python2.6
py2.7-solaris-2.10-i86pc_64-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_64-ucs2/bin/python2.7
py2.4-solaris-2.10-sun4u_32-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_32-ucs2/bin/python2.4
py2.5-solaris-2.10-sun4u_32-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_32-ucs2/bin/python2.5
py2.6-solaris-2.10-sun4u_32-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_32-ucs2/bin/python2.6
py2.7-solaris-2.10-sun4u_32-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_32-ucs2/bin/python2.7
py2.4-solaris-2.10-sun4u_64-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_64-ucs2/bin/python2.4
py2.5-solaris-2.10-sun4u_64-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_64-ucs2/bin/python2.5
py2.6-solaris-2.10-sun4u_64-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_64-ucs2/bin/python2.6
py2.7-solaris-2.10-sun4u_64-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_64-ucs2/bin/python2.7
py2.5-solaris-2.10-sun4u_64-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-sun4u_64-ucs2/bin/python2.5
py2.6-solaris-2.10-sun4u_64-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-sun4u_64-ucs2/bin/python2.6
py2.7-solaris-2.10-sun4u_64-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-sun4u_64-ucs2/bin/python2.7
; these hosts are used to build eggs with no C extensions
py2.4 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.4
py2.5 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.5
py2.6 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.6
py2.7 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.7
[groups]
py2.4-linux-i686 = py2.4-linux-i686-ucs2 py2.4-linux-i686-ucs4
py2.4-linux-x86_64 = py2.4-linux-x86_64-ucs2 py2.4-linux-x86_64-ucs4
py2.5-linux-i686 = py2.5-linux-i686-ucs2 py2.5-linux-i686-ucs4
py2.5-linux-x86_64 = py2.5-linux-x86_64-ucs2 py2.5-linux-x86_64-ucs4
py2.6-linux-i686 = py2.6-linux-i686-ucs2 py2.6-linux-i686-ucs4
py2.6-linux-x86_64 = py2.6-linux-x86_64-ucs2 py2.6-linux-x86_64-ucs4
py2.7-linux-i686 = py2.7-linux-i686-ucs2 py2.7-linux-i686-ucs4
py2.7-linux-x86_64 = py2.7-linux-x86_64-ucs2 py2.7-linux-x86_64-ucs4
py2.4-linux = py2.4-linux-i686 py2.4-linux-x86_64
py2.5-linux = py2.5-linux-i686 py2.5-linux-x86_64
py2.6-linux = py2.6-linux-i686 py2.6-linux-x86_64
py2.7-linux = py2.7-linux-i686 py2.7-linux-x86_64
linux-i686 = py2.4-linux-i686 py2.5-linux-i686 py2.6-linux-i686 py2.7-linux-i686
linux-x86_64 = py2.4-linux-x86_64 py2.5-linux-x86_64 py2.6-linux-x86_64 py2.7-linux-x86_64
linux-i686 = py2.5-linux-i686 py2.6-linux-i686 py2.7-linux-i686
linux-x86_64 = py2.5-linux-x86_64 py2.6-linux-x86_64 py2.7-linux-x86_64
linux = linux-i686 linux-x86_64
py2.4-macosx = py2.4-macosx-10.3-fat-ucs2
py2.5-macosx = py2.5-macosx-10.3-fat-ucs2
py2.6-macosx = py2.6-macosx-10.3-fat-ucs2 py2.6-macosx-10.6-universal-ucs2
py2.7-macosx = py2.7-macosx-10.3-fat-ucs2 py2.7-macosx-10.6-intel-ucs2
macosx = py2.4-macosx py2.5-macosx py2.6-macosx py2.7-macosx
py2.4-solaris-i86pc = py2.4-solaris-2.10-i86pc_32-ucs2 py2.4-solaris-2.10-i86pc_64-ucs2
macosx = py2.5-macosx py2.6-macosx py2.7-macosx
py2.5-solaris-i86pc = py2.5-solaris-2.10-i86pc_32-ucs2 py2.5-solaris-2.10-i86pc_64-ucs2
py2.6-solaris-i86pc = py2.6-solaris-2.10-i86pc_32-ucs2 py2.6-solaris-2.10-i86pc_64-ucs2
py2.7-solaris-i86pc = py2.7-solaris-2.10-i86pc_32-ucs2 py2.7-solaris-2.10-i86pc_64-ucs2
py2.4-solaris-sun4u = py2.4-solaris-2.10-sun4u_32-ucs2 py2.4-solaris-2.10-sun4u_64-ucs2
py2.5-solaris-sun4u = py2.5-solaris-2.10-sun4u_32-ucs2 py2.5-solaris-2.10-sun4u_64-ucs2
py2.6-solaris-sun4u = py2.6-solaris-2.10-sun4u_32-ucs2 py2.6-solaris-2.10-sun4u_64-ucs2
py2.7-solaris-sun4u = py2.7-solaris-2.10-sun4u_32-ucs2 py2.7-solaris-2.10-sun4u_64-ucs2
py2.4-solaris = py2.4-solaris-i86pc py2.4-solaris-sun4u
py2.5-solaris = py2.5-solaris-i86pc py2.5-solaris-sun4u
py2.6-solaris = py2.6-solaris-i86pc py2.6-solaris-sun4u
py2.7-solaris = py2.7-solaris-i86pc py2.7-solaris-sun4u
solaris-i86pc = py2.4-solaris-i86pc py2.5-solaris-i86pc py2.6-solaris-i86pc py2.7-solaris-i86pc
solaris-sun4u = py2.4-solaris-sun4u py2.5-solaris-sun4u py2.6-solaris-sun4u py2.7-solaris-sun4u
solaris-i86pc = py2.5-solaris-i86pc py2.6-solaris-i86pc py2.7-solaris-i86pc
solaris-sun4u = py2.5-solaris-sun4u py2.6-solaris-sun4u py2.7-solaris-sun4u
solaris = solaris-i86pc solaris-sun4u
py2.4-all = py2.4-linux py2.4-macosx py2.4-solaris
py2.5-all = py2.5-linux py2.5-macosx py2.5-solaris
py2.6-all = py2.6-linux py2.6-macosx py2.6-solaris
py2.7-all = py2.7-linux py2.7-macosx py2.7-solaris
; group for building pysam on solaris 10 sparc
;solaris-2.10-sun4u = py2.4-solaris-2.10-sun4u_32-ucs2 py2.5-solaris-2.10-sun4u_32-ucs2 py2.6-solaris-2.10-sun4u_32-ucs2 py2.4-solaris-2.10-sun4u_64-ucs2 py2.5-solaris-2.10-sun4u_64-ucs2 py2.6-solaris-2.10-sun4u_64-ucs2
; the 'all' key is used internally by the build system to specify which hosts
; to build on when no hosts are specified on the dist-eggs.py command line.
all = linux macosx solaris
; the 'noplatform' key, likewise, is for which build hosts should be used when
; building pure python (noplatform) eggs.
noplatform = py2.4 py2.5 py2.6 py2.7
noplatform = py2.5 py2.6 py2.7
; don't build these eggs on these platforms:
[ignore]
GeneTrack = py2.4
python-daemon = py2.4
ctypes = py2.5-linux-i686-ucs2 py2.5-linux-i686-ucs4 py2.6-linux-i686-ucs2 py2.6-linux-i686-ucs4 py2.7-linux-i686-ucs2 py2.7-linux-i686-ucs4 py2.5-linux-x86_64-ucs2 py2.5-linux-x86_64-ucs4 py2.6-linux-x86_64-ucs2 py2.6-linux-x86_64-ucs4 py2.7-linux-x86_64-ucs2 py2.7-linux-x86_64-ucs4 py2.5-macosx-10.3-fat-ucs2 py2.6-macosx-10.3-fat-ucs2 py2.6-macosx-10.6-universal-ucs2 py2.7-macosx-10.3-fat-ucs2 py2.5-solaris-2.10-i86pc_32-ucs2 py2.6-solaris-2.10-i86pc_32-ucs2 py2.7-solaris-2.10-i86pc_32-ucs2 py2.5-solaris-2.10-i86pc_64-ucs2 py2.6-solaris-2.10-i86pc_64-ucs2 py2.7-solaris-2.10-i86pc_64-ucs2 py2.5-solaris-2.10-sun4u_32-ucs2 py2.6-solaris-2.10-sun4u_32-ucs2 py2.7-solaris-2.10-sun4u_32-ucs2 py2.5-solaris-2.10-sun4u_64-ucs2 py2.6-solaris-2.10-sun4u_64-ucs2 py2.7-solaris-2.10-sun4u_64-ucs2
eggs.ini
+1
View File
@@ -17,6 +17,7 @@ Cheetah = 2.2.2
ctypes = 1.0.2
DRMAA_python = 0.2
MarkupSafe = 0.12
mercurial = 2.1.2
MySQL_python = 1.2.3c1
numpy = 1.6.0
pbs_python = 4.1.0
lib/galaxy/app.py
+6 -2
View File
@@ -12,6 +12,7 @@ from galaxy.objectstore import build_object_store_from_config
import galaxy.quota
from galaxy.tags.tag_handler import GalaxyTagHandler
from galaxy.tools.imp_exp import load_history_imp_exp_tools
from galaxy.tools.genome_index import load_genome_index_tools
from galaxy.sample_tracking import external_service_types
from galaxy.openid.providers import OpenIDProviders
@@ -90,6 +91,8 @@ class UniverseApplication( object ):
self.datatypes_registry.load_external_metadata_tool( self.toolbox )
# Load history import/export tools.
load_history_imp_exp_tools( self.toolbox )
# Load genome indexer tool.
load_genome_index_tools( self.toolbox )
# Load security policy.
self.security_agent = self.model.security_agent
self.host_security_agent = galaxy.security.HostAgent( model=self.security_agent.model, permitted_actions=self.security_agent.permitted_actions )
@@ -124,8 +127,9 @@ class UniverseApplication( object ):
if self.config.get_bool( 'enable_beta_job_managers', False ):
from jobs import transfer_manager
self.transfer_manager = transfer_manager.TransferManager( self )
# Start the job queue
self.job_manager = jobs.JobManager( self )
# Start the job manager
from jobs import manager
self.job_manager = manager.JobManager( self )
# FIXME: These are exposed directly for backward compatibility
self.job_queue = self.job_manager.job_queue
self.job_stop_queue = self.job_manager.job_stop_queue
lib/galaxy/config.py
+42 -33
View File
@@ -42,8 +42,7 @@ class Configuration( object ):
tempfile.tempdir = self.new_file_path
self.openid_consumer_cache_path = resolve_path( kwargs.get( "openid_consumer_cache_path", "database/openid_consumer_cache" ), self.root )
self.cookie_path = kwargs.get( "cookie_path", "/" )
# web API
self.enable_api = string_as_bool( kwargs.get( 'enable_api', False ) )
self.genome_data_path = kwargs.get( "genome_data_path", "tool-data/genome" )
# Galaxy OpenID settings
self.enable_openid = string_as_bool( kwargs.get( 'enable_openid', False ) )
self.openid_config = kwargs.get( 'openid_config_file', 'openid_conf.xml' )
@@ -86,6 +85,7 @@ class Configuration( object ):
self.allow_user_dataset_purge = string_as_bool( kwargs.get( "allow_user_dataset_purge", "False" ) )
self.allow_user_impersonation = string_as_bool( kwargs.get( "allow_user_impersonation", "False" ) )
self.new_user_dataset_access_role_default_private = string_as_bool( kwargs.get( "new_user_dataset_access_role_default_private", "False" ) )
self.collect_outputs_from = [ x.strip() for x in kwargs.get( 'collect_outputs_from', 'new_file_path,job_working_directory' ).lower().split(',') ]
self.template_path = resolve_path( kwargs.get( "template_path", "templates" ), self.root )
self.template_cache = resolve_path( kwargs.get( "template_cache_path", "database/compiled_templates" ), self.root )
self.local_job_queue_workers = int( kwargs.get( "local_job_queue_workers", "5" ) )
@@ -105,6 +105,7 @@ class Configuration( object ):
self.smtp_username = kwargs.get( 'smtp_username', None )
self.smtp_password = kwargs.get( 'smtp_password', None )
self.start_job_runners = kwargs.get( 'start_job_runners', None )
self.expose_dataset_path = string_as_bool( kwargs.get( 'expose_dataset_path', 'False' ) )
# External Service types used in sample tracking
self.external_service_type_config_file = resolve_path( kwargs.get( 'external_service_type_config_file', 'external_service_types_conf.xml' ), self.root )
self.external_service_type_path = resolve_path( kwargs.get( 'external_service_type_path', 'external_service_types' ), self.root )
@@ -184,14 +185,43 @@ class Configuration( object ):
# Heartbeat log file name override
if global_conf is not None:
self.heartbeat_log = global_conf.get( 'heartbeat_log', 'heartbeat.log' )
#Store per-tool runner configs.
# Determine which 'server:' this is
self.server_name = 'main'
for arg in sys.argv:
# Crummy, but PasteScript does not give you a way to determine this
if arg.lower().startswith('--server-name='):
self.server_name = arg.split('=', 1)[-1]
# Store advanced job management config
self.job_manager = kwargs.get('job_manager', self.server_name).strip()
self.job_handlers = [ x.strip() for x in kwargs.get('job_handlers', self.server_name).split(',') ]
self.default_job_handlers = [ x.strip() for x in kwargs.get('default_job_handlers', ','.join( self.job_handlers ) ).split(',') ]
# Use database for IPC unless this is a standalone server (or multiple servers doing self dispatching in memory)
self.track_jobs_in_database = True
if ( len( self.job_handlers ) == 1 ) and ( self.job_handlers[0] == self.server_name ) and ( self.job_manager == self.server_name ):
self.track_jobs_in_database = False
# Store per-tool runner configs
self.tool_handlers = self.__read_tool_job_config( global_conf_parser, 'galaxy:tool_handlers', 'name' )
self.tool_runners = self.__read_tool_job_config( global_conf_parser, 'galaxy:tool_runners', 'url' )
self.datatypes_config = kwargs.get( 'datatypes_config_file', 'datatypes_conf.xml' )
# Cloud configuration options
self.enable_cloud_launch = string_as_bool( kwargs.get( 'enable_cloud_launch', False ) )
# Galaxy messaging (AMQP) configuration options
self.amqp = {}
try:
tool_runners_config = global_conf_parser.items("galaxy:tool_runners")
amqp_config = global_conf_parser.items("galaxy_amqp")
except ConfigParser.NoSectionError:
amqp_config = {}
for k, v in amqp_config:
self.amqp[k] = v
self.running_functional_tests = string_as_bool( kwargs.get( 'running_functional_tests', False ) )
def __read_tool_job_config( self, global_conf_parser, section, key ):
try:
tool_runners_config = global_conf_parser.items( section )
# Process config to group multiple configs for the same tool.
tool_runners = {}
rval = {}
for entry in tool_runners_config:
tool_config, url = entry
tool_config, val = entry
tool = None
runner_dict = {}
if tool_config.find("[") != -1:
@@ -206,29 +236,18 @@ class Configuration( object ):
tool = tool_config
# Add runner URL.
runner_dict[ 'url' ] = url
runner_dict[ key ] = val
# Create tool entry if necessary.
if tool not in tool_runners:
tool_runners[ tool ] = []
if tool not in rval:
rval[ tool ] = []
# Add entry to runners.
tool_runners[ tool ].append( runner_dict )
rval[ tool ].append( runner_dict )
self.tool_runners = tool_runners
return rval
except ConfigParser.NoSectionError:
self.tool_runners = []
self.datatypes_config = kwargs.get( 'datatypes_config_file', 'datatypes_conf.xml' )
# Cloud configuration options
self.enable_cloud_launch = string_as_bool( kwargs.get( 'enable_cloud_launch', False ) )
# Galaxy messaging (AMQP) configuration options
self.amqp = {}
try:
amqp_config = global_conf_parser.items("galaxy_amqp")
except ConfigParser.NoSectionError:
amqp_config = {}
for k, v in amqp_config:
self.amqp[k] = v
return []
def get( self, key, default ):
return self.config_dict.get( key, default )
def get_bool( self, key, default ):
@@ -238,16 +257,6 @@ class Configuration( object ):
return default
def check( self ):
paths_to_check = [ self.root, self.tool_path, self.tool_data_path, self.template_path ]
# Look for any tool shed configs and retrieve the tool_path attribute from the <toolbox> tag.
tool_configs = self.tool_configs
if self.migrated_tools_config not in tool_configs:
tool_configs.append( self.migrated_tools_config )
for config_filename in tool_configs:
tree = parse_xml( config_filename )
root = tree.getroot()
tool_path = root.get( 'tool_path' )
if tool_path not in [ None, False ]:
paths_to_check.append( resolve_path( tool_path, self.root ) )
# Check that required directories exist
for path in paths_to_check:
if path not in [ None, False ] and not os.path.isdir( path ):
lib/galaxy/datatypes/binary.py
+11
View File
@@ -34,6 +34,17 @@ class Binary( data.Data ):
"""Returns the mime type of the datatype"""
return 'application/octet-stream'
def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, size=None, offset=None, **kwd):
trans.response.set_content_type(dataset.get_mime())
trans.log_event( "Display dataset id: %s" % str( dataset.id ) )
trans.response.headers['Content-Length'] = int( os.stat( dataset.file_name ).st_size )
to_ext = dataset.extension
valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
fname = ''.join(c in valid_chars and c or '_' for c in dataset.name)[0:150]
trans.response.set_content_type( "application/octet-stream" ) #force octet-stream so Safari doesn't append mime extensions to filename
trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy%s-[%s].%s"' % (dataset.hid, fname, to_ext)
return open( dataset.file_name )
class Ab1( Binary ):
"""Class describing an ab1 binary sequence file"""
file_ext = "ab1"
lib/galaxy/datatypes/converters/sam_to_bam.py
+69
View File
@@ -0,0 +1,69 @@
#!/usr/bin/env python
#Dan Blankenberg
"""
A wrapper script for converting SAM to BAM, with sorting.
%prog input_filename.sam output_filename.bam
"""
import sys, optparse, os, tempfile, subprocess, shutil
CHUNK_SIZE = 2**20 #1mb
def cleanup_before_exit( tmp_dir ):
if tmp_dir and os.path.exists( tmp_dir ):
shutil.rmtree( tmp_dir )
def __main__():
#Parse Command Line
parser = optparse.OptionParser()
(options, args) = parser.parse_args()
assert len( args ) == 2, 'You must specify the input and output filenames'
input_filename, output_filename = args
tmp_dir = tempfile.mkdtemp( prefix='tmp-sam_to_bam_converter-' )
#convert to SAM
unsorted_bam_filename = os.path.join( tmp_dir, 'unsorted.bam' )
unsorted_stderr_filename = os.path.join( tmp_dir, 'unsorted.stderr' )
cmd = 'samtools view -bS "%s" > "%s"' % ( input_filename, unsorted_bam_filename )
proc = subprocess.Popen( args=cmd, stderr=open( unsorted_stderr_filename, 'wb' ), shell=True, cwd=tmp_dir )
return_code = proc.wait()
if return_code:
stderr_target = sys.stderr
else:
stderr_target = sys.stdout
stderr = open( unsorted_stderr_filename )
while True:
chunk = stderr.read( CHUNK_SIZE )
if chunk:
stderr_target.write( chunk )
else:
break
stderr.close()
#sort sam, so indexing will not fail
sorted_stderr_filename = os.path.join( tmp_dir, 'sorted.stderr' )
sorting_prefix = os.path.join( tmp_dir, 'sorted_bam' )
cmd = 'samtools sort -o "%s" "%s" > "%s"' % ( unsorted_bam_filename, sorting_prefix, output_filename )
proc = subprocess.Popen( args=cmd, stderr=open( sorted_stderr_filename, 'wb' ), shell=True, cwd=tmp_dir )
return_code = proc.wait()
if return_code:
stderr_target = sys.stderr
else:
stderr_target = sys.stdout
stderr = open( sorted_stderr_filename )
while True:
chunk = stderr.read( CHUNK_SIZE )
if chunk:
stderr_target.write( chunk )
else:
break
stderr.close()
cleanup_before_exit( tmp_dir )
if __name__=="__main__": __main__()
lib/galaxy/datatypes/converters/sam_to_bam.xml
+2 -2
View File
@@ -1,11 +1,11 @@
<tool id="CONVERTER_sam_to_bam" name="Convert SAM to BAM" version="1.0.0">
<tool id="CONVERTER_sam_to_bam" name="Convert SAM to BAM" version="2.0.0">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<!-- FIXME: conversion will only work if headers for reference sequences are in input file.
To fix this: (a) merge sam_to_bam tool in tools with this conversion (like fasta_to_len
conversion); and (b) define a datatype-specific way to set converter parameters.
-->
<command>samtools view -bS $input1 > $output 2> /dev/null </command>
<command interpreter="python">sam_to_bam.py $input1 $output</command>
<inputs>
<param name="input1" type="data" format="sam" label="SAM file"/>
</inputs>
lib/galaxy/datatypes/data.py
+194 -19
View File
@@ -3,11 +3,39 @@ from galaxy import util
from galaxy.util.odict import odict
from galaxy.util.bunch import Bunch
from galaxy.util import inflector
from galaxy.util.sanitize_html import sanitize_html
from cgi import escape
import mimetypes
import metadata
import zipfile
from metadata import MetadataElement #import directly to maintain ease of use in Datatype class definitions
if sys.version_info[:2] < ( 2, 6 ):
zipfile.BadZipFile = zipfile.error
if sys.version_info[:2] < ( 2, 5 ):
zipfile.LargeZipFile = zipfile.error
tmpd = tempfile.mkdtemp()
comptypes=[]
ziptype = '32'
tmpf = os.path.join( tmpd, 'compression_test.zip' )
try:
archive = zipfile.ZipFile( tmpf, 'w', zipfile.ZIP_DEFLATED, True )
archive.close()
comptypes.append( 'zip' )
ziptype = '64'
except RuntimeError:
log.exception( "Compression error when testing zip compression. This option will be disabled for library downloads." )
except (TypeError, zipfile.LargeZipFile): # ZIP64 is only in Python2.5+. Remove TypeError when 2.4 support is dropped
log.warning( 'Max zip file size is 2GB, ZIP64 not supported' )
comptypes.append( 'zip' )
try:
os.unlink( tmpf )
except OSError:
pass
os.rmdir( tmpd )
log = logging.getLogger(__name__)
# Valid first column and strand column values vor bed, other formats
@@ -39,7 +67,7 @@ class Data( object ):
'test'
>>> type( DataTest.metadata_spec.test.param )
<class 'galaxy.datatypes.metadata.MetadataParameter'>
"""
__metaclass__ = DataMeta
# Add metadata elements
@@ -60,7 +88,7 @@ class Data( object ):
primary_file_name = 'index'
#A per datatype setting (inherited): max file size (in bytes) for setting optional metadata
_max_optional_metadata_filesize = None
def __init__(self, **kwd):
"""Initialize the datatype"""
object.__init__(self, **kwd)
@@ -118,7 +146,7 @@ class Data( object ):
to_check = dataset.metadata.items()
for key, value in to_check:
if key in skip or ( not check and dataset.metadata.spec[key].get( "optional" ) ):
continue #we skip check for optional and nonrequested values here
continue #we skip check for optional and nonrequested values here
if not value:
return True
return False
@@ -142,6 +170,7 @@ class Data( object ):
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disk'
def display_peek(self, dataset ):
"""Create HTML table, used for displaying peek"""
out = ['<table cellspacing="0" cellpadding="3">']
@@ -163,6 +192,151 @@ class Data( object ):
except Exception, exc:
out = "Can't create peek %s" % str( exc )
return out
def _archive_composite_dataset( self, trans, data=None, **kwd ):
# save a composite object into a compressed archive for downloading
params = util.Params( kwd )
valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
outfname = data.name[0:150]
outfname = ''.join(c in valid_chars and c or '_' for c in outfname)
if (params.do_action == None):
params.do_action = 'zip' # default
msg = util.restore_text( params.get( 'msg', '' ) )
messagetype = params.get( 'messagetype', 'done' )
if not data:
msg = "You must select at least one dataset"
messagetype = 'error'
else:
error = False
try:
if (params.do_action == 'zip'):
# Can't use mkstemp - the file must not exist first
tmpd = tempfile.mkdtemp()
tmpf = os.path.join( tmpd, 'library_download.' + params.do_action )
if ziptype == '64':
archive = zipfile.ZipFile( tmpf, 'w', zipfile.ZIP_DEFLATED, True )
else:
archive = zipfile.ZipFile( tmpf, 'w', zipfile.ZIP_DEFLATED )
archive.add = lambda x, y: archive.write( x, y.encode('CP437') )
elif params.do_action == 'tgz':
archive = util.streamball.StreamBall( 'w|gz' )
elif params.do_action == 'tbz':
archive = util.streamball.StreamBall( 'w|bz2' )
except (OSError, zipfile.BadZipFile):
error = True
log.exception( "Unable to create archive for download" )
msg = "Unable to create archive for %s for download, please report this error" % outfname
messagetype = 'error'
if not error:
current_user_roles = trans.get_current_user_roles()
ext = data.extension
path = data.file_name
fname = os.path.split(path)[-1]
efp = data.extra_files_path
htmlname = os.path.splitext(outfname)[0]
if not htmlname.endswith(ext):
htmlname = '%s_%s' % (htmlname,ext)
archname = '%s.html' % htmlname # fake the real nature of the html file
try:
archive.add(data.file_name,archname)
except IOError:
error = True
log.exception( "Unable to add composite parent %s to temporary library download archive" % data.file_name)
msg = "Unable to create archive for download, please report this error"
messagetype = 'error'
for root, dirs, files in os.walk(efp):
for fname in files:
fpath = os.path.join(root,fname)
rpath = os.path.relpath(fpath,efp)
try:
archive.add( fpath,rpath )
except IOError:
error = True
log.exception( "Unable to add %s to temporary library download archive" % rpath)
msg = "Unable to create archive for download, please report this error"
messagetype = 'error'
continue
if not error:
if params.do_action == 'zip':
archive.close()
tmpfh = open( tmpf )
# CANNOT clean up - unlink/rmdir was always failing because file handle retained to return - must rely on a cron job to clean up tmp
trans.response.set_content_type( "application/x-zip-compressed" )
trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s.zip"' % outfname
return tmpfh
else:
trans.response.set_content_type( "application/x-tar" )
outext = 'tgz'
if params.do_action == 'tbz':
outext = 'tbz'
trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s.%s"' % (outfname,outext)
archive.wsgi_status = trans.response.wsgi_status()
archive.wsgi_headeritems = trans.response.wsgi_headeritems()
return archive.stream
return trans.show_error_message( msg )
def _serve_raw(self, trans, dataset, to_ext):
trans.response.headers['Content-Length'] = int( os.stat( dataset.file_name ).st_size )
valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
fname = ''.join(c in valid_chars and c or '_' for c in dataset.name)[0:150]
trans.response.set_content_type( "application/octet-stream" ) #force octet-stream so Safari doesn't append mime extensions to filename
trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy%s-[%s].%s"' % (dataset.hid, fname, to_ext)
return open( dataset.file_name )
def display_data(self, trans, data, preview=False, filename=None, to_ext=None, size=None, offset=None, **kwd):
""" Old display method, for transition """
#Relocate all composite datatype display to a common location.
composite_extensions = trans.app.datatypes_registry.get_composite_extensions( )
composite_extensions.append('html') # for archiving composite datatypes
if isinstance( data, basestring ):
return data
if filename and filename != "index":
# For files in extra_files_path
file_path = trans.app.object_store.get_filename(data.dataset, extra_dir='dataset_%s_files' % data.dataset.id, alt_name=filename)
if os.path.exists( file_path ):
if os.path.isdir( file_path ):
return trans.show_error_message( "Directory listing is not allowed." ) #TODO: Reconsider allowing listing of directories?
mime, encoding = mimetypes.guess_type( file_path )
if not mime:
try:
mime = trans.app.datatypes_registry.get_mimetype_by_extension( ".".split( file_path )[-1] )
except:
mime = "text/plain"
trans.response.set_content_type( mime )
return open( file_path )
else:
return trans.show_error_message( "Could not find '%s' on the extra files path %s." % ( filename, file_path ) )
trans.response.set_content_type(data.get_mime())
trans.log_event( "Display dataset id: %s" % str( data.id ) )
from galaxy import datatypes #DBTODO REMOVE THIS AT REFACTOR
if to_ext or isinstance(data.datatype, datatypes.binary.Binary): # Saving the file, or binary file
if data.extension in composite_extensions:
return self._archive_composite_dataset( trans, data, **kwd )
else:
trans.response.headers['Content-Length'] = int( os.stat( data.file_name ).st_size )
if not to_ext:
to_ext = data.extension
valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
fname = ''.join(c in valid_chars and c or '_' for c in data.name)[0:150]
trans.response.set_content_type( "application/octet-stream" ) #force octet-stream so Safari doesn't append mime extensions to filename
trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy%s-[%s].%s"' % (data.hid, fname, to_ext)
return open( data.file_name )
if not os.path.exists( data.file_name ):
raise paste.httpexceptions.HTTPNotFound( "File Not Found (%s)." % data.file_name )
max_peek_size = 1000000 # 1 MB
if isinstance(data.datatype, datatypes.images.Html):
max_peek_size = 10000000 # 10 MB for html
if not preview or isinstance(data.datatype, datatypes.images.Image) or os.stat( data.file_name ).st_size < max_peek_size:
if trans.app.config.sanitize_all_html and trans.response.get_content_type() == "text/html":
# Sanitize anytime we respond with plain text/html content.
return sanitize_html(open( data.file_name ).read())
return open( data.file_name )
else:
trans.response.set_content_type( "text/html" )
return trans.stream_template_mako( "/dataset/large_file.mako",
truncated_data = open( data.file_name ).read(max_peek_size),
data = data)
def display_name(self, dataset):
"""Returns formatted html of dataset name"""
try:
@@ -183,11 +357,11 @@ class Data( object ):
info = info.replace( '\r', '<br/>' )
if info.find( '\n' ) >= 0:
info = info.replace( '\n', '<br/>' )
# Convert to unicode to display non-ascii characters.
if type( info ) is not unicode:
info = unicode( info, 'utf-8')
return info
except:
return "info unavailable"
@@ -272,7 +446,7 @@ class Data( object ):
def convert_dataset(self, trans, original_dataset, target_type, return_output=False, visible=True, deps=None, set_output_history=True):
"""This function adds a job to the queue to convert a dataset to another type. Returns a message about success/failure."""
converter = trans.app.datatypes_registry.get_converter_by_target_type( original_dataset.ext, target_type )
if converter is None:
raise Exception( "A converter does not exist for %s to %s." % ( original_dataset.ext, target_type ) )
#Generate parameter dictionary
@@ -284,7 +458,7 @@ class Data( object ):
params[value.name] = deps[value.name]
elif value.type == 'data':
input_name = key
params[input_name] = original_dataset
#Run converter, job is dispatched through Queue
converted_dataset = converter.execute( trans, incoming=params, set_output_hid=visible, set_output_history=set_output_history)[1]
@@ -351,18 +525,18 @@ class Data( object ):
@property
def has_resolution(self):
return False
def merge( split_files, output_file):
def merge( split_files, output_file):
"""
TODO: Do we need to merge gzip files using gzjoin? cat seems to work,
but might be brittle. Need to revisit this.
"""
if len(split_files) == 1:
cmd = 'mv -f %s %s' % ( split_files[0], output_file )
cmd = 'mv -f %s %s' % ( split_files[0], output_file )
else:
cmd = 'cat %s > %s' % ( ' '.join(split_files), output_file )
cmd = 'cat %s > %s' % ( ' '.join(split_files), output_file )
result = os.system(cmd)
if result != 0:
raise Exception('Result %s from %s' % (result, cmd))
@@ -377,7 +551,7 @@ class Text( Data ):
def write_from_stream(self, dataset, stream):
"""Writes data from a stream"""
# write it twice for now
# write it twice for now
fd, temp_name = tempfile.mkstemp()
while 1:
chunk = stream.read(1048576)
@@ -468,11 +642,11 @@ class Text( Data ):
"""
if split_params is None:
return
if len(input_datasets) > 1:
raise Exception("Text file splitting does not support multiple files")
input_files = [ds.file_name for ds in input_datasets]
lines_per_file = None
chunk_size = None
if split_params['split_mode'] == 'number_of_parts':
@@ -501,7 +675,7 @@ class Text( Data ):
chunk_size = int(split_params['split_size'])
else:
raise Exception('Unsupported split mode %s' % split_params['split_mode'])
f = open(input_files[0], 'rt')
try:
chunk_idx = 0
@@ -562,7 +736,7 @@ def get_test_fname( fname ):
def get_file_peek( file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5, skipchars=[] ):
"""
Returns the first LINE_COUNT lines wrapped to WIDTH
## >>> fname = get_test_fname('4.bed')
## >>> get_file_peek(fname)
## 'chr22 30128507 31828507 uc003bnx.1_cds_2_0_chr22_29227_f 0 +\n'
@@ -601,11 +775,12 @@ def get_file_peek( file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5, skip
lines.append( line )
count += 1
temp.close()
if file_type in [ 'gzipped', 'binary' ]:
text = "%s file" % file_type
if file_type in [ 'gzipped', 'binary' ]:
text = "%s file" % file_type
else:
try:
text = unicode( '\n'.join( lines ), 'utf-8' )
except UnicodeDecodeError:
text = "binary/unknown file"
return text
lib/galaxy/datatypes/interval.py
+6
View File
@@ -328,6 +328,9 @@ class Interval( Tabular ):
def get_track_resolution( self, dataset, start, end):
return None
def get_track_type( self ):
return "FeatureTrack", {"data": "tabix", "index": "summary_tree"}
class BedGraph( Interval ):
"""Tab delimited chrom/start/end/datavalue dataset"""
@@ -950,6 +953,9 @@ class Gtf( Gff ):
dataset.metadata.attribute_types = attribute_types
dataset.metadata.attributes = len( attribute_types )
Gff.set_meta( self, dataset, overwrite = overwrite, skip = i )
def get_track_type( self ):
return "FeatureTrack", {"data": "tabix", "index": "summary_tree"}
class Wiggle( Tabular, _RemoteCallMixin ):
lib/galaxy/datatypes/registry.py
+2 -2
View File
@@ -2,7 +2,7 @@
Provides mapping between extensions and datatypes, mime-types, etc.
"""
import os, sys, tempfile, threading, logging
import data, tabular, interval, images, sequence, qualityscore, genetics, xml, coverage, tracks, chrominfo, binary, assembly, ngsindex, wsf
import data, tabular, interval, images, sequence, qualityscore, genetics, xml, coverage, tracks, chrominfo, binary, assembly, ngsindex
import galaxy.util
from galaxy.util.odict import odict
from display_applications.application import DisplayApplication
@@ -141,7 +141,7 @@ class Registry( object ):
if hasattr( imported_module, datatype_class_name ):
datatype_class = getattr( imported_module, datatype_class_name )
except Exception, e:
full_path = os.path.join( full_path, proprietary_datatype_module )
full_path = os.path.join( proprietary_path, proprietary_datatype_module )
self.log.debug( "Exception importing proprietary code file %s: %s" % ( str( full_path ), str( e ) ) )
finally:
lock.release()
lib/galaxy/datatypes/tabular.py
+84 -43
View File
@@ -13,34 +13,37 @@ from galaxy.datatypes import metadata
from galaxy.datatypes.metadata import MetadataElement
import galaxy_utils.sequence.vcf
from sniff import *
from galaxy.util.json import to_json_string
log = logging.getLogger(__name__)
class Tabular( data.Text ):
"""Tab delimited data"""
CHUNK_SIZE = 20000
"""Add metadata elements"""
MetadataElement( name="comment_lines", default=0, desc="Number of comment lines", readonly=False, optional=True, no_value=0 )
MetadataElement( name="columns", default=0, desc="Number of columns", readonly=True, visible=False, no_value=0 )
MetadataElement( name="column_types", default=[], desc="Column types", param=metadata.ColumnTypesParameter, readonly=True, visible=False, no_value=[] )
MetadataElement( name="column_names", default=[], desc="Column names", readonly=True, visible=False, optional=True, no_value=[] )
def init_meta( self, dataset, copy_from=None ):
data.Text.init_meta( self, dataset, copy_from=copy_from )
def set_meta( self, dataset, overwrite = True, skip = None, max_data_lines = 100000, **kwd ):
def set_meta( self, dataset, overwrite = True, skip = None, max_data_lines = 100000, max_guess_type_data_lines = None, **kwd ):
"""
Tries to determine the number of columns as well as those columns
that contain numerical values in the dataset. A skip parameter is
used because various tabular data types reuse this function, and
their data type classes are responsible to determine how many invalid
comment lines should be skipped. Using None for skip will cause skip
to be zero, but the first line will be processed as a header. A
max_data_lines parameter is used because various tabular data types
reuse this function, and their data type classes are responsible to
comment lines should be skipped. Using None for skip will cause skip
to be zero, but the first line will be processed as a header. A
max_data_lines parameter is used because various tabular data types
reuse this function, and their data type classes are responsible to
determine how many data lines should be processed to ensure that the
non-optional metadata parameters are properly set; if used, optional
metadata parameters will be set to None, unless the entire file has
already been read. Using None (default) for max_data_lines will
process all data lines.
non-optional metadata parameters are properly set; if used, optional
metadata parameters will be set to None, unless the entire file has
already been read. Using None (default) for max_data_lines will
process all data lines.
Items of interest:
1. We treat 'overwrite' as always True (we always want to set tabular metadata when called).
@@ -57,7 +60,7 @@ class Tabular( data.Text ):
column_type_set_order = [ 'int', 'float', 'list', 'str' ] #Order to set column types in
default_column_type = column_type_set_order[-1] # Default column type is lowest in list
column_type_compare_order = list( column_type_set_order ) #Order to compare column types
column_type_compare_order.reverse()
column_type_compare_order.reverse()
def type_overrules_type( column_type1, column_type2 ):
if column_type1 is None or column_type1 == column_type2:
return False
@@ -74,13 +77,13 @@ class Tabular( data.Text ):
try:
int( column_text )
return True
except:
except:
return False
def is_float( column_text ):
try:
float( column_text )
return True
except:
except:
if column_text.strip().lower() == 'na':
return True #na is special cased to be a float
return False
@@ -116,15 +119,16 @@ class Tabular( data.Text ):
comment_lines += 1
else:
data_lines += 1
fields = line.split( '\t' )
for field_count, field in enumerate( fields ):
if field_count >= len( column_types ): #found a previously unknown column, we append None
column_types.append( None )
column_type = guess_column_type( field )
if type_overrules_type( column_type, column_types[field_count] ):
column_types[field_count] = column_type
if max_guess_type_data_lines is None or data_lines <= max_guess_type_data_lines:
fields = line.split( '\t' )
for field_count, field in enumerate( fields ):
if field_count >= len( column_types ): #found a previously unknown column, we append None
column_types.append( None )
column_type = guess_column_type( field )
if type_overrules_type( column_type, column_types[field_count] ):
column_types[field_count] = column_type
if i == 0 and requested_skip is None:
# This is our first line, people seem to like to upload files that have a header line, but do not
# This is our first line, people seem to like to upload files that have a header line, but do not
# start with '#' (i.e. all column types would then most likely be detected as str). We will assume
# that the first line is always a header (this was previous behavior - it was always skipped). When
# the requested skip is None, we only use the data from the first line if we have no other data for
@@ -146,7 +150,7 @@ class Tabular( data.Text ):
break
i += 1
dataset_fh.close()
#we error on the larger number of columns
#first we pad our column_types by using data from first line
if len( first_line_column_types ) > len( column_types ):
@@ -175,9 +179,19 @@ class Tabular( data.Text ):
except Exception, exc:
out = "Can't create peek %s" % str( exc )
return out
def make_html_peek_header( self, dataset, skipchars=[], column_names=[], column_number_format='%s', column_parameter_alias={}, **kwargs ):
def make_html_peek_header( self, dataset, skipchars=None, column_names=None, column_number_format='%s', column_parameter_alias=None, **kwargs ):
if skipchars is None:
skipchars = []
if column_names is None:
column_names = []
if column_parameter_alias is None:
column_parameter_alias = {}
out = []
try:
if not column_names and dataset.metadata.column_names:
column_names = dataset.metadata.column_names
column_headers = [None] * dataset.metadata.columns
# fill in empty headers with data from column_names
@@ -207,7 +221,10 @@ class Tabular( data.Text ):
except Exception, exc:
raise Exception, "Can't create peek header %s" % str( exc )
return "".join( out )
def make_html_peek_rows( self, dataset, skipchars=[], **kwargs ):
def make_html_peek_rows( self, dataset, skipchars=None, **kwargs ):
if skipchars is None:
skipchars = []
out = []
try:
if not dataset.peek:
@@ -228,6 +245,28 @@ class Tabular( data.Text ):
except Exception, exc:
raise Exception, "Can't create peek rows %s" % str( exc )
return "".join( out )
def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, chunk=None):
#TODO Prevent failure when displaying extremely long > 50kb lines.
if to_ext:
return self._serve_raw(trans, dataset, to_ext)
if chunk:
ck_index = int(chunk)
f = open(dataset.file_name)
f.seek(ck_index * self.CHUNK_SIZE)
# If we aren't at the start of the file, seek to next newline. Do this better eventually.
if f.tell() != 0:
cursor = f.read(1)
while cursor and cursor != '\n':
cursor = f.read(1)
ck_data = f.read(self.CHUNK_SIZE)
cursor = f.read(1)
while cursor and ck_data[-1] != '\n':
ck_data += cursor
cursor = f.read(1)
return to_json_string({'ck_data': ck_data, 'ck_index': ck_index+1})
return trans.fill_template( "/dataset/tabular_chunked.mako",dataset = dataset)
def set_peek( self, dataset, line_count=None, is_multi_byte=False):
super(Tabular, self).set_peek( dataset, line_count=line_count, is_multi_byte=is_multi_byte)
if dataset.metadata.comment_lines:
@@ -276,7 +315,7 @@ class Sam( Tabular ):
def sniff( self, filename ):
"""
Determines whether the file is in SAM format
A file in SAM format consists of lines of tab-separated data.
The following header line may be the first line:
@QNAME FLAG RNAME POS MAPQ CIGAR MRNM MPOS ISIZE SEQ QUAL
@@ -285,12 +324,12 @@ class Sam( Tabular ):
Data in the OPT column is optional and can consist of tab-separated data
For complete details see http://samtools.sourceforge.net/SAM1.pdf
Rules for sniffing as True:
There must be 11 or more columns of data on each line
Columns 2 (FLAG), 4(POS), 5 (MAPQ), 8 (MPOS), and 9 (ISIZE) must be numbers (9 can be negative)
We will only check that up to the first 5 alignments are correctly formatted.
>>> fname = get_test_fname( 'sequence.maf' )
>>> Sam().sniff( fname )
False
@@ -306,7 +345,7 @@ class Sam( Tabular ):
line = line.strip()
if not line:
break #EOF
if line:
if line:
if line[0] != '@':
linePieces = line.split('\t')
if len(linePieces) < 11:
@@ -368,10 +407,10 @@ class Sam( Tabular ):
if result != 0:
raise Exception('Result %s from %s' % (result, cmd))
merge = staticmethod(merge)
def get_track_type( self ):
return "ReadTrack", {"data": "bam", "index": "summary_tree"}
class Pileup( Tabular ):
"""Tab delimited data in pileup (6- or 10-column) format"""
file_ext = "pileup"
@@ -397,7 +436,7 @@ class Pileup( Tabular ):
"""
Checks for 'pileup-ness'
There are two main types of pileup: 6-column and 10-column. For both,
There are two main types of pileup: 6-column and 10-column. For both,
the first three and last two columns are the same. We only check the
first three to allow for some personalization of the format.
@@ -431,27 +470,27 @@ class Pileup( Tabular ):
class ElandMulti( Tabular ):
file_ext = 'elandmulti'
def sniff( self, filename ):
return False
class Vcf( Tabular ):
""" Variant Call Format for describing SNPs and other simple genome variations. """
file_ext = 'vcf'
column_names = [ 'Chrom', 'Pos', 'ID', 'Ref', 'Alt', 'Qual', 'Filter', 'Info', 'Format', 'data' ]
MetadataElement( name="columns", default=10, desc="Number of columns", readonly=True, visible=False )
MetadataElement( name="column_types", default=['str','int','str','str','str','int','str','list','str','str'], param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False )
MetadataElement( name="viz_filter_cols", desc="Score column for visualization", default=[5], param=metadata.ColumnParameter, multiple=True )
def sniff( self, filename ):
headers = get_headers( filename, '\n', count=1 )
return headers[0][0].startswith("##fileformat=VCF")
def display_peek( self, dataset ):
"""Returns formated html of peek"""
return Tabular.make_html_table( self, dataset, column_names=self.column_names )
def get_track_type( self ):
return "VcfTrack", {"data": "tabix", "index": "summary_tree"}
@@ -473,8 +512,10 @@ class Eland( Tabular ):
'POSITION', 'STRAND', 'DESC', 'SRAS', 'PRAS', 'PART_CHROM'
'PART_CONTIG', 'PART_OFFSET', 'PART_STRAND', 'FILT'
]
def make_html_table( self, dataset, skipchars=[] ):
def make_html_table( self, dataset, skipchars=None ):
"""Create HTML table, used for displaying peek"""
if skipchars is None:
skipchars = []
out = ['<table cellspacing="0" cellpadding="3">']
try:
# Generate column header
@@ -495,10 +536,10 @@ class Eland( Tabular ):
def sniff( self, filename ):
"""
Determines whether the file is in ELAND export format
A file in ELAND export format consists of lines of tab-separated data.
There is no header.
Rules for sniffing as True:
There must be 22 columns on each line
LANE, TILEm X, Y, INDEX, READ_NO, SEQ, QUAL, POSITION, *STRAND, FILT must be correct
@@ -517,7 +558,7 @@ class Eland( Tabular ):
line = line.strip()
if not line:
break #EOF
if line:
if line:
linePieces = line.split('\t')
if len(linePieces) != 22:
return False
@@ -563,7 +604,7 @@ class Eland( Tabular ):
#else:
# # Otherwise, read the whole thing and set num data lines.
for i, line in enumerate(dataset_fh):
if line:
if line:
linePieces = line.split('\t')
if len(linePieces) != 22:
raise Exception('%s:%d:Corrupt line!' % (dataset.file_name,i))
@@ -581,5 +622,5 @@ class Eland( Tabular ):
dataset.metadata.tiles = ["%04d" % int(t) for t in tiles.keys()]
dataset.metadata.barcodes = filter(lambda x: x != '0', barcodes.keys()) + ['NoIndex' for x in barcodes.keys() if x == '0']
dataset.metadata.reads = reads.keys()
lib/galaxy/datatypes/util/gff_util.py
+4 -4
View File
@@ -10,8 +10,8 @@ class GFFInterval( GenomicInterval ):
A GFF interval, including attributes. If file is strictly a GFF file,
only attribute is 'group.'
"""
def __init__( self, reader, fields, chrom_col, feature_col, start_col, end_col, \
strand_col, score_col, default_strand, fix_strand=False ):
def __init__( self, reader, fields, chrom_col=0, feature_col=2, start_col=3, end_col=4, \
strand_col=6, score_col=5, default_strand='.', fix_strand=False ):
# HACK: GFF format allows '.' for strand but GenomicInterval does not. To get around this,
# temporarily set strand and then unset after initing GenomicInterval.
unknown_strand = False
@@ -45,8 +45,8 @@ class GFFFeature( GFFInterval ):
"""
A GFF feature, which can include multiple intervals.
"""
def __init__( self, reader, chrom_col, feature_col, start_col, end_col, \
strand_col, score_col, default_strand, fix_strand=False, intervals=[], \
def __init__( self, reader, chrom_col=0, feature_col=2, start_col=3, end_col=4, \
strand_col=6, score_col=5, default_strand='.', fix_strand=False, intervals=[], \
raw_size=0 ):
GFFInterval.__init__( self, reader, intervals[0].fields, chrom_col, feature_col, \
start_col, end_col, strand_col, score_col, default_strand, \
lib/galaxy/datatypes/wsf.py
-156
View File
@@ -1,156 +0,0 @@
"""
SnpFile datatype
"""
import re
import data
from galaxy import util
from galaxy.datatypes.sniff import *
from galaxy.datatypes.tabular import Tabular
from galaxy.datatypes import metadata
from galaxy.datatypes.metadata import MetadataElement
class SnpFile( Tabular ):
""" Webb's SNP file format """
file_ext = 'wsf'
species_regex = re.compile('species=(\S+)')
MetadataElement( name="species", desc="species", default='', no_value='', visible=False, readonly=True )
MetadataElement( name="scaffold", desc="scaffold column", param=metadata.ColumnParameter, default=0 )
MetadataElement( name="pos", desc="pos column", param=metadata.ColumnParameter, default=0 )
MetadataElement( name="ref", desc="ref column", param=metadata.ColumnParameter, default=0 )
MetadataElement( name="rPos", desc="rPos column", param=metadata.ColumnParameter, default=0 )
MetadataElement( name="labels", desc="Number of labels", default=0, no_value=0, visible=False, readonly=True )
MetadataElement( name="label_for_column", desc="Mapping from column to label", default=[], no_value=[], visible=False, readonly=True )
MetadataElement( name="columns_with_label", desc="Mapping from label to columns", param=metadata.DictParameter, default={}, no_value={}, visible=False, readonly=True )
MetadataElement( name="column_headers", desc="Column headers", default=[], no_value=[], visible=False, readonly=True )
def set_meta( self, dataset, overwrite = True, **kwd ):
Tabular.set_meta( self, dataset, overwrite=overwrite, max_data_lines=None, **kwd )
# these two if statements work around a potential bug in metadata.py
if dataset.metadata.labels is None or dataset.metadata.labels == dataset.metadata.spec['labels'].no_value:
self._set_column_labels_metadata( dataset )
if dataset.metadata.column_headers is None or dataset.metadata.column_headers == dataset.metadata.spec['column_headers'].no_value:
self._set_column_headers_metadata( dataset )
self._set_columnParameter_metadata( dataset )
def _set_column_labels_metadata( self, dataset ):
def build_map_from_label_to_comma_separated_column_list( labels ):
map = {}
for index, label in enumerate( labels ):
map.setdefault( label, [] ).append( index )
for label in map:
map[label] = ','.join( [ str( index + 1 ) for index in map[label] ] )
return map
def strip_list_elements( list ):
return [ element.strip() for element in list ]
def initial_comment_lines_of_dataset( dataset ):
comment_lines = []
if dataset.has_data():
try:
fh = open( dataset.file_name, 'r' )
for line in fh:
if not line.startswith('#'):
break
line = line[1:]
line = line.rstrip( '\r\n' )
if line:
comment_lines.append( line )
fh.close()
except:
pass
return comment_lines
def set_metadata_from_comment_lines( dataset ):
labels = []
comment_lines = initial_comment_lines_of_dataset( dataset )
for line in comment_lines:
match = SnpFile.species_regex.match( line )
if match:
dataset.metadata.species = match.group(1)
continue
elems = line.split( '\t' )
if len(elems) > 1:
labels = strip_list_elements( elems )
dataset.metadata.labels = len( labels )
dataset.metadata.label_for_column = labels[:]
if labels:
dataset.metadata.label_for_column.insert(0, '')
dataset.metadata.columns_with_label = build_map_from_label_to_comma_separated_column_list( labels )
set_metadata_from_comment_lines( dataset )
def _set_column_headers_metadata( self, dataset ):
if dataset.metadata.labels < dataset.metadata.columns:
column_headers = dataset.metadata.label_for_column[1:] + [ '' ] * ( dataset.metadata.columns - dataset.metadata.labels )
else:
column_headers = dataset.metadata.label_for_column[1:dataset.metadata.columns+1]
dataset.metadata.column_headers = column_headers
def _set_columnParameter_metadata( self, dataset ):
def unique_column_number_or_zero( string ):
try:
val = int( string )
except:
val = 0
return val
for name in self._metadata_columnParameter_names( dataset ):
if name in dataset.metadata.columns_with_label:
if dataset.metadata.columns_with_label[name]:
column = unique_column_number_or_zero( dataset.metadata.columns_with_label[name] )
if column:
setattr( dataset.metadata, name, column )
def _metadata_columnParameter_names( self, dataset ):
for name, spec in dataset.metadata.spec.items():
if isinstance( spec.param, metadata.ColumnParameter ):
yield name
def set_peek( self, dataset, line_count=None, is_multi_byte=False ):
super(Tabular, self).set_peek( dataset, line_count=line_count, is_multi_byte=is_multi_byte, skipchars=[ '#' ])
def make_html_table( self, dataset, skipchars=[ '#' ] ):
"""Create HTML table, used for displaying peek"""
def table_header_values( dataset ):
headers = dataset.metadata.column_headers[:]
for name in self._metadata_columnParameter_names( dataset ):
col = getattr( dataset.metadata, name )
assert col <= dataset.metadata.columns, Exception( 'ColumnParameter %s %d > %d columns for dataset %s.' % ( name, col, dataset.metadata.columns, dataset.id ) )
if col > 0:
headers[ col - 1 ] = name
return headers
def table_headers( dataset ):
out = [ '<tr>' ]
headers = table_header_values( dataset )
for index, header in enumerate( headers ):
column = index + 1
if header:
out.append( "<th>%d.%s</th>" % ( column, header ) )
else:
out.append( "<th>%d.</th>" % column )
out.append( '</tr>' )
return out
try:
out = ['<table cellspacing="0" cellpadding="3">']
out.extend( table_headers( dataset ) )
out.append( self.make_html_peek_rows( dataset, skipchars=skipchars ) )
out.append( '</table>' )
out = "".join( out )
except Exception, exc:
out = "Can't create peek %s" % exc
return out
lib/galaxy/datatypes/xml.py
+31 -3
View File
@@ -104,13 +104,41 @@ class BlastXml( GenericXml ):
for f in split_files:
h = open(f)
body = False
header = []
header = h.readline()
if not header:
out.close()
h.close()
raise ValueError("BLAST XML file %s was empty" % f)
if header.strip() != '<?xml version="1.0"?>':
out.write(header) #for diagnosis
out.close()
h.close()
raise ValueError("%s is not an XML file!" % f)
line = h.readline()
header += line
if line.strip() not in ['<!DOCTYPE BlastOutput PUBLIC "-//NCBI//NCBI BlastOutput/EN" "http://www.ncbi.nlm.nih.gov/dtd/NCBI_BlastOutput.dtd">',
'<!DOCTYPE BlastOutput PUBLIC "-//NCBI//NCBI BlastOutput/EN" "NCBI_BlastOutput.dtd">']:
out.write(header) #for diagnosis
out.close()
h.close()
raise ValueError("%s is not a BLAST XML file!" % f)
while True:
line = h.readline()
header.append(line)
if not line:
out.write(header) #for diagnosis
out.close()
h.close()
raise ValueError("BLAST XML file %s ended prematurely" % f)
header += line
if "<Iteration>" in line:
break
header = "".join(header)
if len(header) > 10000:
#Something has gone wrong, don't load too much into memory!
#Write what we have to the merged file for diagnostics
out.write(header)
out.close()
h.close()
raise ValueError("BLAST XML file %s has too long a header!" % f)
if "<BlastOutput>" not in header:
out.close()
h.close()
lib/galaxy/jobs/__init__.py
+34 -447
View File
@@ -1,4 +1,15 @@
import logging, threading, sys, os, time, traceback, shutil
"""
Support for running a tool in Galaxy via an internal job management system
"""
import os
import sys
import pwd
import time
import logging
import threading
import traceback
import subprocess
import galaxy
from galaxy import util, model
@@ -9,51 +20,16 @@ from galaxy.datatypes import metadata
from galaxy.util.json import from_json_string
from galaxy.util.expressions import ExpressionContext
from galaxy.jobs.actions.post import ActionBox
import subprocess, pwd
from galaxy.exceptions import ObjectInvalid
from sqlalchemy.sql.expression import and_, or_
import pkg_resources
pkg_resources.require( "PasteDeploy" )
from Queue import Queue, Empty
log = logging.getLogger( __name__ )
# States for running a job. These are NOT the same as data states
JOB_WAIT, JOB_ERROR, JOB_INPUT_ERROR, JOB_INPUT_DELETED, JOB_READY, JOB_DELETED, JOB_ADMIN_DELETED = 'wait', 'error', 'input_error', 'input_deleted', 'ready', 'deleted', 'admin_deleted'
# This file, if created in the job's working directory, will be used for
# setting advanced metadata properties on the job and its associated outputs.
# This interface is currently experimental, is only used by the upload tool,
# and should eventually become API'd
TOOL_PROVIDED_JOB_METADATA_FILE = 'galaxy.json'
class JobManager( object ):
"""
Highest level interface to job management.
TODO: Currently the app accesses "job_queue" and "job_stop_queue" directly.
This should be decoupled.
"""
def __init__( self, app ):
self.app = app
if self.app.config.get_bool( "enable_job_running", True ):
# The dispatcher launches the underlying job runners
self.dispatcher = DefaultJobDispatcher( app )
# Queues for starting and stopping jobs
self.job_queue = JobQueue( app, self.dispatcher )
self.job_stop_queue = JobStopQueue( app, self.dispatcher )
if self.app.config.enable_beta_job_managers:
from galaxy.jobs.deferred import DeferredJobQueue
self.deferred_job_queue = DeferredJobQueue( app )
else:
self.job_queue = self.job_stop_queue = NoopQueue()
def shutdown( self ):
self.job_queue.shutdown()
self.job_stop_queue.shutdown()
class Sleeper( object ):
"""
Provides a 'sleep' method that sleeps for a number of seconds *unless*
@@ -70,238 +46,6 @@ class Sleeper( object ):
self.condition.notify()
self.condition.release()
class JobQueue( object ):
"""
Job manager, waits for jobs to be runnable and then dispatches to
a JobRunner.
"""
STOP_SIGNAL = object()
def __init__( self, app, dispatcher ):
"""Start the job manager"""
self.app = app
self.sa_session = app.model.context
self.job_lock = False
# Should we read jobs form the database, or use an in memory queue
self.track_jobs_in_database = app.config.get_bool( 'track_jobs_in_database', False )
# Keep track of the pid that started the job manager, only it
# has valid threads
self.parent_pid = os.getpid()
# Contains new jobs. Note this is not used if track_jobs_in_database is True
self.queue = Queue()
# Contains jobs that are waiting (only use from monitor thread)
## This and jobs_to_check[] are closest to a "Job Queue"
self.waiting_jobs = []
# Helper for interruptable sleep
self.sleeper = Sleeper()
self.running = True
self.dispatcher = dispatcher
self.monitor_thread = threading.Thread( target=self.__monitor )
# Recover jobs at startup
if app.config.get_bool( 'enable_job_recovery', True ):
self.__check_jobs_at_startup()
# Start the queue
self.monitor_thread.start()
log.info( "job manager started" )
def __check_jobs_at_startup( self ):
"""
Checks all jobs that are in the 'new', 'queued' or 'running' state in
the database and requeues or cleans up as necessary. Only run as the
job manager starts.
"""
model = self.app.model # DBTODO Why?
for job in self.sa_session.query( model.Job ).filter( model.Job.state == model.Job.states.NEW ):
if job.tool_id not in self.app.toolbox.tools_by_id:
log.warning( "Tool '%s' removed from tool config, unable to recover job: %s" % ( job.tool_id, job.id ) )
JobWrapper( job, self ).fail( 'This tool was disabled before the job completed. Please contact your Galaxy administrator, or' )
else:
log.debug( "no runner: %s is still in new state, adding to the jobs queue" %job.id )
self.queue.put( ( job.id, job.tool_id ) )
for job in self.sa_session.query( model.Job ).enable_eagerloads( False ).filter( ( model.Job.state == model.Job.states.RUNNING ) | ( model.Job.state == model.Job.states.QUEUED ) ):
if job.tool_id not in self.app.toolbox.tools_by_id:
log.warning( "Tool '%s' removed from tool config, unable to recover job: %s" % ( job.tool_id, job.id ) )
JobWrapper( job, self ).fail( 'This tool was disabled before the job completed. Please contact your Galaxy administrator, or' )
elif job.job_runner_name is None:
log.debug( "no runner: %s is still in queued state, adding to the jobs queue" %job.id )
if self.track_jobs_in_database:
job.state = model.Job.states.NEW
else:
self.queue.put( ( job.id, job.tool_id ) )
else:
job_wrapper = JobWrapper( job, self )
self.dispatcher.recover( job, job_wrapper )
if self.sa_session.dirty:
self.sa_session.flush()
def __monitor( self ):
"""
Continually iterate the waiting jobs, checking is each is ready to
run and dispatching if so.
"""
# HACK: Delay until after forking, we need a way to do post fork notification!!!
time.sleep( 10 )
while self.running:
try:
self.__monitor_step()
except:
log.exception( "Exception in monitor_step" )
# Sleep
self.sleeper.sleep( 1 )
def __monitor_step( self ):
"""
Called repeatedly by `monitor` to process waiting jobs. Gets any new
jobs (either from the database or from its own queue), then iterates
over all new and waiting jobs to check the state of the jobs each
depends on. If the job has dependencies that have not finished, it
it goes to the waiting queue. If the job has dependencies with errors,
it is marked as having errors and removed from the queue. Otherwise,
the job is dispatched.
"""
# Pull all new jobs from the queue at once
jobs_to_check = []
if self.track_jobs_in_database:
# Clear the session so we get fresh states for job and all datasets
self.sa_session.expunge_all()
# Fetch all new jobs
jobs_to_check = self.sa_session.query( model.Job ).enable_eagerloads( False ) \
.filter( model.Job.state == model.Job.states.NEW ).all()
else:
# Get job objects and append to watch queue for any which were
# previously waiting
for job_id in self.waiting_jobs:
jobs_to_check.append( self.sa_session.query( model.Job ).get( job_id ) )
try:
while 1:
message = self.queue.get_nowait()
if message is self.STOP_SIGNAL:
return
# Unpack the message
job_id, tool_id = message
# Get the job object and append to watch queue
jobs_to_check.append( self.sa_session.query( model.Job ).get( job_id ) )
except Empty:
pass
# Iterate over new and waiting jobs and look for any that are
# ready to run
new_waiting_jobs = []
for job in jobs_to_check:
try:
# Check the job's dependencies, requeue if they're not done
job_state = self.__check_if_ready_to_run( job )
if job_state == JOB_WAIT:
if not self.track_jobs_in_database:
new_waiting_jobs.append( job.id )
elif job_state == JOB_INPUT_ERROR:
log.info( "job %d unable to run: one or more inputs in error state" % job.id )
elif job_state == JOB_INPUT_DELETED:
log.info( "job %d unable to run: one or more inputs deleted" % job.id )
elif job_state == JOB_READY:
if self.job_lock:
log.info( "Job dispatch attempted for %s, but prevented by administrative lock." % job.id )
if not self.track_jobs_in_database:
new_waiting_jobs.append( job.id )
else:
self.dispatcher.put( JobWrapper( job, self ) )
log.info( "job %d dispatched" % job.id )
elif job_state == JOB_DELETED:
log.info( "job %d deleted by user while still queued" % job.id )
elif job_state == JOB_ADMIN_DELETED:
log.info( "job %d deleted by admin while still queued" % job.id )
else:
log.error( "unknown job state '%s' for job %d" % ( job_state, job.id ) )
if not self.track_jobs_in_database:
new_waiting_jobs.append( job.id )
except Exception:
log.exception( "failure running job %d" % job.id )
# Update the waiting list
self.waiting_jobs = new_waiting_jobs
# Done with the session
self.sa_session.remove()
def __check_if_ready_to_run( self, job ):
"""
Check if a job is ready to run by verifying that each of its input
datasets is ready (specifically in the OK state). If any input dataset
has an error, fail the job and return JOB_INPUT_ERROR. If any input
dataset is deleted, fail the job and return JOB_INPUT_DELETED. If all
input datasets are in OK state, return JOB_READY indicating that the
job can be dispatched. Otherwise, return JOB_WAIT indicating that input
datasets are still being prepared.
"""
if job.state == model.Job.states.DELETED:
return JOB_DELETED
elif job.state == model.Job.states.ERROR:
return JOB_ADMIN_DELETED
elif self.app.config.enable_quotas:
quota = self.app.quota_agent.get_quota( job.user )
if quota is not None:
try:
usage = self.app.quota_agent.get_usage( user=job.user, history=job.history )
if usage > quota:
return JOB_WAIT
except AssertionError, e:
pass # No history, should not happen with an anon user
for dataset_assoc in job.input_datasets + job.input_library_datasets:
idata = dataset_assoc.dataset
if not idata:
continue
# don't run jobs for which the input dataset was deleted
if idata.deleted:
JobWrapper( job, self ).fail( "input data %d (file: %s) was deleted before the job started" % ( idata.hid, idata.file_name ) )
return JOB_INPUT_DELETED
# an error in the input data causes us to bail immediately
elif idata.state == idata.states.ERROR:
JobWrapper( job, self ).fail( "input data %d is in error state" % ( idata.hid ) )
return JOB_INPUT_ERROR
elif idata.state == idata.states.FAILED_METADATA:
JobWrapper( job, self ).fail( "input data %d failed to properly set metadata" % ( idata.hid ) )
return JOB_INPUT_ERROR
elif idata.state != idata.states.OK and not ( idata.state == idata.states.SETTING_METADATA and job.tool_id is not None and job.tool_id == self.app.datatypes_registry.set_external_metadata_tool.id ):
# need to requeue
return JOB_WAIT
return self.__check_user_jobs( job )
def __check_user_jobs( self, job ):
if not self.app.config.user_job_limit:
return JOB_READY
if job.user:
count = self.sa_session.query( model.Job ).enable_eagerloads( False ) \
.filter( and_( model.Job.user_id == job.user.id,
or_( model.Job.state == model.Job.states.RUNNING,
model.Job.state == model.Job.states.QUEUED ) ) ).count()
elif job.galaxy_session:
count = self.sa_session.query( model.Job ).enable_eagerloads( False ) \
.filter( and_( model.Job.session_id == job.galaxy_session.id,
or_( model.Job.state == model.Job.states.RUNNING,
model.Job.state == model.Job.states.QUEUED ) ) ).count()
else:
log.warning( 'Job %s is not associated with a user or session so job concurrency limit cannot be checked.' % job.id )
return JOB_READY
if count >= self.app.config.user_job_limit:
return JOB_WAIT
return JOB_READY
def put( self, job_id, tool ):
"""Add a job to the queue (by job identifier)"""
if not self.track_jobs_in_database:
self.queue.put( ( job_id, tool.id ) )
self.sleeper.wake()
def shutdown( self ):
"""Attempts to gracefully shut down the worker thread"""
if self.parent_pid != os.getpid():
# We're not the real job queue, do nothing
return
else:
log.info( "sending stop signal to worker thread" )
self.running = False
if not self.track_jobs_in_database:
self.queue.put( self.STOP_SIGNAL )
self.sleeper.wake()
log.info( "job queue stopped" )
self.dispatcher.shutdown()
class JobWrapper( object ):
"""
Wraps a 'model.Job' with convenience methods for running processes and
@@ -400,9 +144,12 @@ class JobWrapper( object ):
self.file_name = dataset.file_name
self.metadata = dict()
self.children = []
jeha = self.sa_session.query( model.JobExportHistoryArchive ).filter_by( job=job ).first()
if jeha:
out_data[ "output_file" ] = FakeDatasetAssociation( dataset=jeha.dataset )
special = self.sa_session.query( model.JobExportHistoryArchive ).filter_by( job=job ).first()
if not special:
special = self.sa_session.query( model.GenomeIndexToolData ).filter_by( job=job ).first()
if special:
out_data[ "output_file" ] = FakeDatasetAssociation( dataset=special.dataset )
# These can be passed on the command line if wanted as $userId $userEmail
if job.history and job.history.user: # check for anonymous user!
userId = '%d' % job.history.user.id
@@ -698,8 +445,11 @@ class JobWrapper( object ):
param_dict = self.tool.params_from_strings( param_dict, self.app )
# Check for and move associated_files
self.tool.collect_associated_files(out_data, self.working_directory)
gitd = self.sa_session.query( model.GenomeIndexToolData ).filter_by( job=job ).first()
if gitd:
self.tool.collect_associated_files({'' : gitd}, self.working_directory)
# Create generated output children and primary datasets and add to param_dict
collected_datasets = {'children':self.tool.collect_child_datasets(out_data),'primary':self.tool.collect_primary_datasets(out_data)}
collected_datasets = {'children':self.tool.collect_child_datasets(out_data, self.working_directory),'primary':self.tool.collect_primary_datasets(out_data, self.working_directory)}
param_dict.update({'__collected_datasets__':collected_datasets})
# Certain tools require tasks to be completed after job execution
# ( this used to be performed in the "exec_after_process" hook, but hooks are deprecated ).
@@ -736,6 +486,7 @@ class JobWrapper( object ):
self.external_output_metadata.cleanup_external_metadata( self.sa_session )
galaxy.tools.imp_exp.JobExportHistoryArchiveWrapper( self.job_id ).cleanup_after_job( self.sa_session )
galaxy.tools.imp_exp.JobImportHistoryArchiveWrapper( self.job_id ).cleanup_after_job( self.sa_session )
galaxy.tools.genome_index.GenomeIndexToolWrapper( self.job_id ).postprocessing( self.sa_session, self.app )
self.app.object_store.delete(self.get_job(), base_dir='job_work', entire_dir=True, dir_only=True, extra_dir=str(self.job_id))
except:
log.exception( "Unable to cleanup job %d" % self.job_id )
@@ -791,9 +542,11 @@ class JobWrapper( object ):
else:
return self.false_path
job = self.get_job()
# Job output datasets are combination of output datasets, library datasets, and jeha datasets.
jeha = self.sa_session.query( model.JobExportHistoryArchive ).filter_by( job=job ).first()
jeha_false_path = None
# Job output datasets are combination of history, library, jeha and gitd datasets.
special = self.sa_session.query( model.JobExportHistoryArchive ).filter_by( job=job ).first()
if not special:
special = self.sa_session.query( model.GenomeIndexToolData ).filter_by( job=job ).first()
false_path = None
if self.app.config.outputs_to_working_directory:
self.output_paths = []
self.output_hdas_and_paths = {}
@@ -802,14 +555,14 @@ class JobWrapper( object ):
dsp = DatasetPath( hda.dataset.id, hda.dataset.file_name, false_path )
self.output_paths.append( dsp )
self.output_hdas_and_paths[name] = hda, dsp
if jeha:
jeha_false_path = os.path.abspath( os.path.join( self.working_directory, "galaxy_dataset_%d.dat" % jeha.dataset.id ) )
if special:
false_path = os.path.abspath( os.path.join( self.working_directory, "galaxy_dataset_%d.dat" % special.dataset.id ) )
else:
results = [ ( da.name, da.dataset, DatasetPath( da.dataset.dataset.id, da.dataset.file_name ) ) for da in job.output_datasets + job.output_library_datasets ]
self.output_paths = [t[2] for t in results]
self.output_hdas_and_paths = dict([(t[0], t[1:]) for t in results])
if jeha:
dsp = DatasetPath( jeha.dataset.id, jeha.dataset.file_name, jeha_false_path )
if special:
dsp = DatasetPath( special.dataset.id, special.dataset.file_name, false_path )
self.output_paths.append( dsp )
return self.output_paths
@@ -1177,179 +930,13 @@ class TaskWrapper(JobWrapper):
# There is no metadata setting for tasks. This is handled after the merge, at the job level.
return ""
class DefaultJobDispatcher( object ):
def __init__( self, app ):
self.app = app
self.job_runners = {}
start_job_runners = ["local"]
if app.config.start_job_runners is not None:
start_job_runners.extend( app.config.start_job_runners.split(",") )
if app.config.use_tasked_jobs:
start_job_runners.append("tasks")
for name in start_job_runners:
self._load_plugin( name )
def _load_plugin( self, name ):
module_name = 'galaxy.jobs.runners.' + name
try:
module = __import__( module_name )
except:
log.exception( 'Job runner is not loadable: %s' % module_name )
return
for comp in module_name.split( "." )[1:]:
module = getattr( module, comp )
if '__all__' not in dir( module ):
log.error( 'Runner "%s" does not contain a list of exported classes in __all__' % module_name )
return
for obj in module.__all__:
display_name = ':'.join( ( module_name, obj ) )
runner = getattr( module, obj )
self.job_runners[name] = runner( self.app )
log.debug( 'Loaded job runner: %s' % display_name )
def __get_runner_name( self, job_wrapper ):
if self.app.config.use_tasked_jobs and job_wrapper.tool.parallelism is not None and not isinstance(job_wrapper, TaskWrapper):
runner_name = "tasks"
else:
runner_name = ( job_wrapper.get_job_runner().split(":", 1) )[0]
return runner_name
def put( self, job_wrapper ):
try:
runner_name = self.__get_runner_name( job_wrapper )
if self.app.config.use_tasked_jobs and job_wrapper.tool.parallelism is not None and isinstance(job_wrapper, TaskWrapper):
#DBTODO Refactor
log.debug( "dispatching task %s, of job %d, to %s runner" %( job_wrapper.task_id, job_wrapper.job_id, runner_name ) )
else:
log.debug( "dispatching job %d to %s runner" %( job_wrapper.job_id, runner_name ) )
self.job_runners[runner_name].put( job_wrapper )
except KeyError:
log.error( 'put(): (%s) Invalid job runner: %s' % ( job_wrapper.job_id, runner_name ) )
job_wrapper.fail( 'Unable to run job due to a misconfiguration of the Galaxy job running system. Please contact a site administrator.' )
def stop( self, job ):
runner_name = ( job.job_runner_name.split(":", 1) )[0]
log.debug( "stopping job %d in %s runner" %( job.id, runner_name ) )
try:
self.job_runners[runner_name].stop_job( job )
except KeyError:
log.error( 'stop(): (%s) Invalid job runner: %s' % ( job_wrapper.job_id, runner_name ) )
# Job and output dataset states have already been updated, so nothing is done here.
def recover( self, job, job_wrapper ):
runner_name = ( job.job_runner_name.split(":", 1) )[0]
log.debug( "recovering job %d in %s runner" %( job.id, runner_name ) )
try:
self.job_runners[runner_name].recover( job, job_wrapper )
except KeyError:
log.error( 'recover(): (%s) Invalid job runner: %s' % ( job_wrapper.job_id, runner_name ) )
job_wrapper.fail( 'Unable to run job due to a misconfiguration of the Galaxy job running system. Please contact a site administrator.' )
def shutdown( self ):
for runner in self.job_runners.itervalues():
runner.shutdown()
class JobStopQueue( object ):
"""
A queue for jobs which need to be terminated prematurely.
"""
STOP_SIGNAL = object()
def __init__( self, app, dispatcher ):
self.app = app
self.sa_session = app.model.context
self.dispatcher = dispatcher
self.track_jobs_in_database = app.config.get_bool( 'track_jobs_in_database', False )
# Keep track of the pid that started the job manager, only it
# has valid threads
self.parent_pid = os.getpid()
# Contains new jobs. Note this is not used if track_jobs_in_database is True
self.queue = Queue()
# Contains jobs that are waiting (only use from monitor thread)
self.waiting = []
# Helper for interruptable sleep
self.sleeper = Sleeper()
self.running = True
self.monitor_thread = threading.Thread( target=self.monitor )
self.monitor_thread.start()
log.info( "job stopper started" )
def monitor( self ):
"""
Continually iterate the waiting jobs, stop any that are found.
"""
# HACK: Delay until after forking, we need a way to do post fork notification!!!
time.sleep( 10 )
while self.running:
try:
self.monitor_step()
except:
log.exception( "Exception in monitor_step" )
# Sleep
self.sleeper.sleep( 1 )
def monitor_step( self ):
"""
Called repeatedly by `monitor` to stop jobs.
"""
# Pull all new jobs from the queue at once
jobs_to_check = []
if self.track_jobs_in_database:
# Clear the session so we get fresh states for job and all datasets
self.sa_session.expunge_all()
# Fetch all new jobs
newly_deleted_jobs = self.sa_session.query( model.Job ).enable_eagerloads( False ) \
.filter( model.Job.state == model.Job.states.DELETED_NEW ).all()
for job in newly_deleted_jobs:
jobs_to_check.append( ( job, None ) )
# Also pull from the queue (in the case of Administrative stopped jobs)
try:
while 1:
message = self.queue.get_nowait()
if message is self.STOP_SIGNAL:
return
# Unpack the message
job_id, error_msg = message
# Get the job object and append to watch queue
jobs_to_check.append( ( self.sa_session.query( model.Job ).get( job_id ), error_msg ) )
except Empty:
pass
for job, error_msg in jobs_to_check:
if error_msg is not None:
job.state = job.states.ERROR
job.info = error_msg
else:
job.state = job.states.DELETED
self.sa_session.add( job )
self.sa_session.flush()
if job.job_runner_name is not None:
# tell the dispatcher to stop the job
self.dispatcher.stop( job )
def put( self, job_id, error_msg=None ):
self.queue.put( ( job_id, error_msg ) )
def shutdown( self ):
"""Attempts to gracefully shut down the worker thread"""
if self.parent_pid != os.getpid():
# We're not the real job queue, do nothing
return
else:
log.info( "sending stop signal to worker thread" )
self.running = False
if not self.track_jobs_in_database:
self.queue.put( self.STOP_SIGNAL )
self.sleeper.wake()
log.info( "job stopper stopped" )
class NoopQueue( object ):
"""
Implements the JobQueue / JobStopQueue interface but does nothing
"""
def put( self, *args ):
return
def put_stop( self, *args ):
return
def shutdown( self ):
return
lib/galaxy/jobs/actions/post.py
+1 -2
View File
@@ -203,11 +203,10 @@ class RenameDatasetAction(DefaultJobAction):
@classmethod
def get_config_form(cls, trans):
form = """
if ((pja.action_arguments !== undefined) && (pja.action_arguments.newname !== undefined)){
if (pja.action_arguments && pja.action_arguments.newname){
p_str += "<label for='pja__"+pja.output_name+"__RenameDatasetAction__newname'>New output name:</label>\
<input type='text' name='pja__"+pja.output_name+"__RenameDatasetAction__newname' value=\\"" + pja.action_arguments.newname.replace(/"/g, "&quot;") + "\\"/>";
}
else{
p_str += "<label for='pja__"+pja.output_name+"__RenameDatasetAction__newname'>New output name:</label>\
<input type='text' name='pja__"+pja.output_name+"__RenameDatasetAction__newname' value=''/>";
lib/galaxy/jobs/deferred/deferred_indexing.py
View File
lib/galaxy/jobs/deferred/genome_transfer.py
+250
View File
@@ -0,0 +1,250 @@
"""
Module for managing genome transfer jobs.
"""
import logging, shutil, gzip, bz2, zipfile, tempfile, tarfile, sys
from galaxy import eggs
from sqlalchemy import and_
from galaxy.util.odict import odict
from galaxy.workflow.modules import module_factory
from galaxy.jobs.actions.post import ActionBox
from galaxy.tools.parameters import visit_input_values
from galaxy.tools.parameters.basic import DataToolParameter
from galaxy.tools.data import ToolDataTableManager
from galaxy.datatypes.checkers import *
from galaxy.datatypes.sequence import Fasta
from data_transfer import *
log = logging.getLogger( __name__ )
__all__ = [ 'GenomeTransferPlugin' ]
class GenomeTransferPlugin( DataTransfer ):
locations = {}
def __init__( self, app ):
super( GenomeTransferPlugin, self ).__init__( app )
self.app = app
self.tool = app.toolbox.tools_by_id['__GENOME_INDEX__']
self.sa_session = app.model.context.current
tdtman = ToolDataTableManager()
xmltree = tdtman.load_from_config_file(app.config.tool_data_table_config_path)
for node in xmltree:
table = node.get('name')
location = node.findall('file')[0].get('path')
self.locations[table] = location
def create_job( self, trans, url, dbkey, intname, indexes ):
job = trans.app.transfer_manager.new( protocol='http', url=url )
params = dict( user=trans.user.id, transfer_job_id=job.id, protocol='http', type='init_transfer', url=url, dbkey=dbkey, indexes=indexes, intname=intname, liftover=None )
deferred = trans.app.model.DeferredJob( state = self.app.model.DeferredJob.states.NEW, plugin = 'GenomeTransferPlugin', params = params )
self.sa_session.add( deferred )
self.sa_session.flush()
return deferred.id
def check_job( self, job ):
if job.params['type'] == 'init_transfer':
if not hasattr(job, 'transfer_job'):
job.transfer_job = self.sa_session.query( self.app.model.TransferJob ).get( int( job.params[ 'transfer_job_id' ] ) )
else:
self.sa_session.refresh( job.transfer_job )
if job.transfer_job.state == 'done':
transfer = job.transfer_job
transfer.state = 'downloaded'
job.params['type'] = 'extract_transfer'
self.sa_session.add( job )
self.sa_session.add( transfer )
self.sa_session.flush()
return self.job_states.READY
elif job.transfer_job.state == 'running':
return self.job_states.WAIT
elif job.transfer_job.state == 'new':
assert job.params[ 'protocol' ] in [ 'http', 'ftp', 'https' ], 'Unknown protocol %s' % job.params[ 'protocol' ]
self.app.transfer_manager.run( job.transfer_job )
self.sa_session.add( job.transfer_job )
self.sa_session.flush()
return self.job_states.WAIT
else:
log.error( "An error occurred while downloading from %s" % job.params[ 'url' ] )
return self.job_states.INVALID
elif job.params[ 'type' ] == 'extract_transfer':
return self.job_states.READY
def get_job_status( self, jobid ):
job = self.sa_session.query( self.app.model.DeferredJob ).get( int( jobid ) )
if not hasattr( job, 'transfer_job' ):
job.transfer_job = self.sa_session.query( self.app.model.TransferJob ).get( int( job.params[ 'transfer_job_id' ] ) )
else:
self.sa_session.refresh( job.transfer_job )
return job
def run_job( self, job ):
params = job.params
dbkey = params[ 'dbkey' ]
if not hasattr( job, 'transfer_job' ):
job.transfer_job = self.sa_session.query( self.app.model.TransferJob ).get( int( job.params[ 'transfer_job_id' ] ) )
else:
self.sa_session.refresh( job.transfer_job )
transfer = job.transfer_job
if params[ 'type' ] == 'extract_transfer':
CHUNK_SIZE = 2**20
destpath = os.path.join( self.app.config.get( 'genome_data_path', 'tool-data/genome' ), job.params[ 'dbkey' ], 'seq' )
destfile = '%s.fa' % job.params[ 'dbkey' ]
destfilepath = os.path.join( destpath, destfile )
tmpprefix = '%s_%s_download_unzip_' % ( job.params['dbkey'], job.params[ 'transfer_job_id' ] )
tmppath = os.path.dirname( os.path.abspath( transfer.path ) )
if not os.path.exists( destpath ):
os.makedirs( destpath )
protocol = job.params[ 'protocol' ]
data_type = self._check_compress( transfer.path )
if data_type is None:
sniffer = Fasta()
if sniffer.sniff( transfer.path ):
data_type = 'fasta'
fd, uncompressed = tempfile.mkstemp( prefix=tmpprefix, dir=tmppath, text=False )
if data_type in [ 'tar.gzip', 'tar.bzip' ]:
fp = open( transfer.path, 'r' )
tar = tarfile.open( mode = 'r:*', bufsize = CHUNK_SIZE, fileobj = fp )
files = tar.getmembers()
for filename in files:
z = tar.extractfile(filename)
try:
chunk = z.read( CHUNK_SIZE )
except IOError:
os.close( fd )
log.error( 'Problem decompressing compressed data' )
exit()
if not chunk:
break
os.write( fd, chunk )
os.write( fd, '\n' )
os.close( fd )
tar.close()
fp.close()
elif data_type == 'gzip':
compressed = gzip.open( transfer.path, mode = 'rb' )
while 1:
try:
chunk = compressed.read( CHUNK_SIZE )
except IOError:
compressed.close()
log.error( 'Problem decompressing compressed data' )
exit()
if not chunk:
break
os.write( fd, chunk )
os.write( fd, '\n' )
os.close( fd )
compressed.close()
elif data_type == 'bzip':
compressed = bz2.BZ2File( transfer.path, mode = 'r' )
while 1:
try:
chunk = compressed.read( CHUNK_SIZE )
except IOError:
compressed.close()
log.error( 'Problem decompressing compressed data' )
exit()
if not chunk:
break
os.write( fd, chunk )
os.write( fd, '\n' )
os.close( fd )
compressed.close()
elif data_type == 'zip':
uncompressed_name = None
unzipped = False
z = zipfile.ZipFile( transfer.path )
z.debug = 3
for name in z.namelist():
if name.endswith('/'):
continue
if sys.version_info[:2] >= ( 2, 6 ):
zipped_file = z.open( name )
while 1:
try:
chunk = zipped_file.read( CHUNK_SIZE )
except IOError:
os.close( fd )
log.error( 'Problem decompressing zipped data' )
return self.app.model.DeferredJob.states.INVALID
if not chunk:
break
os.write( fd, chunk )
os.write( fd, '\n' )
zipped_file.close()
else:
try:
outfile = open( fd, 'wb' )
outfile.write( z.read( name ) )
outfile.close()
except IOError:
os.close( fd )
log.error( 'Problem decompressing zipped data' )
return
os.close( fd )
z.close()
elif data_type == 'fasta':
uncompressed = transfer.path
else:
job.state = self.app.model.DeferredJob.states.INVALID
log.error( "Unrecognized compression format for file %s." % transfer.path )
self.sa_session.add( job )
self.sa_session.flush()
return
shutil.move( uncompressed, destfilepath )
if os.path.exists( transfer.path ):
os.remove( transfer.path )
os.chmod( destfilepath, 0644 )
fastaline = '\t'.join( [ dbkey, dbkey, params[ 'intname' ], os.path.abspath( destfilepath ) ] )
self._add_line( 'all_fasta', fastaline )
job.state = self.app.model.DeferredJob.states.OK
job.params[ 'type' ] = 'finish_transfer'
transfer.path = os.path.abspath(destfilepath)
transfer.state = 'done'
self.sa_session.add( job )
self.sa_session.add( transfer )
self.sa_session.flush()
if transfer.state == 'done' and params[ 'indexes' ] is not None:
for indexer in params[ 'indexes' ]:
incoming = dict(indexer=indexer, dbkey=params[ 'dbkey' ], intname=params[ 'intname' ], path=transfer.path, user=params['user'] )
deferred = self.tool.execute( self, set_output_hid=False, history=None, incoming=incoming, transfer=transfer, deferred=job )
return self.app.model.DeferredJob.states.OK
def _check_compress( self, filepath ):
retval = ''
if tarfile.is_tarfile( filepath ):
retval = 'tar.'
if check_zip( filepath ):
return 'zip'
is_bzipped, is_valid = check_bz2( filepath )
if is_bzipped and is_valid:
return retval + 'bzip'
is_gzipped, is_valid = check_gzip( filepath )
if is_gzipped and is_valid:
return retval + 'gzip'
return None
def _add_line( self, locfile, newline ):
filepath = self.locations[ locfile ]
origlines = []
output = []
comments = []
with open( filepath, 'r' ) as destfile:
for line in destfile:
if line.startswith( '#' ):
comments.append( line.strip() )
else:
origlines.append( line.strip() )
if newline not in origlines:
origlines.append( newline )
output.extend( comments )
origlines.sort()
output.extend( origlines )
with open( filepath, 'w+' ) as destfile:
destfile.write( '\n'.join( output ) )
lib/galaxy/jobs/deferred/liftover_transfer.py
+134
View File
@@ -0,0 +1,134 @@
"""
Module for managing genome transfer jobs.
"""
import logging, shutil, gzip, tempfile, sys
from galaxy import eggs
from sqlalchemy import and_
from galaxy.util.odict import odict
from galaxy.workflow.modules import module_factory
from galaxy.jobs.actions.post import ActionBox
from galaxy.tools.parameters import visit_input_values
from galaxy.tools.parameters.basic import DataToolParameter
from galaxy.tools.data import ToolDataTableManager
from galaxy.datatypes.checkers import *
from data_transfer import *
log = logging.getLogger( __name__ )
__all__ = [ 'LiftOverTransferPlugin' ]
class LiftOverTransferPlugin( DataTransfer ):
locations = {}
def __init__( self, app ):
super( LiftOverTransferPlugin, self ).__init__( app )
self.app = app
self.sa_session = app.model.context.current
def create_job( self, trans, url, dbkey, from_genome, to_genome, destfile ):
job = trans.app.transfer_manager.new( protocol='http', url=url )
params = dict( user=trans.user.id, transfer_job_id=job.id, protocol='http',
type='init_transfer', dbkey=dbkey, from_genome=from_genome,
to_genome=to_genome, destfile=destfile )
deferred = trans.app.model.DeferredJob( state = self.app.model.DeferredJob.states.NEW, plugin = 'LiftOverTransferPlugin', params = params )
self.sa_session.add( deferred )
self.sa_session.flush()
return deferred.id
def check_job( self, job ):
if job.params['type'] == 'init_transfer':
if not hasattr(job, 'transfer_job'):
job.transfer_job = self.sa_session.query( self.app.model.TransferJob ).get( int( job.params[ 'transfer_job_id' ] ) )
else:
self.sa_session.refresh( job.transfer_job )
if job.transfer_job.state == 'done':
transfer = job.transfer_job
transfer.state = 'downloaded'
job.params['type'] = 'extract_transfer'
self.sa_session.add( job )
self.sa_session.add( transfer )
self.sa_session.flush()
return self.job_states.READY
elif job.transfer_job.state == 'running':
return self.job_states.WAIT
elif job.transfer_job.state == 'new':
assert job.params[ 'protocol' ] in [ 'http', 'ftp', 'https' ], 'Unknown protocol %s' % job.params[ 'protocol' ]
self.app.transfer_manager.run( job.transfer_job )
self.sa_session.add( job.transfer_job )
self.sa_session.flush()
return self.job_states.WAIT
else:
log.error( "An error occurred while downloading from %s" % job.params[ 'url' ] )
return self.job_states.INVALID
elif job.params[ 'type' ] == 'extract_transfer':
return self.job_states.READY
def get_job_status( self, jobid ):
job = self.sa_session.query( self.app.model.DeferredJob ).get( int( jobid ) )
return job
def run_job( self, job ):
params = job.params
dbkey = params[ 'dbkey' ]
source = params[ 'from_genome' ]
target = params[ 'to_genome' ]
if not hasattr( job, 'transfer_job' ):
job.transfer_job = self.sa_session.query( self.app.model.TransferJob ).get( int( job.params[ 'transfer_job_id' ] ) )
else:
self.sa_session.refresh( job.transfer_job )
transfer = job.transfer_job
if params[ 'type' ] == 'extract_transfer':
CHUNK_SIZE = 2**20
destpath = os.path.join( self.app.config.get( 'genome_data_path', 'tool-data/genome' ), job.params[ 'dbkey' ], 'liftOver' )
destfile = job.params[ 'destfile' ]
destfilepath = os.path.join( destpath, destfile )
tmpprefix = '%s_%s_download_unzip_' % ( job.params['dbkey'], job.params[ 'transfer_job_id' ] )
tmppath = os.path.dirname( os.path.abspath( transfer.path ) )
if not os.path.exists( destpath ):
os.makedirs( destpath )
fd, uncompressed = tempfile.mkstemp( prefix=tmpprefix, dir=tmppath, text=False )
chain = gzip.open( transfer.path, 'rb' )
while 1:
try:
chunk = chain.read( CHUNK_SIZE )
except IOError:
os.close( fd )
log.error( 'Problem decompressing compressed data' )
exit()
if not chunk:
break
os.write( fd, chunk )
os.close( fd )
chain.close()
# Replace the gzipped file with the decompressed file if it's safe to do so
shutil.move( uncompressed, destfilepath )
os.remove( transfer.path )
os.chmod( destfilepath, 0644 )
locline = '\t'.join( [ source, target, os.path.abspath( destfilepath ) ] )
self._add_line( locline )
job.state = self.app.model.DeferredJob.states.OK
job.params[ 'type' ] = 'finish_transfer'
transfer.path = os.path.abspath(destfilepath)
transfer.state = 'done'
self.sa_session.add( job )
self.sa_session.add( transfer )
self.sa_session.flush()
return self.app.model.DeferredJob.states.OK
def _add_line( self, newline ):
filepath = 'tool-data/liftOver.loc'
origlines = []
with open( filepath, 'r' ) as destfile:
for line in destfile:
origlines.append( line.strip() )
if newline not in origlines:
origlines.append( newline )
with open( filepath, 'w+' ) as destfile:
destfile.write( '\n'.join( origlines ) )
lib/galaxy/jobs/handler.py
+430
View File
@@ -0,0 +1,430 @@
"""
Galaxy job handler, prepares, runs, tracks, and finishes Galaxy jobs
"""
import os
import time
import logging
import threading
from Queue import Queue, Empty
from sqlalchemy.sql.expression import and_, or_
from galaxy import util, model
from galaxy.jobs import Sleeper, JobWrapper, TaskWrapper
log = logging.getLogger( __name__ )
# States for running a job. These are NOT the same as data states
JOB_WAIT, JOB_ERROR, JOB_INPUT_ERROR, JOB_INPUT_DELETED, JOB_READY, JOB_DELETED, JOB_ADMIN_DELETED = 'wait', 'error', 'input_error', 'input_deleted', 'ready', 'deleted', 'admin_deleted'
class JobHandler( object ):
"""
Handle the preparation, running, tracking, and finishing of jobs
"""
def __init__( self, app ):
self.app = app
# The dispatcher launches the underlying job runners
self.dispatcher = DefaultJobDispatcher( app )
# Queues for starting and stopping jobs
self.job_queue = JobHandlerQueue( app, self.dispatcher )
self.job_stop_queue = JobHandlerStopQueue( app, self.dispatcher )
def start( self ):
self.job_queue.start()
def shutdown( self ):
self.job_queue.shutdown()
self.job_stop_queue.shutdown()
class JobHandlerQueue( object ):
"""
Job manager, waits for jobs to be runnable and then dispatches to
a JobRunner.
"""
STOP_SIGNAL = object()
def __init__( self, app, dispatcher ):
"""Start the job manager"""
self.app = app
self.dispatcher = dispatcher
self.sa_session = app.model.context
self.track_jobs_in_database = self.app.config.track_jobs_in_database
# Keep track of the pid that started the job manager, only it
# has valid threads
self.parent_pid = os.getpid()
# Contains new jobs. Note this is not used if track_jobs_in_database is True
self.queue = Queue()
# Contains jobs that are waiting (only use from monitor thread)
## This and jobs_to_check[] are closest to a "Job Queue"
self.waiting_jobs = []
# Helper for interruptable sleep
self.sleeper = Sleeper()
self.running = True
self.monitor_thread = threading.Thread( target=self.__monitor )
def start( self ):
"""
The JobManager should start, and then start its Handler, if it has one.
"""
# Recover jobs at startup
self.__check_jobs_at_startup()
# Start the queue
self.monitor_thread.start()
log.info( "job handler queue started" )
def __check_jobs_at_startup( self ):
"""
Checks all jobs that are in the 'new', 'queued' or 'running' state in
the database and requeues or cleans up as necessary. Only run as the
job manager starts.
"""
for job in self.sa_session.query( model.Job ).enable_eagerloads( False ) \
.filter( ( ( model.Job.state == model.Job.states.NEW ) \
| ( model.Job.state == model.Job.states.RUNNING ) \
| ( model.Job.state == model.Job.states.QUEUED ) ) \
& ( model.Job.handler == self.app.config.server_name ) ):
if job.tool_id not in self.app.toolbox.tools_by_id:
log.warning( "(%s) Tool '%s' removed from tool config, unable to recover job" % ( job.id, job.tool_id ) )
JobWrapper( job, self ).fail( 'This tool was disabled before the job completed. Please contact your Galaxy administrator.' )
elif job.job_runner_name is None:
log.debug( "(%s) No job runner assigned and job still in '%s' state, adding to the job handler queue" % ( job.id, job.state ) )
if self.track_jobs_in_database:
job.state = model.Job.states.NEW
else:
self.queue.put( ( job.id, job.tool_id ) )
else:
job_wrapper = JobWrapper( job, self )
self.dispatcher.recover( job, job_wrapper )
if self.sa_session.dirty:
self.sa_session.flush()
def __monitor( self ):
"""
Continually iterate the waiting jobs, checking is each is ready to
run and dispatching if so.
"""
while self.running:
try:
self.__monitor_step()
except:
log.exception( "Exception in monitor_step" )
# Sleep
self.sleeper.sleep( 1 )
def __monitor_step( self ):
"""
Called repeatedly by `monitor` to process waiting jobs. Gets any new
jobs (either from the database or from its own queue), then iterates
over all new and waiting jobs to check the state of the jobs each
depends on. If the job has dependencies that have not finished, it
it goes to the waiting queue. If the job has dependencies with errors,
it is marked as having errors and removed from the queue. Otherwise,
the job is dispatched.
"""
# Pull all new jobs from the queue at once
jobs_to_check = []
if self.track_jobs_in_database:
# Clear the session so we get fresh states for job and all datasets
self.sa_session.expunge_all()
# Fetch all new jobs
jobs_to_check = self.sa_session.query( model.Job ).enable_eagerloads( False ) \
.filter( ( model.Job.state == model.Job.states.NEW ) \
& ( model.Job.handler == self.app.config.server_name ) ).all()
else:
# Get job objects and append to watch queue for any which were
# previously waiting
for job_id in self.waiting_jobs:
jobs_to_check.append( self.sa_session.query( model.Job ).get( job_id ) )
try:
while 1:
message = self.queue.get_nowait()
if message is self.STOP_SIGNAL:
return
# Unpack the message
job_id, tool_id = message
# Get the job object and append to watch queue
jobs_to_check.append( self.sa_session.query( model.Job ).get( job_id ) )
except Empty:
pass
# Iterate over new and waiting jobs and look for any that are
# ready to run
new_waiting_jobs = []
for job in jobs_to_check:
try:
# Check the job's dependencies, requeue if they're not done
job_state = self.__check_if_ready_to_run( job )
if job_state == JOB_WAIT:
if not self.track_jobs_in_database:
new_waiting_jobs.append( job.id )
elif job_state == JOB_INPUT_ERROR:
log.info( "(%d) Job unable to run: one or more inputs in error state" % job.id )
elif job_state == JOB_INPUT_DELETED:
log.info( "(%d) Job unable to run: one or more inputs deleted" % job.id )
elif job_state == JOB_READY:
self.dispatcher.put( JobWrapper( job, self ) )
log.info( "(%d) Job dispatched" % job.id )
elif job_state == JOB_DELETED:
log.info( "(%d) Job deleted by user while still queued" % job.id )
elif job_state == JOB_ADMIN_DELETED:
log.info( "(%d) Job deleted by admin while still queued" % job.id )
else:
log.error( "(%d) Job in unknown state '%s'" % ( job.id, job_state ) )
if not self.track_jobs_in_database:
new_waiting_jobs.append( job.id )
except Exception:
log.exception( "failure running job %d" % job.id )
# Update the waiting list
self.waiting_jobs = new_waiting_jobs
# Done with the session
self.sa_session.remove()
def __check_if_ready_to_run( self, job ):
"""
Check if a job is ready to run by verifying that each of its input
datasets is ready (specifically in the OK state). If any input dataset
has an error, fail the job and return JOB_INPUT_ERROR. If any input
dataset is deleted, fail the job and return JOB_INPUT_DELETED. If all
input datasets are in OK state, return JOB_READY indicating that the
job can be dispatched. Otherwise, return JOB_WAIT indicating that input
datasets are still being prepared.
"""
if job.state == model.Job.states.DELETED:
return JOB_DELETED
elif job.state == model.Job.states.ERROR:
return JOB_ADMIN_DELETED
elif self.app.config.enable_quotas:
quota = self.app.quota_agent.get_quota( job.user )
if quota is not None:
try:
usage = self.app.quota_agent.get_usage( user=job.user, history=job.history )
if usage > quota:
return JOB_WAIT
except AssertionError, e:
pass # No history, should not happen with an anon user
for dataset_assoc in job.input_datasets + job.input_library_datasets:
idata = dataset_assoc.dataset
if not idata:
continue
# don't run jobs for which the input dataset was deleted
if idata.deleted:
JobWrapper( job, self ).fail( "input data %s (file: %s) was deleted before the job started" % ( idata.hid, idata.file_name ) )
return JOB_INPUT_DELETED
# an error in the input data causes us to bail immediately
elif idata.state == idata.states.ERROR:
JobWrapper( job, self ).fail( "input data %s is in error state" % ( idata.hid ) )
return JOB_INPUT_ERROR
elif idata.state == idata.states.FAILED_METADATA:
JobWrapper( job, self ).fail( "input data %s failed to properly set metadata" % ( idata.hid ) )
return JOB_INPUT_ERROR
elif idata.state != idata.states.OK and not ( idata.state == idata.states.SETTING_METADATA and job.tool_id is not None and job.tool_id == self.app.datatypes_registry.set_external_metadata_tool.id ):
# need to requeue
return JOB_WAIT
return self.__check_user_jobs( job )
def __check_user_jobs( self, job ):
if not self.app.config.user_job_limit:
return JOB_READY
if job.user:
count = self.sa_session.query( model.Job ).enable_eagerloads( False ) \
.filter( and_( model.Job.user_id == job.user.id,
or_( model.Job.state == model.Job.states.RUNNING,
model.Job.state == model.Job.states.QUEUED ) ) ).count()
elif job.galaxy_session:
count = self.sa_session.query( model.Job ).enable_eagerloads( False ) \
.filter( and_( model.Job.session_id == job.galaxy_session.id,
or_( model.Job.state == model.Job.states.RUNNING,
model.Job.state == model.Job.states.QUEUED ) ) ).count()
else:
log.warning( 'Job %s is not associated with a user or session so job concurrency limit cannot be checked.' % job.id )
return JOB_READY
if count >= self.app.config.user_job_limit:
return JOB_WAIT
return JOB_READY
def put( self, job_id, tool_id ):
"""Add a job to the queue (by job identifier)"""
if not self.track_jobs_in_database:
self.queue.put( ( job_id, tool_id ) )
self.sleeper.wake()
def shutdown( self ):
"""Attempts to gracefully shut down the worker thread"""
if self.parent_pid != os.getpid():
# We're not the real job queue, do nothing
return
else:
log.info( "sending stop signal to worker thread" )
self.running = False
if not self.app.config.track_jobs_in_database:
self.queue.put( self.STOP_SIGNAL )
self.sleeper.wake()
log.info( "job handler queue stopped" )
self.dispatcher.shutdown()
class JobHandlerStopQueue( object ):
"""
A queue for jobs which need to be terminated prematurely.
"""
STOP_SIGNAL = object()
def __init__( self, app, dispatcher ):
self.app = app
self.dispatcher = dispatcher
self.sa_session = app.model.context
# Keep track of the pid that started the job manager, only it
# has valid threads
self.parent_pid = os.getpid()
# Contains new jobs. Note this is not used if track_jobs_in_database is True
self.queue = Queue()
# Contains jobs that are waiting (only use from monitor thread)
self.waiting = []
# Helper for interruptable sleep
self.sleeper = Sleeper()
self.running = True
self.monitor_thread = threading.Thread( target=self.monitor )
self.monitor_thread.start()
log.info( "job handler stop queue started" )
def monitor( self ):
"""
Continually iterate the waiting jobs, stop any that are found.
"""
# HACK: Delay until after forking, we need a way to do post fork notification!!!
time.sleep( 10 )
while self.running:
try:
self.monitor_step()
except:
log.exception( "Exception in monitor_step" )
# Sleep
self.sleeper.sleep( 1 )
def monitor_step( self ):
"""
Called repeatedly by `monitor` to stop jobs.
"""
# Pull all new jobs from the queue at once
jobs_to_check = []
if self.app.config.track_jobs_in_database:
# Clear the session so we get fresh states for job and all datasets
self.sa_session.expunge_all()
# Fetch all new jobs
newly_deleted_jobs = self.sa_session.query( model.Job ).enable_eagerloads( False ) \
.filter( ( model.Job.state == model.Job.states.DELETED_NEW ) \
& ( model.Job.handler == self.app.config.server_name ) ).all()
for job in newly_deleted_jobs:
jobs_to_check.append( ( job, None ) )
# Also pull from the queue (in the case of Administrative stopped jobs)
try:
while 1:
message = self.queue.get_nowait()
if message is self.STOP_SIGNAL:
return
# Unpack the message
job_id, error_msg = message
# Get the job object and append to watch queue
jobs_to_check.append( ( self.sa_session.query( model.Job ).get( job_id ), error_msg ) )
except Empty:
pass
for job, error_msg in jobs_to_check:
if error_msg is not None:
job.state = job.states.ERROR
job.info = error_msg
else:
job.state = job.states.DELETED
self.sa_session.add( job )
self.sa_session.flush()
if job.job_runner_name is not None:
# tell the dispatcher to stop the job
self.dispatcher.stop( job )
def put( self, job_id, error_msg=None ):
self.queue.put( ( job_id, error_msg ) )
def shutdown( self ):
"""Attempts to gracefully shut down the worker thread"""
if self.parent_pid != os.getpid():
# We're not the real job queue, do nothing
return
else:
log.info( "sending stop signal to worker thread" )
self.running = False
if not self.app.config.track_jobs_in_database:
self.queue.put( self.STOP_SIGNAL )
self.sleeper.wake()
log.info( "job handler stop queue stopped" )
class DefaultJobDispatcher( object ):
def __init__( self, app ):
self.app = app
self.job_runners = {}
start_job_runners = ["local"]
if app.config.start_job_runners is not None:
start_job_runners.extend( [ x.strip() for x in util.listify( app.config.start_job_runners ) ] )
if app.config.use_tasked_jobs:
start_job_runners.append("tasks")
for name in start_job_runners:
self._load_plugin( name )
def _load_plugin( self, name ):
module_name = 'galaxy.jobs.runners.' + name
try:
module = __import__( module_name )
except:
log.exception( 'Job runner is not loadable: %s' % module_name )
return
for comp in module_name.split( "." )[1:]:
module = getattr( module, comp )
if '__all__' not in dir( module ):
log.error( 'Runner "%s" does not contain a list of exported classes in __all__' % module_name )
return
for obj in module.__all__:
display_name = ':'.join( ( module_name, obj ) )
runner = getattr( module, obj )
self.job_runners[name] = runner( self.app )
log.debug( 'Loaded job runner: %s' % display_name )
def __get_runner_name( self, job_wrapper ):
if self.app.config.use_tasked_jobs and job_wrapper.tool.parallelism is not None and not isinstance(job_wrapper, TaskWrapper):
runner_name = "tasks"
else:
runner_name = ( job_wrapper.get_job_runner().split(":", 1) )[0]
return runner_name
def put( self, job_wrapper ):
try:
runner_name = self.__get_runner_name( job_wrapper )
if self.app.config.use_tasked_jobs and job_wrapper.tool.parallelism is not None and isinstance(job_wrapper, TaskWrapper):
#DBTODO Refactor
log.debug( "dispatching task %s, of job %d, to %s runner" %( job_wrapper.task_id, job_wrapper.job_id, runner_name ) )
else:
log.debug( "dispatching job %d to %s runner" %( job_wrapper.job_id, runner_name ) )
self.job_runners[runner_name].put( job_wrapper )
except KeyError:
log.error( 'put(): (%s) Invalid job runner: %s' % ( job_wrapper.job_id, runner_name ) )
job_wrapper.fail( 'Unable to run job due to a misconfiguration of the Galaxy job running system. Please contact a site administrator.' )
def stop( self, job ):
runner_name = ( job.job_runner_name.split(":", 1) )[0]
log.debug( "stopping job %d in %s runner" %( job.id, runner_name ) )
try:
self.job_runners[runner_name].stop_job( job )
except KeyError:
log.error( 'stop(): (%s) Invalid job runner: %s' % ( job_wrapper.job_id, runner_name ) )
# Job and output dataset states have already been updated, so nothing is done here.
def recover( self, job, job_wrapper ):
runner_name = ( job.job_runner_name.split(":", 1) )[0]
log.debug( "recovering job %d in %s runner" %( job.id, runner_name ) )
try:
self.job_runners[runner_name].recover( job, job_wrapper )
except KeyError:
log.error( 'recover(): (%s) Invalid job runner: %s' % ( job_wrapper.job_id, runner_name ) )
job_wrapper.fail( 'Unable to run job due to a misconfiguration of the Galaxy job running system. Please contact a site administrator.' )
def shutdown( self ):
for runner in self.job_runners.itervalues():
runner.shutdown()
lib/galaxy/jobs/manager.py
+276
View File
@@ -0,0 +1,276 @@
"""
Top-level Galaxy job manager, moves jobs to handler(s)
"""
import os
import time
import random
import logging
import threading
from Queue import Queue, Empty
from sqlalchemy.sql.expression import and_, or_
from galaxy import model
from galaxy.jobs import handler, Sleeper, NoopQueue
from galaxy.util.json import from_json_string
log = logging.getLogger( __name__ )
class JobManager( object ):
"""
Highest level interface to job management.
TODO: Currently the app accesses "job_queue" and "job_stop_queue" directly.
This should be decoupled.
"""
def __init__( self, app ):
self.app = app
self.job_handler = NoopHandler()
if self.app.config.server_name in self.app.config.job_handlers:
self.job_handler = handler.JobHandler( app )
if self.app.config.server_name == self.app.config.job_manager:
job_handler = NoopHandler()
# In the case that webapp == manager == handler, pass jobs in memory
if not self.app.config.track_jobs_in_database:
job_handler = self.job_handler
# Otherwise, even if the manager == one of the handlers, its handler will pick up jobs from the database
self.job_queue = JobManagerQueue( app, job_handler )
self.job_stop_queue = JobManagerStopQueue( app, job_handler )
if self.app.config.enable_beta_job_managers:
from galaxy.jobs.deferred import DeferredJobQueue
self.deferred_job_queue = DeferredJobQueue( app )
else:
self.job_queue = self.job_stop_queue = NoopQueue()
self.job_handler.start()
def shutdown( self ):
self.job_queue.shutdown()
self.job_stop_queue.shutdown()
self.job_handler.shutdown()
class JobManagerQueue( object ):
"""
Job manager, waits for jobs to be runnable and then dispatches to a
JobHandler.
"""
STOP_SIGNAL = object()
def __init__( self, app, job_handler ):
self.app = app
self.job_handler = job_handler # the (singular) handler if we are passing jobs in memory
self.sa_session = app.model.context
self.job_lock = False
# Keep track of the pid that started the job manager, only it
# has valid threads
self.parent_pid = os.getpid()
# Contains new jobs. Note this is not used if track_jobs_in_database is True
self.queue = Queue()
# Helper for interruptable sleep
self.sleeper = Sleeper()
self.running = True
self.monitor_thread = threading.Thread( target=self.__monitor )
# Recover jobs at startup
self.__check_jobs_at_startup()
# Start the queue
self.monitor_thread.start()
log.info( "job manager queue started" )
def __check_jobs_at_startup( self ):
"""
Checks all jobs that are in the 'new', 'queued' or 'running' state in
the database and requeues or cleans up as necessary. Only run as the
job manager starts.
"""
for job in self.sa_session.query( model.Job ).enable_eagerloads( False ) \
.filter( ( ( model.Job.state == model.Job.states.NEW ) \
| ( model.Job.state == model.Job.states.RUNNING ) \
| ( model.Job.state == model.Job.states.QUEUED ) ) \
& ( model.Job.handler == None ) ):
if job.tool_id not in self.app.toolbox.tools_by_id:
log.warning( "(%s) Tool '%s' removed from tool config, unable to recover job" % ( job.id, job.tool_id ) )
JobWrapper( job, self ).fail( 'This tool was disabled before the job completed. Please contact your Galaxy administrator.' )
else:
job.handler = self.__get_handler( job ) # handler's recovery method will take it from here
log.info( "(%d) Job in '%s' state had no handler at job manager startup, assigned '%s' handler" % ( job.id, job.state, job.handler ) )
if self.sa_session.dirty:
self.sa_session.flush()
def __monitor( self ):
"""
Continually iterate the waiting jobs and dispatch to a handler
"""
# HACK: Delay until after forking, we need a way to do post fork notification!!!
time.sleep( 10 )
while self.running:
try:
self.__monitor_step()
except:
log.exception( "Exception in monitor_step" )
# Sleep
self.sleeper.sleep( 1 )
def __monitor_step( self ):
"""
Called repeatedly by `monitor` to process waiting jobs. Gets any new
jobs (either from the database or from its own queue), then assigns a
handler.
"""
# Do nothing if the queue is locked
if self.job_lock:
log.info( 'Job queue is administratively locked, sleeping...' )
time.sleep( 10 )
return
# Pull all new jobs from the queue at once
jobs_to_check = []
if self.app.config.track_jobs_in_database:
# Clear the session so we get fresh states for job and all datasets
self.sa_session.expunge_all()
# Fetch all new jobs
jobs_to_check = self.sa_session.query( model.Job ).enable_eagerloads( False ) \
.filter( ( model.Job.state == model.Job.states.NEW ) \
& ( model.Job.handler == None ) ).all()
else:
# Get job objects and append to watch queue for any which were
# previously waiting
try:
while 1:
message = self.queue.get_nowait()
if message is self.STOP_SIGNAL:
return
# Unpack the message
job_id, tool_id = message
# Get the job object and append to watch queue
jobs_to_check.append( self.sa_session.query( model.Job ).get( job_id ) )
except Empty:
pass
for job in jobs_to_check:
job.handler = self.__get_handler( job )
log.debug( "(%s) Job assigned to handler '%s'" % ( job.id, job.handler ) )
self.sa_session.add( job )
# If tracking in the database, handlers will pick up the job now
self.sa_session.flush()
time.sleep( 5 )
# This only does something in the case that there is only one handler and it is this Galaxy process
for job in jobs_to_check:
self.job_handler.job_queue.put( job.id, job.tool_id )
def __get_handler( self, job ):
try:
params = None
if job.params:
params = from_json_string( job.params )
return self.app.toolbox.tools_by_id.get( job.tool_id, None ).get_job_handler( params )
except:
log.exception( "(%s) Caught exception attempting to get tool-specific job handler for tool '%s', selecting at random from available handlers instead:" % ( job.id, job.tool_id ) )
return random.choice( self.app.config.job_handlers )
def put( self, job_id, tool ):
"""Add a job to the queue (by job identifier)"""
if not self.app.config.track_jobs_in_database:
self.queue.put( ( job_id, tool.id ) )
self.sleeper.wake()
def shutdown( self ):
"""Attempts to gracefully shut down the worker thread"""
if self.parent_pid != os.getpid():
# We're not the real job queue, do nothing
return
else:
log.info( "sending stop signal to worker thread" )
self.running = False
if not self.app.config.track_jobs_in_database:
self.queue.put( self.STOP_SIGNAL )
self.sleeper.wake()
log.info( "job manager queue stopped" )
class JobManagerStopQueue( object ):
"""
A queue for jobs which need to be terminated prematurely.
"""
STOP_SIGNAL = object()
def __init__( self, app, job_handler ):
self.app = app
self.job_handler = job_handler
self.sa_session = app.model.context
# Keep track of the pid that started the job manager, only it
# has valid threads
self.parent_pid = os.getpid()
# Contains new jobs. Note this is not used if track_jobs_in_database is True
self.queue = Queue()
# Contains jobs that are waiting (only use from monitor thread)
self.waiting = []
# Helper for interruptable sleep
self.sleeper = Sleeper()
self.running = True
self.monitor_thread = threading.Thread( target=self.monitor )
self.monitor_thread.start()
log.info( "job manager stop queue started" )
def monitor( self ):
"""
Continually iterate the waiting jobs, stop any that are found.
"""
# HACK: Delay until after forking, we need a way to do post fork notification!!!
time.sleep( 10 )
while self.running:
try:
self.monitor_step()
except:
log.exception( "Exception in monitor_step" )
# Sleep
self.sleeper.sleep( 1 )
def monitor_step( self ):
"""
Called repeatedly by `monitor` to stop jobs.
"""
jobs_to_check = []
# Pull from the queue even if tracking in the database (in the case of Administrative stopped jobs)
try:
while 1:
message = self.queue.get_nowait()
if message is self.STOP_SIGNAL:
return
# Unpack the message
job_id, error_msg = message
# Get the job object and append to watch queue
jobs_to_check.append( ( self.sa_session.query( model.Job ).get( job_id ), error_msg ) )
except Empty:
pass
# If tracking in the database, the handler will pick up the stop itself. Otherwise, notify the handler.
for job, error_msg in jobs_to_check:
self.job_handler.job_stop_queue.put( job.id, error_msg )
def put( self, job_id, error_msg=None ):
self.queue.put( ( job_id, error_msg ) )
def shutdown( self ):
"""Attempts to gracefully shut down the worker thread"""
if self.parent_pid != os.getpid():
# We're not the real job queue, do nothing
return
else:
log.info( "sending stop signal to worker thread" )
self.running = False
if not self.app.config.track_jobs_in_database:
self.queue.put( self.STOP_SIGNAL )
self.sleeper.wake()
log.info( "job manager stop queue stopped" )
class NoopHandler( object ):
def __init__( self, *args, **kwargs ):
self.job_queue = NoopQueue()
self.job_stop_queue = NoopQueue()
def start( self ):
pass
def shutdown( self, *args ):
pass
lib/galaxy/jobs/runners/tasks.py
+1 -1
View File
@@ -83,7 +83,7 @@ class TaskedJobRunner( object ):
for task in tasks:
tw = TaskWrapper(task, job_wrapper.queue)
task_wrappers.append(tw)
self.app.job_manager.dispatcher.put(tw)
self.app.job_manager.job_handler.dispatcher.put(tw)
tasks_incomplete = False
count_complete = 0
sleep_time = 1
lib/galaxy/model/__init__.py
+44 -16
View File
@@ -132,6 +132,7 @@ class Job( object ):
self.job_runner_external_id = None
self.post_job_actions = []
self.imported = False
self.handler = None
def add_parameter( self, name, value ):
self.parameters.append( JobParameter( name, value ) )
@@ -171,14 +172,11 @@ class Job( object ):
if not dataset.deleted:
return False
return True
def mark_deleted( self, enable_job_running=True, track_jobs_in_database=False ):
def mark_deleted( self, track_jobs_in_database=False ):
"""
Mark this job as deleted, and mark any output datasets as discarded.
"""
# This could be handled with *just* track_jobs_in_database, but I
# didn't want to make setting track_jobs_in_database required in
# non-runner configs.
if not enable_job_running or track_jobs_in_database:
if track_jobs_in_database:
self.state = Job.states.DELETED_NEW
else:
self.state = Job.states.DELETED
@@ -303,6 +301,20 @@ class JobImportHistoryArchive( object ):
self.history = history
self.archive_dir=archive_dir
class GenomeIndexToolData( object ):
def __init__( self, job=None, params=None, dataset=None, deferred_job=None, \
transfer_job=None, fasta_path=None, created_time=None, modified_time=None, \
dbkey=None, user=None, indexer=None ):
self.job = job
self.dataset = dataset
self.fasta_path = fasta_path
self.user = user
self.indexer = indexer
self.created_time = created_time
self.modified_time = modified_time
self.deferred = deferred_job
self.transfer = transfer_job
class DeferredJob( object ):
states = Bunch( NEW = 'new',
WAITING = 'waiting',
@@ -1128,8 +1140,10 @@ class HistoryDatasetAssociation( DatasetInstance ):
def clear_associated_files( self, metadata_safe = False, purge = False ):
# metadata_safe = True means to only clear when assoc.metadata_safe == False
for assoc in self.implicitly_converted_datasets:
if not metadata_safe or not assoc.metadata_safe:
if not assoc.deleted and ( not metadata_safe or not assoc.metadata_safe ):
assoc.clear( purge = purge )
for assoc in self.implicitly_converted_parent_datasets:
assoc.clear( purge = purge, delete_dataset = False )
def get_display_name( self ):
## Name can be either a string or a unicode object. If string, convert to unicode object assuming 'utf-8' format.
hda_name = self.name
@@ -1188,6 +1202,12 @@ class HistoryDatasetAssociationDisplayAtAuthorization( object ):
self.history_dataset_association = hda
self.user = user
self.site = site
class HistoryDatasetAssociationSubset( object ):
def __init__(self, hda, subset, location):
self.hda = hda
self.subset = subset
self.location = location
class Library( object, APIItem ):
permitted_actions = get_permitted_actions( filter='LIBRARY' )
@@ -1536,21 +1556,26 @@ class LibraryDatasetDatasetAssociation( DatasetInstance ):
else:
return template.get_widgets( trans.user )
return []
def templates_dict( self ):
def templates_dict( self, use_name=False ):
"""
Returns a dict of template info
"""
#TODO: Should have a method that allows names and labels to be returned together in a structured way
template_data = {}
for temp_info in self.info_association:
template = temp_info.template
content = temp_info.info.content
tmp_dict = {}
for field in template.fields:
tmp_dict[field['label']] = content[field['name']]
if use_name:
name = field[ 'name' ]
else:
name = field[ 'label' ]
tmp_dict[ name ] = content.get( field[ 'name' ] )
template_data[template.name] = tmp_dict
return template_data
def templates_json( self ):
return simplejson.dumps( self.templates_dict() )
def templates_json( self, use_name=False ):
return simplejson.dumps( self.templates_dict( use_name=use_name ) )
def get_display_name( self ):
"""
@@ -1617,12 +1642,14 @@ class ImplicitlyConvertedDatasetAssociation( object ):
self.purged = purged
self.metadata_safe = metadata_safe
def clear( self, purge = False ):
def clear( self, purge = False, delete_dataset = True ):
self.deleted = True
if self.dataset:
self.dataset.deleted = True
self.dataset.purged = purge
if purge: #do something with purging
if delete_dataset:
self.dataset.deleted = True
if purge:
self.dataset.purged = True
if purge and self.dataset.deleted: #do something with purging
self.purged = True
try: os.unlink( self.file_name )
except Exception, e: print "Failed to purge associated file (%s) from disk: %s" % ( self.file_name, e )
@@ -2620,8 +2647,8 @@ class APIKeys( object ):
class ToolShedRepository( object ):
def __init__( self, id=None, create_time=None, tool_shed=None, name=None, description=None, owner=None, installed_changeset_revision=None,
changeset_revision=None, metadata=None, includes_datatypes=False, update_available=False, deleted=False, uninstalled=False,
dist_to_shed=False ):
changeset_revision=None, ctx_rev=None, metadata=None, includes_datatypes=False, update_available=False, deleted=False,
uninstalled=False, dist_to_shed=False ):
self.id = id
self.create_time = create_time
self.tool_shed = tool_shed
@@ -2630,6 +2657,7 @@ class ToolShedRepository( object ):
self.owner = owner
self.installed_changeset_revision = installed_changeset_revision
self.changeset_revision = changeset_revision
self.ctx_rev = ctx_rev
self.metadata = metadata
self.includes_datatypes = includes_datatypes
self.update_available = update_available
lib/galaxy/model/mapping.py
+40 -1
View File
@@ -76,6 +76,7 @@ UserOpenID.table = Table( "galaxy_user_openid", metadata,
Column( "session_id", Integer, ForeignKey( "galaxy_session.id" ), index=True ),
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
Column( "openid", TEXT, index=True, unique=True ),
Column( "provider", TrimmedString( 255 ) ),
)
History.table = Table( "history", metadata,
@@ -143,6 +144,12 @@ HistoryDatasetAssociationDisplayAtAuthorization.table = Table( "history_dataset_
Column( "history_dataset_association_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ),
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
Column( "site", TrimmedString( 255 ) ) )
HistoryDatasetAssociationSubset.table = Table( "history_dataset_association_subset", metadata,
Column( "id", Integer, primary_key=True ),
Column( "history_dataset_association_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ),
Column( "history_dataset_association_subset_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ),
Column( "location", Unicode(255), index=True) )
ImplicitlyConvertedDatasetAssociation.table = Table( "implicitly_converted_dataset_association", metadata,
Column( "id", Integer, primary_key=True ),
@@ -375,6 +382,7 @@ ToolShedRepository.table = Table( "tool_shed_repository", metadata,
Column( "owner", TrimmedString( 255 ), index=True ),
Column( "installed_changeset_revision", TrimmedString( 255 ) ),
Column( "changeset_revision", TrimmedString( 255 ), index=True ),
Column( "ctx_rev", TrimmedString( 10 ) ),
Column( "metadata", JSONType, nullable=True ),
Column( "includes_datatypes", Boolean, index=True, default=False ),
Column( "update_available", Boolean, default=False ),
@@ -421,7 +429,8 @@ Job.table = Table( "job", metadata,
Column( "job_runner_external_id", String( 255 ) ),
Column( "object_store_id", TrimmedString( 255 ), index=True ),
Column( "imported", Boolean, default=False, index=True ),
Column( "params", TrimmedString(255), index=True ) )
Column( "params", TrimmedString(255), index=True ),
Column( "handler", TrimmedString( 255 ), index=True ) )
JobParameter.table = Table( "job_parameter", metadata,
Column( "id", Integer, primary_key=True ),
@@ -483,6 +492,19 @@ JobImportHistoryArchive.table = Table( "job_import_history_archive", metadata,
Column( "archive_dir", TEXT )
)
GenomeIndexToolData.table = Table( "genome_index_tool_data", metadata,
Column( "id", Integer, primary_key=True ),
Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ),
Column( "deferred_job_id", Integer, ForeignKey( "deferred_job.id" ), index=True ),
Column( "transfer_job_id", Integer, ForeignKey( "transfer_job.id" ), index=True ),
Column( "dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ),
Column( "fasta_path", String( 255 ) ),
Column( "created_time", DateTime, default=now ),
Column( "modified_time", DateTime, default=now, onupdate=now ),
Column( "indexer", String( 64 ) ),
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
)
Task.table = Table( "task", metadata,
Column( "id", Integer, primary_key=True ),
Column( "create_time", DateTime, default=now ),
@@ -1196,6 +1218,9 @@ assign_mapper( context, HistoryDatasetAssociation, HistoryDatasetAssociation.tab
implicitly_converted_datasets=relation(
ImplicitlyConvertedDatasetAssociation,
primaryjoin=( ImplicitlyConvertedDatasetAssociation.table.c.hda_parent_id == HistoryDatasetAssociation.table.c.id ) ),
implicitly_converted_parent_datasets=relation(
ImplicitlyConvertedDatasetAssociation,
primaryjoin=( ImplicitlyConvertedDatasetAssociation.table.c.hda_id == HistoryDatasetAssociation.table.c.id ) ),
children=relation(
HistoryDatasetAssociation,
primaryjoin=( HistoryDatasetAssociation.table.c.parent_id == HistoryDatasetAssociation.table.c.id ),
@@ -1231,6 +1256,13 @@ assign_mapper( context, Dataset, Dataset.table,
assign_mapper( context, HistoryDatasetAssociationDisplayAtAuthorization, HistoryDatasetAssociationDisplayAtAuthorization.table,
properties=dict( history_dataset_association = relation( HistoryDatasetAssociation ),
user = relation( User ) ) )
assign_mapper( context, HistoryDatasetAssociationSubset, HistoryDatasetAssociationSubset.table,
properties=dict( hda = relation( HistoryDatasetAssociation,
primaryjoin=( HistoryDatasetAssociationSubset.table.c.history_dataset_association_id == HistoryDatasetAssociation.table.c.id ) ),
subset = relation( HistoryDatasetAssociation,
primaryjoin=( HistoryDatasetAssociationSubset.table.c.history_dataset_association_subset_id == HistoryDatasetAssociation.table.c.id ) )
) )
assign_mapper( context, ImplicitlyConvertedDatasetAssociation, ImplicitlyConvertedDatasetAssociation.table,
properties=dict( parent_hda=relation(
@@ -1493,6 +1525,13 @@ assign_mapper( context, JobExportHistoryArchive, JobExportHistoryArchive.table,
assign_mapper( context, JobImportHistoryArchive, JobImportHistoryArchive.table,
properties=dict( job = relation( Job ), history = relation( History ) ) )
assign_mapper( context, GenomeIndexToolData, GenomeIndexToolData.table,
properties=dict( job = relation( Job ),
dataset = relation( Dataset ),
user = relation( User ),
deferred = relation( DeferredJob, backref='deferred_job' ),
transfer = relation( TransferJob, backref='transfer_job' ) ) )
assign_mapper( context, PostJobAction, PostJobAction.table,
properties=dict(workflow_step = relation( WorkflowStep, backref='post_job_actions', primaryjoin=(WorkflowStep.table.c.id == PostJobAction.table.c.workflow_step_id))))
lib/galaxy/model/migrate/versions/0094_add_job_handler_col.py
+49
View File
@@ -0,0 +1,49 @@
"""
Migration script to create "handler" column in job table.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
metadata = MetaData( migrate_engine )
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
# Column to add.
handler_col = Column( "handler", TrimmedString(255), index=True )
def display_migration_details():
print ""
print "This migration script adds a 'handler' column to the Job table."
def upgrade():
print __doc__
metadata.reflect()
# Add column to Job table.
try:
Job_table = Table( "job", metadata, autoload=True )
handler_col.create( Job_table )
assert handler_col is Job_table.c.handler
except Exception, e:
print str(e)
log.debug( "Adding column 'handler' to job table failed: %s" % str( e ) )
def downgrade():
metadata.reflect()
# Drop column from Job table.
try:
Job_table = Table( "job", metadata, autoload=True )
handler_col = Job_table.c.handler
handler_col.drop()
except Exception, e:
log.debug( "Dropping column 'handler' from job table failed: %s" % ( str( e ) ) )
lib/galaxy/model/migrate/versions/0095_hda_subsets.py
+44
View File
@@ -0,0 +1,44 @@
"""
Migration script to create table for tracking history_dataset_association subsets.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
metadata = MetaData( migrate_engine )
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
# Table to add.
HistoryDatasetAssociationSubset_table = Table( "history_dataset_association_subset", metadata,
Column( "id", Integer, primary_key=True ),
Column( "history_dataset_association_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ),
Column( "history_dataset_association_subset_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ),
Column( "location", Unicode(255), index=True)
)
def upgrade():
print __doc__
metadata.reflect()
# Create history_dataset_association_subset.
try:
HistoryDatasetAssociationSubset_table.create()
except Exception, e:
print str(e)
log.debug( "Creating history_dataset_association_subset table failed: %s" % str( e ) )
def downgrade():
metadata.reflect()
# Drop history_dataset_association_subset table.
try:
HistoryDatasetAssociationSubset_table.drop()
except Exception, e:
print str(e)
log.debug( "Dropping history_dataset_association_subset table failed: %s" % str( e ) )
lib/galaxy/model/migrate/versions/0096_openid_provider.py
+45
View File
@@ -0,0 +1,45 @@
"""
Migration script to add column to openid table for provider.
Remove any OpenID entries with nonunique GenomeSpace Identifier
"""
BAD_IDENTIFIER = 'https://identity.genomespace.org/identityServer/xrd.jsp'
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
from galaxy.model.custom_types import TrimmedString
import logging
log = logging.getLogger( __name__ )
metadata = MetaData( migrate_engine )
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
def upgrade():
print __doc__
metadata.reflect()
try:
OpenID_table = Table( "galaxy_user_openid", metadata, autoload=True )
c = Column( "provider", TrimmedString( 255 ) )
c.create( OpenID_table )
assert c is OpenID_table.c.provider
except Exception, e:
print "Adding provider column to galaxy_user_openid table failed: %s" % str( e )
log.debug( "Adding provider column to galaxy_user_openid table failed: %s" % str( e ) )
try:
cmd = "DELETE FROM galaxy_user_openid WHERE openid='%s'" % ( BAD_IDENTIFIER )
db_session.execute( cmd )
except Exception, e:
log.debug( "Deleting bad Identifiers from galaxy_user_openid failed: %s" % str( e ) )
def downgrade():
metadata.reflect()
try:
OpenID_table = Table( "galaxy_user_openid", metadata, autoload=True )
OpenID_table.c.provider.drop()
except Exception, e:
print "Dropping provider column from galaxy_user_openid table failed: %s" % str( e )
log.debug( "Dropping provider column from galaxy_user_openid table failed: %s" % str( e ) )
lib/galaxy/model/migrate/versions/0097_add_ctx_rev_column.py
+43
View File
@@ -0,0 +1,43 @@
"""
Migration script to add the ctx_rev column to the tool_shed_repository table.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import datetime
now = datetime.datetime.utcnow
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
import sys, logging
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler( sys.stdout )
format = "%(name)s %(levelname)s %(asctime)s %(message)s"
formatter = logging.Formatter( format )
handler.setFormatter( formatter )
log.addHandler( handler )
metadata = MetaData( migrate_engine )
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
def upgrade():
print __doc__
metadata.reflect()
ToolShedRepository_table = Table( "tool_shed_repository", metadata, autoload=True )
col = Column( "ctx_rev", TrimmedString( 10 ) )
try:
col.create( ToolShedRepository_table )
assert col is ToolShedRepository_table.c.ctx_rev
except Exception, e:
print "Adding ctx_rev column to the tool_shed_repository table failed: %s" % str( e )
def downgrade():
metadata.reflect()
ToolShedRepository_table = Table( "tool_shed_repository", metadata, autoload=True )
try:
ToolShedRepository_table.c.ctx_rev.drop()
except Exception, e:
print "Dropping column ctx_rev from the tool_shed_repository table failed: %s" % str( e )
lib/galaxy/model/migrate/versions/0098_genome_index_tool_data_table.py
+53
View File
@@ -0,0 +1,53 @@
"""
Migration script to create the genome_index_tool_data table.
"""
from sqlalchemy import *
from migrate import *
import datetime
now = datetime.datetime.utcnow
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
import sys, logging
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler( sys.stdout )
format = "%(name)s %(levelname)s %(asctime)s %(message)s"
formatter = logging.Formatter( format )
handler.setFormatter( formatter )
log.addHandler( handler )
metadata = MetaData( migrate_engine )
# New table in changeset TODO:TODO
GenomeIndexToolData_table = Table( "genome_index_tool_data", metadata,
Column( "id", Integer, primary_key=True ),
Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ),
Column( "dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ),
Column( "deferred_job_id", Integer, ForeignKey( "deferred_job.id" ), index=True ),
Column( "transfer_job_id", Integer, ForeignKey( "transfer_job.id" ), index=True ),
Column( "fasta_path", String( 255 ) ),
Column( "created_time", DateTime, default=now ),
Column( "modified_time", DateTime, default=now, onupdate=now ),
Column( "indexer", String( 64 ) ),
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
)
def upgrade():
print __doc__
metadata.reflect()
try:
GenomeIndexToolData_table.create()
except Exception, e:
log.debug( "Creating genome_index_tool_data table failed: %s" % str( e ) )
def downgrade():
metadata.reflect()
try:
GenomeIndexToolData_table.drop()
except Exception, e:
log.debug( "Dropping genome_index_tool_data table failed: %s" % str( e ) )
lib/galaxy/objectstore/__init__.py
+4 -1
View File
@@ -223,7 +223,7 @@ class DiskObjectStore(ObjectStore):
if not os.path.exists(path):
return self._construct_path(obj, base_dir=base_dir, dir_only=dir_only, extra_dir=extra_dir, extra_dir_at_root=extra_dir_at_root, alt_name=alt_name)
def _construct_path(self, obj, old_style=False, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None):
def _construct_path(self, obj, old_style=False, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, **kwargs):
""" Construct the expected absolute path for accessing the object
identified by `obj`.id.
@@ -963,6 +963,9 @@ class DistributedObjectStore(ObjectStore):
return self.__call_method('get_filename', obj, ObjectNotFound, True, **kwargs)
def update_from_file(self, obj, **kwargs):
if kwargs.get('create', False):
self.create(obj, **kwargs)
kwargs['create'] = False
return self.__call_method('update_from_file', obj, ObjectNotFound, True, **kwargs)
def get_object_url(self, obj, **kwargs):
lib/galaxy/openid/providers.py
+17 -2
View File
@@ -9,6 +9,9 @@ from galaxy.util.odict import odict
log = logging.getLogger( __name__ )
NO_PROVIDER_ID = 'None'
RESERVED_PROVIDER_IDS = [ NO_PROVIDER_ID ]
class OpenIDProvider( object ):
'''An OpenID Provider object.'''
@classmethod
@@ -22,7 +25,9 @@ class OpenIDProvider( object ):
op_endpoint_url = provider_elem.find( 'op_endpoint_url' )
if op_endpoint_url is not None:
op_endpoint_url = op_endpoint_url.text
never_associate_with_user = string_as_bool( provider_elem.get( 'never_associate_with_user', 'False' ) )
assert (provider_id and provider_name and op_endpoint_url), Exception( "OpenID Provider improperly configured" )
assert provider_id not in RESERVED_PROVIDER_IDS, Exception( 'Specified OpenID Provider uses a reserved id: %s' % ( provider_id ) )
sreg_required = []
sreg_optional = []
use_for = {}
@@ -45,8 +50,8 @@ class OpenIDProvider( object ):
sreg_required = None
sreg_optional = None
use_for = None
return cls( provider_id, provider_name, op_endpoint_url, sreg_required, sreg_optional, use_for, store_user_preference )
def __init__( self, id, name, op_endpoint_url, sreg_required=None, sreg_optional=None, use_for=None, store_user_preference=None ):
return cls( provider_id, provider_name, op_endpoint_url, sreg_required=sreg_required, sreg_optional=sreg_optional, use_for=use_for, store_user_preference=store_user_preference, never_associate_with_user=never_associate_with_user )
def __init__( self, id, name, op_endpoint_url, sreg_required=None, sreg_optional=None, use_for=None, store_user_preference=None, never_associate_with_user=None ):
'''When sreg options are not specified, defaults are used.'''
self.id = id
self.name = name
@@ -71,6 +76,10 @@ class OpenIDProvider( object ):
self.store_user_preference = store_user_preference
else:
self.store_user_preference = {}
if never_associate_with_user:
self.never_associate_with_user = True
else:
self.never_associate_with_user = False
def post_authentication( self, trans, openid_manager, info ):
sreg_attributes = openid_manager.get_sreg( info )
for store_pref_name, store_pref_value_name in self.store_user_preference.iteritems():
@@ -80,9 +89,12 @@ class OpenIDProvider( object ):
raise Exception( 'Only sreg is currently supported.' )
trans.sa_session.add( trans.user )
trans.sa_session.flush()
def has_post_authentication_actions( self ):
return bool( self.store_user_preference )
class OpenIDProviders( object ):
'''Collection of OpenID Providers'''
NO_PROVIDER_ID = NO_PROVIDER_ID
@classmethod
def from_file( cls, filename ):
try:
@@ -107,6 +119,7 @@ class OpenIDProviders( object ):
self.providers = providers
else:
self.providers = odict()
self._banned_identifiers = [ provider.op_endpoint_url for provider in self.providers.itervalues() if provider.never_associate_with_user ]
def __iter__( self ):
for provider in self.providers.itervalues():
yield provider
@@ -115,3 +128,5 @@ class OpenIDProviders( object ):
return self.providers[ name ]
else:
return default
def new_provider_from_identifier( self, identifier ):
return OpenIDProvider( None, identifier, identifier, never_associate_with_user = identifier in self._banned_identifiers )
lib/galaxy/security/validate_user_input.py
+16
View File
@@ -1,6 +1,8 @@
import re
VALID_PUBLICNAME_RE = re.compile( "^[a-z0-9\-]+$" )
VALID_PUBLICNAME_SUB = re.compile( "[^a-z0-9\-]" )
FILL_CHAR = '-'
def validate_email( trans, email, user=None, check_dup=True ):
message = ''
@@ -31,6 +33,20 @@ def validate_publicname( trans, publicname, user=None ):
return "Public name is taken; please choose another"
return ''
def transform_publicname( trans, publicname, user=None ):
# User names must be at least four characters in length and contain only lower-case
# letters, numbers, and the '-' character.
#TODO: Enhance to allow generation of semi-random publicnnames e.g., when valid but taken
if user and user.username == publicname:
return publicname
elif publicname not in [ 'None', None, '' ]:
publicname = publicname.lower()
publicname = re.sub( VALID_PUBLICNAME_SUB, FILL_CHAR, publicname )
publicname = publicname.ljust( 4, FILL_CHAR )[:255]
if not trans.sa_session.query( trans.app.model.User ).filter_by( username=publicname ).first():
return publicname
return ''
def validate_password( trans, password, confirm ):
if len( password ) < 6:
return "Use a password of at least 6 characters"
lib/galaxy/tool_shed/install_manager.py
+61 -69
View File
@@ -7,6 +7,7 @@ from galaxy.tools import ToolSection
from galaxy.util.json import from_json_string, to_json_string
from galaxy.util.shed_util import *
from galaxy.util.odict import odict
log = logging.getLogger( __name__ )
class InstallManager( object ):
@@ -119,7 +120,7 @@ class InstallManager( object ):
is_displayed = True
return is_displayed, tool_sections
def handle_repository_contents( self, current_working_dir, repository_clone_url, relative_install_dir, repository_elem, repository_name, description,
changeset_revision, tmp_name ):
changeset_revision, ctx_rev ):
# Generate the metadata for the installed tool shed repository, among other things. It is critical that the installed repository is
# updated to the desired changeset_revision before metadata is set because the process for setting metadata uses the repository files on disk.
# The values for the keys in each of the following dictionaries will be a list to allow for the same tool to be displayed in multiple places
@@ -144,6 +145,7 @@ class InstallManager( object ):
repository_name,
description,
changeset_revision,
ctx_rev,
repository_clone_url,
metadata_dict,
dist_to_shed=True )
@@ -154,9 +156,15 @@ class InstallManager( object ):
# Handle missing data table entries for tool parameters that are dynamically generated select lists.
repository_tools_tups = handle_missing_data_table_entry( self.app, self.tool_path, sample_files, repository_tools_tups )
# Handle missing index files for tool parameters that are dynamically generated select lists.
repository_tools_tups = handle_missing_index_file( self.app, self.tool_path, sample_files, repository_tools_tups )
# Handle tools that use fabric scripts to install dependencies.
handle_tool_dependencies( current_working_dir, relative_install_dir, repository_tools_tups )
repository_tools_tups, sample_files_copied = handle_missing_index_file( self.app, self.tool_path, sample_files, repository_tools_tups )
# Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance.
copy_sample_files( self.app, sample_files, sample_files_copied=sample_files_copied )
if 'tool_dependencies_config' in metadata_dict:
# Install tool dependencies.
status, message = handle_tool_dependencies( self.app, repository_clone_url, metadata_dict[ 'tool_dependencies_config' ] )
if status != 'ok' and message:
print 'The following error occurred while installing tool dependencies:'
print message
add_to_tool_panel( self.app,
repository_name,
repository_clone_url,
@@ -166,11 +174,6 @@ class InstallManager( object ):
self.migrated_tools_config,
tool_panel_dict=tool_panel_dict_for_display,
new_install=True )
# Remove the temporary file
try:
os.unlink( tmp_name )
except:
pass
if 'datatypes_config' in metadata_dict:
datatypes_config = os.path.abspath( metadata_dict[ 'datatypes_config' ] )
# Load proprietary data types required by tools. The value of override is not important here since the Galaxy server will be started
@@ -193,7 +196,7 @@ class InstallManager( object ):
self.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict )
return tool_shed_repository, metadata_dict
def install_repository( self, repository_elem ):
# Install a single repository, loading contained tools into the tool config.
# Install a single repository, loading contained tools into the tool panel.
name = repository_elem.get( 'name' )
description = repository_elem.get( 'description' )
changeset_revision = repository_elem.get( 'changeset_revision' )
@@ -206,65 +209,55 @@ class InstallManager( object ):
tool_shed_url = self.__get_url_from_tool_shed( self.tool_shed )
repository_clone_url = os.path.join( tool_shed_url, 'repos', self.repository_owner, name )
relative_install_dir = os.path.join( clone_dir, name )
returncode, tmp_name = clone_repository( name, clone_dir, current_working_dir, repository_clone_url )
if returncode == 0:
returncode, tmp_name = update_repository( current_working_dir, relative_install_dir, changeset_revision )
if returncode == 0:
tool_shed_repository, metadata_dict = self.handle_repository_contents( current_working_dir,
repository_clone_url,
relative_install_dir,
repository_elem,
name,
description,
changeset_revision,
tmp_name )
if 'tools' in metadata_dict:
# Get the tool_versions from the tool shed for each tool in the installed change set.
url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \
( tool_shed_url, tool_shed_repository.name, self.repository_owner, changeset_revision )
response = urllib2.urlopen( url )
text = response.read()
response.close()
if text:
tool_version_dicts = from_json_string( text )
handle_tool_versions( self.app, tool_version_dicts, tool_shed_repository )
else:
# Set the tool versions since they seem to be missing for this repository in the tool shed.
# CRITICAL NOTE: These default settings may not properly handle all parent/child associations.
for tool_dict in metadata_dict[ 'tools' ]:
flush_needed = False
tool_id = tool_dict[ 'guid' ]
old_tool_id = tool_dict[ 'id' ]
tool_version = tool_dict[ 'version' ]
tool_version_using_old_id = get_tool_version( self.app, old_tool_id )
tool_version_using_guid = get_tool_version( self.app, tool_id )
if not tool_version_using_old_id:
tool_version_using_old_id = self.app.model.ToolVersion( tool_id=old_tool_id,
tool_shed_repository=tool_shed_repository )
self.app.sa_session.add( tool_version_using_old_id )
self.app.sa_session.flush()
if not tool_version_using_guid:
tool_version_using_guid = self.app.model.ToolVersion( tool_id=tool_id,
tool_shed_repository=tool_shed_repository )
self.app.sa_session.add( tool_version_using_guid )
self.app.sa_session.flush()
# Associate the two versions as parent / child.
tool_version_association = get_tool_version_association( self.app,
tool_version_using_old_id,
tool_version_using_guid )
if not tool_version_association:
tool_version_association = self.app.model.ToolVersionAssociation( tool_id=tool_version_using_guid.id,
parent_id=tool_version_using_old_id.id )
self.app.sa_session.add( tool_version_association )
self.app.sa_session.flush()
ctx_rev = get_ctx_rev( tool_shed_url, name, self.repository_owner, changeset_revision )
clone_repository( repository_clone_url, os.path.abspath( relative_install_dir ), ctx_rev )
tool_shed_repository, metadata_dict = self.handle_repository_contents( current_working_dir,
repository_clone_url,
relative_install_dir,
repository_elem,
name,
description,
changeset_revision,
ctx_rev )
if 'tools' in metadata_dict:
# Get the tool_versions from the tool shed for each tool in the installed change set.
url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy&no_reset=true' % \
( tool_shed_url, tool_shed_repository.name, self.repository_owner, changeset_revision )
response = urllib2.urlopen( url )
text = response.read()
response.close()
if text:
tool_version_dicts = from_json_string( text )
handle_tool_versions( self.app, tool_version_dicts, tool_shed_repository )
else:
tmp_stderr = open( tmp_name, 'rb' )
print "Error updating repository ', name, "': ', str( tmp_stderr.read() )
tmp_stderr.close()
else:
tmp_stderr = open( tmp_name, 'rb' )
print "Error cloning repository '", name, "': ", str( tmp_stderr.read() )
tmp_stderr.close()
# Set the tool versions since they seem to be missing for this repository in the tool shed.
# CRITICAL NOTE: These default settings may not properly handle all parent/child associations.
for tool_dict in metadata_dict[ 'tools' ]:
flush_needed = False
tool_id = tool_dict[ 'guid' ]
old_tool_id = tool_dict[ 'id' ]
tool_version = tool_dict[ 'version' ]
tool_version_using_old_id = get_tool_version( self.app, old_tool_id )
tool_version_using_guid = get_tool_version( self.app, tool_id )
if not tool_version_using_old_id:
tool_version_using_old_id = self.app.model.ToolVersion( tool_id=old_tool_id,
tool_shed_repository=tool_shed_repository )
self.app.sa_session.add( tool_version_using_old_id )
self.app.sa_session.flush()
if not tool_version_using_guid:
tool_version_using_guid = self.app.model.ToolVersion( tool_id=tool_id,
tool_shed_repository=tool_shed_repository )
self.app.sa_session.add( tool_version_using_guid )
self.app.sa_session.flush()
# Associate the two versions as parent / child.
tool_version_association = get_tool_version_association( self.app,
tool_version_using_old_id,
tool_version_using_guid )
if not tool_version_association:
tool_version_association = self.app.model.ToolVersionAssociation( tool_id=tool_version_using_guid.id,
parent_id=tool_version_using_old_id.id )
self.app.sa_session.add( tool_version_association )
self.app.sa_session.flush()
@property
def non_shed_tool_panel_configs( self ):
# Get the non-shed related tool panel config file names from the Galaxy config - the default is tool_conf.xml.
@@ -276,7 +269,6 @@ class InstallManager( object ):
root = tree.getroot()
tool_path = root.get( 'tool_path', None )
if tool_path is None:
# There will be a problem here if the user has defined 2 non-shed related configs.
config_filenames.append( config_filename )
return config_filenames
def __get_url_from_tool_shed( self, tool_shed ):
lib/galaxy/tool_shed/migrate/check.py
+46 -45
View File
@@ -54,51 +54,52 @@ def verify_tools( app, url, galaxy_config_file, engine_options={} ):
config_arg = ''
if os.path.abspath( os.path.join( os.getcwd(), 'universe_wsgi.ini' ) ) != galaxy_config_file:
config_arg = ' -c %s' % galaxy_config_file.replace( os.path.abspath( os.getcwd() ), '.' )
# Automatically update the value of the migrate_tools.version database table column.
cmd = 'sh manage_tools.sh%s upgrade' % config_arg
proc = subprocess.Popen( args=cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
return_code = proc.wait()
output = proc.stdout.read( 32768 )
if return_code != 0:
raise Exception( "Error attempting to update the value of migrate_tools.version: %s" % output )
elif missing_tool_configs:
if len( tool_panel_configs ) == 1:
plural = ''
tool_panel_config_file_names = tool_panel_configs[ 0 ]
else:
plural = 's'
tool_panel_config_file_names = ', '.join( tool_panel_configs )
msg = "\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
msg += "\n\nThe list of files at the end of this message refers to tools that are configured to load into the tool panel for\n"
msg += "this Galaxy instance, but have been removed from the Galaxy distribution. These tools can be automatically installed\n"
msg += "from the Galaxy tool shed at http://toolshed.g2.bx.psu.edu.\n\n"
msg += "To skip this process, attempt to start your Galaxy server again (e.g., sh run.sh or whatever you use). If you do this,\n"
msg += "be aware that these tools will no longer be available in your Galaxy tool panel, and entries for each of them should\n"
msg += "be removed from your file%s named %s.\n\n" % ( plural, tool_panel_config_file_names )
msg += "CRITICAL NOTE IF YOU PLAN TO INSTALL\n"
msg += "The location in which the tool repositories will be installed is the value of the 'tool_path' attribute in the <tool>\n"
msg += 'tag of the file named ./migrated_tool_conf.xml (i.e., <toolbox tool_path="../shed_tools">). The default location\n'
msg += "setting is '../shed_tools', which may be problematic for some cluster environments, so make sure to change it before\n"
msg += "you execute the installation process if appropriate. The configured location must be outside of the Galaxy installation\n"
msg += "directory or it must be in a sub-directory protected by a properly configured .hgignore file if the directory is within\n"
msg += "the Galaxy installation directory hierarchy. This is because tool shed repositories will be installed using mercurial's\n"
msg += "clone feature, which creates .hg directories and associated mercurial repository files. Not having .hgignore properly\n"
msg += "configured could result in undesired behavior when modifying or updating your local Galaxy instance or the tool shed\n"
msg += "repositories if they are in directories that pose conflicts. See mercurial's .hgignore documentation at the following\n"
msg += "URL for details.\n\nhttp://mercurial.selenic.com/wiki/.hgignore\n\n"
msg += output
msg += "After the installation process finishes, you can start your Galaxy server. As part of this installation process,\n"
msg += "entries for each of the following tool config files will be added to the file named ./migrated_tool_conf.xml, so these\n"
msg += "tools will continue to be loaded into your tool panel. Because of this, existing entries for these files should be\n"
msg += "removed from your file%s named %s, but only after the installation process finishes.\n\n" % ( plural, tool_panel_config_file_names )
for i, missing_tool_config in enumerate( missing_tool_configs ):
msg += "%s\n" % missing_tool_config
# Should we do the following?
#if i > 10:
# msg += "\n...and %d more tools...\n" % ( len( missing_tool_configs ) - ( i + 1 ) )
# break
msg += "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n"
raise Exception( msg )
if not app.config.running_functional_tests:
# Automatically update the value of the migrate_tools.version database table column.
cmd = 'sh manage_tools.sh%s upgrade' % config_arg
proc = subprocess.Popen( args=cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
return_code = proc.wait()
output = proc.stdout.read( 32768 )
if return_code != 0:
raise Exception( "Error attempting to update the value of migrate_tools.version: %s" % output )
elif missing_tool_configs:
if len( tool_panel_configs ) == 1:
plural = ''
tool_panel_config_file_names = tool_panel_configs[ 0 ]
else:
plural = 's'
tool_panel_config_file_names = ', '.join( tool_panel_configs )
msg = "\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
msg += "\n\nThe list of files at the end of this message refers to tools that are configured to load into the tool panel for\n"
msg += "this Galaxy instance, but have been removed from the Galaxy distribution. These tools can be automatically installed\n"
msg += "from the Galaxy tool shed at http://toolshed.g2.bx.psu.edu.\n\n"
msg += "To skip this process, attempt to start your Galaxy server again (e.g., sh run.sh or whatever you use). If you do this,\n"
msg += "be aware that these tools will no longer be available in your Galaxy tool panel, and entries for each of them should\n"
msg += "be removed from your file%s named %s.\n\n" % ( plural, tool_panel_config_file_names )
msg += "CRITICAL NOTE IF YOU PLAN TO INSTALL\n"
msg += "The location in which the tool repositories will be installed is the value of the 'tool_path' attribute in the <tool>\n"
msg += 'tag of the file named ./migrated_tool_conf.xml (i.e., <toolbox tool_path="../shed_tools">). The default location\n'
msg += "setting is '../shed_tools', which may be problematic for some cluster environments, so make sure to change it before\n"
msg += "you execute the installation process if appropriate. The configured location must be outside of the Galaxy installation\n"
msg += "directory or it must be in a sub-directory protected by a properly configured .hgignore file if the directory is within\n"
msg += "the Galaxy installation directory hierarchy. This is because tool shed repositories will be installed using mercurial's\n"
msg += "clone feature, which creates .hg directories and associated mercurial repository files. Not having .hgignore properly\n"
msg += "configured could result in undesired behavior when modifying or updating your local Galaxy instance or the tool shed\n"
msg += "repositories if they are in directories that pose conflicts. See mercurial's .hgignore documentation at the following\n"
msg += "URL for details.\n\nhttp://mercurial.selenic.com/wiki/.hgignore\n\n"
msg += output
msg += "After the installation process finishes, you can start your Galaxy server. As part of this installation process,\n"
msg += "entries for each of the following tool config files will be added to the file named ./migrated_tool_conf.xml, so these\n"
msg += "tools will continue to be loaded into your tool panel. Because of this, existing entries for these files should be\n"
msg += "removed from your file%s named %s, but only after the installation process finishes.\n\n" % ( plural, tool_panel_config_file_names )
for i, missing_tool_config in enumerate( missing_tool_configs ):
msg += "%s\n" % missing_tool_config
# Should we do the following?
#if i > 10:
# msg += "\n...and %d more tools...\n" % ( len( missing_tool_configs ) - ( i + 1 ) )
# break
msg += "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n"
raise Exception( msg )
else:
log.info( "At migrate_tools version %d" % db_schema.version )
lib/galaxy/tool_shed/update_manager.py
+1 -1
View File
@@ -34,7 +34,7 @@ class UpdateManager( object ):
log.info( 'Transfer job restarter shutting down...' )
def check_for_update( self, repository ):
tool_shed_url = get_url_from_repository_tool_shed( self.app, repository )
url = '%s/repository/check_for_updates?name=%s&owner=%s&changeset_revision=%s&webapp=update_manager' % \
url = '%s/repository/check_for_updates?name=%s&owner=%s&changeset_revision=%s&webapp=update_manager&no_reset=true' % \
( tool_shed_url, repository.name, repository.owner, repository.changeset_revision )
response = urllib2.urlopen( url )
text = response.read()
lib/galaxy/tools/__init__.py
+261 -71
View File
@@ -5,7 +5,7 @@ import pkg_resources
pkg_resources.require( "simplejson" )
import logging, os, string, sys, tempfile, glob, shutil, types, urllib, subprocess
import logging, os, string, sys, tempfile, glob, shutil, types, urllib, subprocess, random
import simplejson
import binascii
from UserDict import DictMixin
@@ -31,6 +31,7 @@ from cgi import FieldStorage
from galaxy.util.hash_util import *
from galaxy.util import listify
from galaxy.util.shed_util import *
from galaxy.web import url_for
from galaxy.visualization.tracks.visual_analytics import TracksterConfig
@@ -114,17 +115,17 @@ class ToolBox( object ):
tool_path = self.tool_root_dir
# Only load the panel_dict under certain conditions.
load_panel_dict = not self.integrated_tool_panel_config_has_contents
for elem in root:
for index, elem in enumerate( root ):
if parsing_shed_tool_conf:
config_elems.append( elem )
if elem.tag == 'tool':
self.load_tool_tag_set( elem, self.tool_panel, self.integrated_tool_panel, tool_path, load_panel_dict, guid=elem.get( 'guid' ) )
self.load_tool_tag_set( elem, self.tool_panel, self.integrated_tool_panel, tool_path, load_panel_dict, guid=elem.get( 'guid' ), index=index )
elif elem.tag == 'workflow':
self.load_workflow_tag_set( elem, self.tool_panel, self.integrated_tool_panel, load_panel_dict )
self.load_workflow_tag_set( elem, self.tool_panel, self.integrated_tool_panel, load_panel_dict, index=index )
elif elem.tag == 'section':
self.load_section_tag_set( elem, tool_path, load_panel_dict )
self.load_section_tag_set( elem, tool_path, load_panel_dict, index=index )
elif elem.tag == 'label':
self.load_label_tag_set( elem, self.tool_panel, self.integrated_tool_panel )
self.load_label_tag_set( elem, self.tool_panel, self.integrated_tool_panel, load_panel_dict, index=index )
if parsing_shed_tool_conf:
shed_tool_conf_dict = dict( config_filename=config_filename,
tool_path=tool_path,
@@ -286,7 +287,7 @@ class ToolBox( object ):
self.app.model.ToolShedRepository.table.c.owner == owner,
self.app.model.ToolShedRepository.table.c.installed_changeset_revision == installed_changeset_revision ) ) \
.first()
def load_tool_tag_set( self, elem, panel_dict, integrated_panel_dict, tool_path, load_panel_dict, guid=None ):
def load_tool_tag_set( self, elem, panel_dict, integrated_panel_dict, tool_path, load_panel_dict, guid=None, index=None ):
try:
path = elem.get( "file" )
if guid is None:
@@ -354,10 +355,13 @@ class ToolBox( object ):
if load_panel_dict:
panel_dict[ key ] = tool
# Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file.
integrated_panel_dict[ key ] = tool
if key in integrated_panel_dict or index is None:
integrated_panel_dict[ key ] = tool
else:
integrated_panel_dict.insert( index, key, tool )
except:
log.exception( "Error reading tool from path: %s" % path )
def load_workflow_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict ):
def load_workflow_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ):
try:
# TODO: should id be encoded?
workflow_id = elem.get( 'id' )
@@ -367,16 +371,22 @@ class ToolBox( object ):
if load_panel_dict:
panel_dict[ key ] = workflow
# Always load workflows into the integrated_panel_dict.
integrated_panel_dict[ key ] = workflow
if key in integrated_panel_dict or index is None:
integrated_panel_dict[ key ] = workflow
else:
integrated_panel_dict.insert( index, key, workflow )
except:
log.exception( "Error loading workflow: %s" % workflow_id )
def load_label_tag_set( self, elem, panel_dict, integrated_panel_dict ):
def load_label_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ):
label = ToolSectionLabel( elem )
key = 'label_' + label.id
if not self.integrated_tool_panel_config_has_contents:
if load_panel_dict:
panel_dict[ key ] = label
integrated_panel_dict[ key ] = label
def load_section_tag_set( self, elem, tool_path, load_panel_dict ):
if key in integrated_panel_dict or index is None:
integrated_panel_dict[ key ] = label
else:
integrated_panel_dict.insert( index, key, label )
def load_section_tag_set( self, elem, tool_path, load_panel_dict, index=None ):
key = 'section_' + elem.get( "id" )
if key in self.tool_panel:
section = self.tool_panel[ key ]
@@ -390,17 +400,20 @@ class ToolBox( object ):
else:
integrated_section = ToolSection( elem )
integrated_elems = integrated_section.elems
for sub_elem in elem:
for sub_index, sub_elem in enumerate( elem ):
if sub_elem.tag == 'tool':
self.load_tool_tag_set( sub_elem, elems, integrated_elems, tool_path, load_panel_dict, guid=sub_elem.get( 'guid' ) )
self.load_tool_tag_set( sub_elem, elems, integrated_elems, tool_path, load_panel_dict, guid=sub_elem.get( 'guid' ), index=sub_index )
elif sub_elem.tag == 'workflow':
self.load_workflow_tag_set( sub_elem, elems, integrated_elems, load_panel_dict )
self.load_workflow_tag_set( sub_elem, elems, integrated_elems, load_panel_dict, index=sub_index )
elif sub_elem.tag == 'label':
self.load_label_tag_set( sub_elem, elems, integrated_elems )
self.load_label_tag_set( sub_elem, elems, integrated_elems, load_panel_dict, index=sub_index )
if load_panel_dict:
self.tool_panel[ key ] = section
# Always load sections into the integrated_tool_panel.
self.integrated_tool_panel[ key ] = integrated_section
if key in self.integrated_tool_panel or index is None:
self.integrated_tool_panel[ key ] = integrated_section
else:
self.integrated_tool_panel.insert( index, key, integrated_section )
def load_tool( self, config_file, guid=None ):
"""Load a single tool from the file named by `config_file` and return an instance of `Tool`."""
# Parse XML configuration file and get the root element
@@ -474,17 +487,119 @@ class ToolBox( object ):
Returns a SQLAlchemy session
"""
return self.app.model.context
def to_dict( self, trans, in_panel=True, trackster=False ):
def filter_for_panel( item, filters ):
"""
Filters tool panel elements so that only those that are compatible
with provided filters are kept.
"""
def _apply_filter( filter_item, filter_list ):
for filter_method in filter_list:
if not filter_method( filter_item ):
return False
return True
if isinstance( item, Tool ):
if _apply_filter( item, filters[ 'tool' ] ):
return item
elif isinstance( item, ToolSectionLabel ):
if _apply_filter( item, filters[ 'label' ] ):
return item
elif isinstance( item, ToolSection ):
# Filter section item-by-item. Only show a label if there are
# non-filtered tools below it.
if _apply_filter( item, filters[ 'section' ] ):
cur_label_key = None
tools_under_label = False
filtered_elems = item.elems.copy()
for key, section_item in item.elems.items():
if isinstance( section_item, Tool ):
# Filter tool.
if _apply_filter( section_item, filters[ 'tool' ] ):
tools_under_label = True
else:
del filtered_elems[ key ]
elif isinstance( section_item, ToolSectionLabel ):
# If there is a label and it does not have tools,
# remove it.
if ( cur_label_key and not tools_under_label ) or not _apply_filter( section_item, filters[ 'label' ] ):
del filtered_elems[ cur_label_key ]
# Reset attributes for new label.
cur_label_key = key
tools_under_label = False
# Handle last label.
if cur_label_key and not tools_under_label:
del filtered_elems[ cur_label_key ]
# Only return section if there are elements.
if len( filtered_elems ) != 0:
copy = item.copy()
copy.elems = filtered_elems
return copy
return None
#
# Dictify toolbox.
#
if in_panel:
panel_elts = [ val for val in self.tool_panel.itervalues() ]
# Filter if necessary.
filters = dict( tool=[ lambda x: not x._is_hidden_for_user( trans.user ) ], section=[], label=[] ) #hidden tools filter
if trackster:
filters[ 'tool' ].append( lambda x: x.trackster_conf ) # If tool has a trackster config, it can be used in Trackster.
filtered_panel_elts = []
for index, elt in enumerate( panel_elts ):
elt = filter_for_panel( elt, filters )
if elt:
filtered_panel_elts.append( elt )
panel_elts = filtered_panel_elts
# Produce panel.
rval = []
for elt in panel_elts:
rval.append( elt.to_dict( trans, for_link=True ) )
else:
tools = []
for id, tool in self.app.toolbox.tools_by_id.items():
tools.append( tool.to_dict( trans ) )
rval = tools
return rval
class ToolSection( object ):
"""
A group of tools with similar type/purpose that will be displayed as a
group in the user interface.
"""
def __init__( self, elem ):
self.name = elem.get( "name" )
self.id = elem.get( "id" )
self.version = elem.get( "version" ) or ''
def __init__( self, elem=None ):
f = lambda elem, val: elem is not None and elem.get( val ) or ''
self.name = f( elem, 'name' )
self.id = f( elem, 'id' )
self.version = f( elem, 'version' )
self.elems = odict()
def copy( self ):
copy = ToolSection()
copy.name = self.name
copy.id = self.id
copy.version = self.version
copy.elems = self.elems.copy()
return copy
def to_dict( self, trans, for_link=False ):
""" Return a dict that includes section's attributes. """
section_elts = []
for key, val in self.elems.items():
section_elts.append( val.to_dict( trans, for_link=for_link ) )
return { 'type': 'section', 'id': self.id, 'name': self.name, 'version': self.version, 'elems': section_elts }
class ToolSectionLabel( object ):
"""
@@ -495,6 +610,10 @@ class ToolSectionLabel( object ):
self.text = elem.get( "text" )
self.id = elem.get( "id" )
self.version = elem.get( "version" ) or ''
def to_dict( self, trans, **kwargs ):
""" Return a dict that includes label's attributes. """
return { 'type': 'label', 'id': self.id, 'name': self.text, 'version': self.version }
class DefaultToolState( object ):
"""
@@ -572,22 +691,24 @@ class ToolOutput( object ):
def __iter__( self ):
return iter( ( self.format, self.metadata_source, self.parent ) )
def to_dict( self ):
return {
'name': self.name,
'format': self.format,
'label': self.label,
'hidden': self.hidden
}
class ToolRequirement( object ):
"""
Represents an external requirement that must be available for the tool to
run (for example, a program, package, or library). Requirements can
optionally assert a specific version, or reference a command to execute a
fabric script. If fabric is used, the type is 'fabfile' and the version
attribute is not used since the fabric script includes all necessary
information for automatic dependency installation.
Represents an external requirement that must be available for the tool to run (for example, a program, package, or library).
Requirements can optionally assert a specific version.
"""
def __init__( self, name=None, type=None, version=None, fabfile=None, method=None ):
def __init__( self, name=None, type=None, version=None ):
self.name = name
self.type = type
self.version = version
self.fabfile = fabfile
self.method = method
class ToolParallelismInfo(object):
"""
@@ -625,6 +746,8 @@ class Tool:
self.check_values = True
self.nginx_upload = False
self.input_required = False
self.display_interface = True
self.require_login = False
# Define a place to keep track of all input parameters. These
# differ from the inputs dictionary in that inputs can be page
# elements like conditionals, but input_params are basic form
@@ -669,31 +792,35 @@ class Tool:
if tool_version:
return tool_version.get_version_ids( self.app )
return []
def get_job_runner( self, job_params=None ):
# Look through runners to find one with matching parameters.
selected_runner = None
if len( self.job_runners ) == 1:
# Most tools have a single runner.
selected_runner = self.job_runners[0]
def __get_job_run_config( self, run_configs, key, job_params=None ):
# Look through runners/handlers to find one with matching parameters.
available_configs = []
if len( run_configs ) == 1:
# Most tools have a single config.
return run_configs[0][ key ] # return to avoid random when this will be the case most of the time
elif job_params is None:
# Use job runner with no params
for runner in self.job_runners:
if "params" not in runner:
selected_runner = runner
# Use job config with no params
for config in run_configs:
if "params" not in config:
available_configs.append( config )
else:
# Find runner with matching parameters.
for runner in self.job_runners:
if "params" in runner:
# Find config with matching parameters.
for config in run_configs:
if "params" in config:
match = True
runner_params = runner[ "params" ]
config_params = config[ "params" ]
for param, value in job_params.items():
if param not in runner_params or \
runner_params[ param ] != job_params[ param ]:
if param not in config_params or \
config_params[ param ] != job_params[ param ]:
match = False
break
if match:
selected_runner = runner
return selected_runner[ "url" ]
available_configs.append( config )
return random.choice( available_configs )[ key ]
def get_job_runner( self, job_params=None ):
return self.__get_job_run_config( self.job_runners, key='url', job_params=job_params )
def get_job_handler( self, job_params=None ):
return self.__get_job_run_config( self.job_handlers, key='name', job_params=job_params )
def parse( self, root, guid=None ):
"""
Read tool configuration from the element `root` and fill in `self`.
@@ -721,6 +848,8 @@ class Tool:
# Useful i.e. when an indeterminate number of outputs are created by
# a tool.
self.force_history_refresh = util.string_as_bool( root.get( 'force_history_refresh', 'False' ) )
self.display_interface = util.string_as_bool( root.get( 'display_interface', str( self.display_interface ) ) )
self.require_login = util.string_as_bool( root.get( 'require_login', str( self.require_login ) ) )
# Load input translator, used by datasource tools to change
# names/values of incoming parameters
self.input_translator = root.find( "request_param_translation" )
@@ -758,6 +887,12 @@ class Tool:
self.parallelism = ToolParallelismInfo(parallelism)
else:
self.parallelism = None
# Set job handler(s). Each handler is a dict with 'url' and, optionally, 'params'.
self_id = self.id.lower()
self.job_handlers = [ { "name" : name } for name in self.app.config.default_job_handlers ]
# Set custom handler(s) if they're defined.
if self_id in self.app.config.tool_handlers:
self.job_handlers = self.app.config.tool_handlers[ self_id ]
# Set job runner(s). Each runner is a dict with 'url' and, optionally, 'params'.
if self.app.config.start_job_runners is None:
# Jobs are always local regardless of tool config if no additional
@@ -767,7 +902,6 @@ class Tool:
# Set job runner to the cluster default
self.job_runners = [ { "url" : self.app.config.default_cluster_job_runner } ]
# Set custom runner(s) if they're defined.
self_id = self.id.lower()
if self_id in self.app.config.tool_runners:
self.job_runners = self.app.config.tool_runners[ self_id ]
# Is this a 'hidden' tool (hidden in tool menu)
@@ -843,7 +977,7 @@ class Tool:
self.is_workflow_compatible = self.check_workflow_compatible()
# Trackster configuration.
trackster_conf = root.find( "trackster_conf" )
if trackster_conf:
if trackster_conf is not None:
self.trackster_conf = TracksterConfig.parse( trackster_conf )
else:
self.trackster_conf = None
@@ -1199,19 +1333,8 @@ class Tool:
for requirement_elem in requirements_elem.findall( 'requirement' ):
name = util.xml_text( requirement_elem )
type = requirement_elem.get( "type", "package" )
if type == 'fabfile':
# The fabric script will include all necessary information for
# automatically installing the tool dependencies.
fabfile = requirement_elem.get( "fabfile" )
method = requirement_elem.get( "method" )
version = None
else:
# For backward compatibility, requirements tag sets should not require the
# use of a fabric script.
version = requirement_elem.get( "version" )
fabfile = None
method = None
requirement = ToolRequirement( name=name, type=type, version=version, fabfile=fabfile, method=method )
version = requirement_elem.get( "version", None )
requirement = ToolRequirement( name=name, type=type, version=version )
self.requirements.append( requirement )
def check_workflow_compatible( self ):
"""
@@ -1326,6 +1449,8 @@ class Tool:
# on the standard run form) or "URL" (a parameter provided by
# external data source tools).
if "runtool_btn" not in incoming and "URL" not in incoming:
if not self.display_interface:
return 'message.mako', dict( status='info', message="The interface for this tool cannot be displayed", refresh_frames=['everything'] )
return "tool_form.mako", dict( errors={}, tool_state=state, param_values={}, incoming={} )
# Process incoming data
if not( self.check_values ):
@@ -1374,6 +1499,8 @@ class Tool:
state.page += 1
# Fill in the default values for the next page
self.fill_in_new_state( trans, self.inputs_by_page[ state.page ], state.inputs )
if not self.display_interface:
return 'message.mako', dict( status='info', message="The interface for this tool cannot be displayed", refresh_frames=['everything'] )
return 'tool_form.mako', dict( errors=errors, tool_state=state )
else:
try:
@@ -1386,6 +1513,8 @@ class Tool:
except:
pass
# Just a refresh, render the form with updated state and errors.
if not self.display_interface:
return 'message.mako', dict( status='info', message="The interface for this tool cannot be displayed", refresh_frames=['everything'] )
return 'tool_form.mako', dict( errors=errors, tool_state=state )
def find_fieldstorage( self, x ):
if isinstance( x, FieldStorage ):
@@ -2110,7 +2239,7 @@ class Tool:
shutil.rmtree(temp_file_path)
except:
continue
def collect_child_datasets( self, output):
def collect_child_datasets( self, output, job_working_directory ):
"""
Look for child dataset files, create HDA and attach to parent.
"""
@@ -2118,7 +2247,12 @@ class Tool:
# Loop through output file names, looking for generated children in
# form of 'child_parentId_designation_visibility_extension'
for name, outdata in output.items():
for filename in glob.glob(os.path.join(self.app.config.new_file_path,"child_%i_*" % outdata.id) ):
filenames = []
if 'new_file_path' in self.app.config.collect_outputs_from:
filenames.extend( glob.glob(os.path.join(self.app.config.new_file_path,"child_%i_*" % outdata.id) ) )
if 'job_working_directory' in self.app.config.collect_outputs_from:
filenames.extend( glob.glob(os.path.join(job_working_directory,"child_%i_*" % outdata.id) ) )
for filename in filenames:
if not name in children:
children[name] = {}
fields = os.path.basename(filename).split("_")
@@ -2138,7 +2272,7 @@ class Tool:
sa_session=self.sa_session )
self.app.security_agent.copy_dataset_permissions( outdata.dataset, child_dataset.dataset )
# Move data from temp location to dataset location
self.app.object_store.update_from_file(child_dataset.dataset, filename, create=True)
self.app.object_store.update_from_file(child_dataset.dataset, file_name=filename, create=True)
self.sa_session.add( child_dataset )
self.sa_session.flush()
child_dataset.set_size()
@@ -2170,7 +2304,7 @@ class Tool:
self.sa_session.add( child_dataset )
self.sa_session.flush()
return children
def collect_primary_datasets( self, output):
def collect_primary_datasets( self, output, job_working_directory ):
"""
Find any additional datasets generated by a tool and attach (for
cases where number of outputs is not known in advance).
@@ -2180,7 +2314,12 @@ class Tool:
# datasets in form of:
# 'primary_associatedWithDatasetID_designation_visibility_extension(_DBKEY)'
for name, outdata in output.items():
for filename in glob.glob(os.path.join(self.app.config.new_file_path,"primary_%i_*" % outdata.id) ):
filenames = []
if 'new_file_path' in self.app.config.collect_outputs_from:
filenames.extend( glob.glob(os.path.join(self.app.config.new_file_path,"primary_%i_*" % outdata.id) ) )
if 'job_working_directory' in self.app.config.collect_outputs_from:
filenames.extend( glob.glob(os.path.join(job_working_directory,"primary_%i_*" % outdata.id) ) )
for filename in filenames:
if not name in primary_datasets:
primary_datasets[name] = {}
fields = os.path.basename(filename).split("_")
@@ -2205,7 +2344,7 @@ class Tool:
self.sa_session.add( primary_data )
self.sa_session.flush()
# Move data from temp location to dataset location
self.app.object_store.update_from_file(primary_data.dataset, filename, create=True)
self.app.object_store.update_from_file(primary_data.dataset, file_name=filename, create=True)
primary_data.set_size()
primary_data.name = "%s (%s)" % ( outdata.name, designation )
primary_data.info = outdata.info
@@ -2238,6 +2377,54 @@ class Tool:
self.sa_session.add( new_data )
self.sa_session.flush()
return primary_datasets
def _is_hidden_for_user( self, user ):
if self.hidden or ( not user and self.require_login ):
return True
return False
def to_dict( self, trans, for_link=False, for_display=False ):
""" Returns dict of tool. """
# Basic information
tool_dict = { 'id': self.id, 'name': self.name,
'version': self.version, 'description': self.description }
if for_link:
# Create tool link.
if not self.tool_type.startswith( 'data_source' ):
link = url_for( controller='tool_runner', tool_id=self.id )
else:
link = url_for( self.action, **self.get_static_param_values( trans ) )
# Basic information
tool_dict.update( { 'type': 'tool', 'link': link,
'min_width': self.uihints.get( 'minwidth', -1 ),
'target': self.target } )
if for_display:
# Dictify inputs.
inputs = []
for name, input in self.inputs.items():
param_dict = { 'name' : name, 'label' : input.label }
if isinstance( input, DataToolParameter ):
param_dict.update( { 'type' : 'data', 'html' : urllib.quote( input.get_html( trans ) ) } )
elif isinstance( input, SelectToolParameter ):
param_dict.update( { 'type' : 'select', 'html' : urllib.quote( input.get_html( trans ) ) } )
elif isinstance( input, Conditional ):
# TODO.
pass
else:
param_dict.update( { 'type' : '??', 'init_value' : input.value, \
'html' : urllib.quote( input.get_html( trans ) ) } )
inputs.append( param_dict )
tool_dict[ 'inputs' ] = inputs
# Dictify outputs.
pass
return tool_dict
class DataSourceTool( Tool ):
"""
@@ -2369,6 +2556,9 @@ class ExportHistoryTool( Tool ):
class ImportHistoryTool( Tool ):
tool_type = 'import_history'
class GenomeIndexTool( Tool ):
tool_type = 'index_genome'
# Populate tool_type to ToolClass mappings
tool_types = {}
for tool_class in [ Tool, DataDestinationTool, SetMetadataTool, DataSourceTool, AsyncDataSourceTool ]:
lib/galaxy/tools/actions/index_genome.py
+65
View File
@@ -0,0 +1,65 @@
import tempfile
from __init__ import ToolAction
from galaxy.util.odict import odict
from galaxy.tools.genome_index import *
import logging
log = logging.getLogger( __name__ )
class GenomeIndexToolAction( ToolAction ):
"""Tool action used for exporting a history to an archive. """
def execute( self, tool, trans, *args, **kwargs ):
#
# Get genome to index.
#
incoming = kwargs['incoming']
#
# Create the job and output dataset objects
#
job = trans.app.model.Job()
job.tool_id = tool.id
job.user_id = incoming['user']
start_job_state = job.state # should be job.states.NEW
job.state = job.states.WAITING # we need to set job state to something other than NEW, or else when tracking jobs in db it will be picked up before we have added input / output parameters
trans.sa_session.add( job )
# Create dataset that will serve as archive.
temp_dataset = trans.app.model.Dataset( state=trans.app.model.Dataset.states.NEW )
trans.sa_session.add( temp_dataset )
trans.sa_session.flush() # ensure job.id and archive_dataset.id are available
trans.app.object_store.create( temp_dataset ) # set the object store id, create dataset (because galaxy likes having datasets)
#
# Setup job and job wrapper.
#
# Add association for keeping track of job, history, archive relationship.
user = trans.sa_session.query( trans.app.model.User ).get( int( incoming['user'] ) )
assoc = trans.app.model.GenomeIndexToolData( job=job, dataset=temp_dataset, fasta_path=incoming['path'], \
indexer=incoming['indexer'], user=user, \
deferred_job=kwargs['deferred'], transfer_job=kwargs['transfer'] )
trans.sa_session.add( assoc )
job_wrapper = GenomeIndexToolWrapper( job )
cmd_line = job_wrapper.setup_job( assoc )
#
# Add parameters to job_parameter table.
#
# Set additional parameters.
incoming[ '__GENOME_INDEX_COMMAND__' ] = cmd_line
for name, value in tool.params_to_strings( incoming, trans.app ).iteritems():
job.add_parameter( name, value )
job.state = start_job_state # job inputs have been configured, restore initial job state
trans.sa_session.flush()
# Queue the job for execution
trans.app.job_queue.put( job.id, tool )
log.info( "Added genome index job to the job queue, id: %s" % str( job.id ) )
return job, odict()
lib/galaxy/tools/actions/metadata.py
+4 -1
View File
@@ -1,6 +1,7 @@
from __init__ import ToolAction
from galaxy.datatypes.metadata import JobExternalOutputMetadataWrapper
from galaxy.util.odict import odict
from galaxy.util.json import to_json_string
import logging
log = logging.getLogger( __name__ )
@@ -8,7 +9,7 @@ log = logging.getLogger( __name__ )
class SetMetadataToolAction( ToolAction ):
"""Tool action used for setting external metadata on an existing dataset"""
def execute( self, tool, trans, incoming = {}, set_output_hid = False, overwrite = True, history=None ):
def execute( self, tool, trans, incoming = {}, set_output_hid = False, overwrite = True, history=None, job_params=None ):
for name, value in incoming.iteritems():
if isinstance( value, trans.app.model.HistoryDatasetAssociation ):
dataset = value
@@ -30,6 +31,8 @@ class SetMetadataToolAction( ToolAction ):
job.tool_id = tool.id
if trans.user:
job.user_id = trans.user.id
if job_params:
job.params = to_json_string( job_params )
start_job_state = job.state #should be job.states.NEW
try:
# For backward compatibility, some tools may not have versions yet.
lib/galaxy/tools/genome_index/__init__.py
+184
View File
@@ -0,0 +1,184 @@
import os, shutil, logging, tempfile, json, tarfile
from galaxy import model, util
from galaxy.web.framework.helpers import to_unicode
from galaxy.model.item_attrs import UsesAnnotations
from galaxy.util.json import *
from galaxy.web.base.controller import UsesHistory
from galaxy.tools.data import ToolDataTableManager
log = logging.getLogger(__name__)
def load_genome_index_tools( toolbox ):
""" Adds tools for indexing genomes via the main job runner. """
# Use same process as that used in load_external_metadata_tool; see that
# method for why create tool description files on the fly.
tool_xml_text = """
<tool id="__GENOME_INDEX__" name="Index Genome" version="0.1" tool_type="genome_index">
<type class="GenomeIndexTool" module="galaxy.tools"/>
<action module="galaxy.tools.actions.index_genome" class="GenomeIndexToolAction"/>
<command>$__GENOME_INDEX_COMMAND__ $output_file $output_file.files_path</command>
<inputs>
<param name="__GENOME_INDEX_COMMAND__" type="hidden"/>
</inputs>
<outputs>
<data format="txt" name="output_file"/>
</outputs>
</tool>
"""
# Load export tool.
tmp_name = tempfile.NamedTemporaryFile()
tmp_name.write( tool_xml_text )
tmp_name.flush()
genome_index_tool = toolbox.load_tool( tmp_name.name )
toolbox.tools_by_id[ genome_index_tool.id ] = genome_index_tool
log.debug( "Loaded genome index tool: %s", genome_index_tool.id )
class GenomeIndexToolWrapper( object ):
""" Provides support for performing jobs that index a genome. """
def __init__( self, job_id ):
self.locations = dict()
self.job_id = job_id
def setup_job( self, genobj ):
""" Perform setup for job to index a genome and return an archive. Method generates
attribute files, sets the corresponding attributes in the associated database
object, and returns a command line for running the job. The command line
includes the command, inputs, and options; it does not include the output
file because it must be set at runtime. """
#
# Create and return command line for running tool.
#
scriptpath = os.path.join( os.path.abspath( os.getcwd() ), "lib/galaxy/tools/genome_index/index_genome.py" )
return "python %s %s %s" % ( scriptpath, genobj.indexer, genobj.fasta_path )
def postprocessing( self, sa_session, app ):
""" Finish the job, move the finished indexes to their final resting place,
and update the .loc files where applicable. """
gitd = sa_session.query( model.GenomeIndexToolData ).filter_by( job_id=self.job_id ).first()
indexdirs = dict( bfast='bfast_index', bowtie='bowtie_index', bowtie2='bowtie2_index',
bwa='bwa_index', perm='perm_%s_index', picard='srma_index', sam='sam_index' )
if gitd:
destination = None
tdtman = ToolDataTableManager()
xmltree = tdtman.load_from_config_file(app.config.tool_data_table_config_path)
for node in xmltree:
table = node.get('name')
location = node.findall('file')[0].get('path')
self.locations[table] = os.path.abspath( location )
locbase = os.path.abspath( os.path.split( self.locations['all_fasta'] )[0] )
deferred = sa_session.query( model.DeferredJob ).filter_by( id=gitd.deferred_job_id ).first()
params = deferred.params
dbkey = params[ 'dbkey' ]
basepath = os.path.join( os.path.abspath( app.config.genome_data_path ), dbkey )
intname = params[ 'intname' ]
indexer = gitd.indexer
workingdir = os.path.abspath( gitd.dataset.extra_files_path )
fp = open( gitd.dataset.get_file_name(), 'r' )
logloc = json.load( fp )
fp.close()
location = []
indexdata = gitd.dataset.extra_files_path
if indexer == '2bit':
indexdata = os.path.join( workingdir, '%s.2bit' % dbkey )
destination = os.path.join( basepath, 'seq', '%s.2bit' % dbkey )
location.append( dict( line='\t'.join( [ 'seq', dbkey, os.path.join( destination, '%s.2bit' % dbkey ) ] ), file= os.path.join( locbase, 'alignseq.loc' ) ) )
elif indexer == 'bowtie':
self._ex_tar( workingdir, 'cs.tar' )
destination = os.path.join( basepath, 'bowtie_index' )
for var in [ 'nt', 'cs' ]:
for line in logloc[ var ]:
idx = line
if var == 'nt':
locfile = self.locations[ 'bowtie_indexes' ]
locdir = os.path.join( destination, idx )
else:
locfile = self.locations[ 'bowtie_indexes_color' ]
locdir = os.path.join( destination, var, idx )
location.append( dict( line='\t'.join( [ dbkey, dbkey, intname, locdir ] ), file=locfile ) )
elif indexer == 'bowtie2':
destination = os.path.join( basepath, 'bowtie2_index' )
for line in logloc[ 'nt' ]:
idx = line
locfile = self.locations[ 'bowtie2_indexes' ]
locdir = os.path.join( destination, idx )
location.append( dict( line='\t'.join( [ dbkey, dbkey, intname, locdir ] ), file=locfile ) )
elif indexer == 'bwa':
self._ex_tar( workingdir, 'cs.tar' )
destination = os.path.join( basepath, 'bwa_index' )
for var in [ 'nt', 'cs' ]:
for line in logloc[ var ]:
idx = line
if var == 'nt':
locfile = self.locations[ 'bwa_indexes' ]
locdir = os.path.join( destination, idx )
else:
locfile = self.locations[ 'bwa_indexes_color' ]
locdir = os.path.join( destination, var, idx )
location.append( dict( line='\t'.join( [ dbkey, dbkey, intname, locdir ] ), file=locfile ) )
elif indexer == 'perm':
self._ex_tar( workingdir, 'cs.tar' )
destination = os.path.join( basepath, 'perm_index' )
for var in [ 'nt', 'cs' ]:
for line in logloc[ var ]:
idx = line.pop()
if var == 'nt':
locfile = self.locations[ 'perm_base_indexes' ]
locdir = os.path.join( destination, idx )
else:
locfile = self.locations[ 'perm_color_indexes' ]
locdir = os.path.join( destination, var, idx )
line.append( locdir )
location.append( dict( line='\t'.join( line ), file=locfile ) )
elif indexer == 'picard':
destination = os.path.join( basepath, 'srma_index' )
for var in [ 'nt' ]:
for line in logloc[ var ]:
idx = line
locfile = self.locations[ 'picard_indexes' ]
locdir = os.path.join( destination, idx )
location.append( dict( line='\t'.join( [ dbkey, dbkey, intname, locdir ] ), file=locfile ) )
elif indexer == 'sam':
destination = os.path.join( basepath, 'sam_index' )
for var in [ 'nt' ]:
for line in logloc[ var ]:
locfile = self.locations[ 'sam_fa_indexes' ]
locdir = os.path.join( destination, line )
location.append( dict( line='\t'.join( [ 'index', dbkey, locdir ] ), file=locfile ) )
if destination is not None and os.path.exists( os.path.split( destination )[0] ) and not os.path.exists( destination ):
log.debug( 'Moving %s to %s' % ( indexdata, destination ) )
shutil.move( indexdata, destination )
if indexer not in [ '2bit' ]:
genome = '%s.fa'
target = os.path.join( destination, genome )
farel = os.path.relpath( os.path.join( basepath, 'seq', genome ), destination )
os.symlink( farel, target )
if os.path.exists( os.path.join( destination, 'cs' ) ):
target = os.path.join( destination, 'cs', genome )
farel = os.path.relpath( os.path.join( basepath, 'seq', genome ), os.path.join( destination, 'cs' ) )
os.symlink( os.path.join( farel, target ) )
for line in location:
self._add_line( line[ 'file' ], line[ 'line' ] )
def _ex_tar( self, directory, filename ):
fh = tarfile.open( os.path.join( directory, filename ) )
fh.extractall( path=directory )
fh.close()
os.remove( os.path.join( directory, filename ) )
def _add_line( self, locfile, newline ):
filepath = locfile
origlines = []
output = []
comments = []
with open( filepath, 'r' ) as destfile:
for line in destfile:
origlines.append( line.strip() )
if newline not in origlines:
origlines.append( newline )
with open( filepath, 'w+' ) as destfile:
destfile.write( '\n'.join( origlines ) )
lib/galaxy/tools/genome_index/index_genome.py
+315
View File
@@ -0,0 +1,315 @@
#!/usr/bin/env python
"""
Export a history to an archive file using attribute files.
usage: %prog history_attrs dataset_attrs job_attrs out_file
-G, --gzip: gzip archive file
"""
import optparse, sys, os, tempfile, time, subprocess, shlex, json, tarfile, shutil
class ManagedIndexer():
def __init__( self, output_file, infile, workingdir ):
self.workingdir = os.path.abspath( workingdir )
self.outfile = open( os.path.abspath( output_file ), 'w' )
self.basedir = os.path.split( self.workingdir )[0]
self.fasta = os.path.abspath( infile )
self.locations = dict( nt=[], cs=[] )
self.log = []
self.indexers = {
'bwa': '_bwa',
'bowtie': '_bowtie',
'bowtie2': '_bowtie2',
'2bit': '_twobit',
'perm': '_perm',
'bfast': '_bfast',
'picard': '_picard',
'sam': '_sam'
}
if not os.path.exists( self.workingdir ):
os.makedirs( self.workingdir )
self.logfile = open( os.path.join( self.workingdir, 'ManagedIndexer.log' ), 'w+' )
def run_indexer( self, indexer ):
self.fapath = self.fasta
self.fafile = os.path.basename( self.fapath )
with WithChDir( self.basedir ):
if indexer not in self.indexers:
raise KeyError, 'The requested indexing function does not exist'
else:
with WithChDir( self.workingdir ):
self._log( 'Running indexer %s.' % indexer )
result = getattr( self, self.indexers[ indexer ] )()
if result is None:
self._log( 'Error running indexer %s.' % indexer )
self._flush_files()
raise Exception
else:
self._log( 'Indexer %s completed successfully.' % indexer )
self._flush_files()
def _flush_files( self ):
json.dump( self.locations, self.outfile )
self.outfile.close()
self.logfile.close()
def _log( self, stuff ):
timestamp = time.strftime('%Y-%m-%d %H:%M:%S %z')
self.logfile.write( "[%s] %s\n" % (timestamp, stuff) )
def _bwa( self ):
with WithChDir( self.workingdir ):
if not os.path.exists( self.fafile ):
os.symlink( os.path.relpath( self.fapath ), self.fafile )
command = shlex.split( 'bwa index -a bwtsw %s' % self.fafile )
result = subprocess.call( command, stderr=self.logfile, stdout=self.logfile )
if result != 0:
newcommand = shlex.split( 'bwa index -c %s' % self.fafile )
result = call( newcommand, stderr=self.logfile, stdout=self.logfile )
if result == 0:
self.locations[ 'nt' ].append( self.fafile )
os.remove( self.fafile )
os.makedirs( 'cs' )
with WithChDir( 'cs' ):
if not os.path.exists( self.fafile ):
os.symlink( os.path.relpath( self.fapath ), self.fafile )
command = shlex.split( 'bwa index -a bwtsw -c %s' % self.fafile )
result = subprocess.call( command, stderr=self.logfile, stdout=self.logfile )
if result != 0:
newcommand = shlex.split( 'bwa index -c %s' % self.fafile )
result = call( newcommand, stderr=self.logfile, stdout=self.logfile )
if result == 0:
self.locations[ 'cs' ].append( self.fafile )
os.remove( self.fafile )
else:
return False
else:
self.locations[ 'cs' ].append( self.fafile )
os.remove( self.fafile )
temptar = tarfile.open( 'cs.tar', 'w' )
temptar.add( 'cs' )
temptar.close()
shutil.rmtree( 'cs' )
return True
else:
return False
def _bowtie( self ):
ref_base = os.path.splitext(self.fafile)[0]
if not os.path.exists( self.fafile ):
os.symlink( os.path.relpath( self.fapath ), self.fafile )
command = shlex.split( 'bowtie-build -f %s %s' % ( self.fafile, ref_base ) )
result = subprocess.call( command, stderr=self.logfile, stdout=self.logfile )
if result == 0:
self.locations[ 'nt' ].append( ref_base )
os.remove( self.fafile )
indexdir = os.path.join( os.getcwd(), 'cs' )
os.makedirs( indexdir )
with WithChDir( indexdir ):
ref_base = os.path.splitext(self.fafile)[0]
if not os.path.exists( self.fafile ):
os.symlink( os.path.relpath( self.fapath ), self.fafile )
command = shlex.split( 'bowtie-build -C -f %s %s' % ( self.fafile, ref_base ) )
result = subprocess.call( command, stderr=self.logfile, stdout=self.logfile )
if result == 0:
self.locations[ 'cs' ].append( ref_base )
else:
return False
os.remove( os.path.join( indexdir, self.fafile ) )
temptar = tarfile.open( 'cs.tar', 'w' )
temptar.add( 'cs' )
temptar.close()
shutil.rmtree( 'cs' )
return True
else:
return False
def _bowtie2( self ):
ref_base = os.path.splitext(self.fafile)[0]
if not os.path.exists( self.fafile ):
os.symlink( os.path.relpath( self.fapath ), self.fafile )
command = shlex.split( 'bowtie2-build %s %s' % ( self.fafile, ref_base ) )
result = subprocess.call( command, stderr=self.logfile, stdout=self.logfile )
if result == 0:
self.locations[ 'nt' ].append( ref_base )
os.remove( self.fafile )
return True
else:
return False
def _twobit( self ):
"""Index reference files using 2bit for random access.
"""
ref_base = os.path.splitext(self.fafile)[0]
out_file = "%s.2bit" % ref_base
if not os.path.exists( self.fafile ):
os.symlink( os.path.relpath( self.fapath ), self.fafile )
command = shlex.split( 'faToTwoBit %s %s' % ( self.fafile, out_file ) )
result = subprocess.call( command, stderr=self.logfile, stdout=self.logfile )
if result == 0:
self.locations['nt'].append( out_file )
os.remove( self.fafile )
return True
else:
return False
def _perm( self ):
local_ref = self.fafile
if not os.path.exists( local_ref ):
os.symlink( os.path.relpath( self.fapath ), self.fafile )
genome = os.path.splitext( local_ref )[0]
read_length = 50
for seed in [ 'F3', 'F4' ]:
key = '%s_%s_%s' % (genome, seed, read_length)
desc = '%s: seed=%s, read length=%s' % (genome, seed, read_length)
index = "%s_base_%s_%s.index" % (genome, seed, read_length)
command = shlex.split("PerM %s %s --readFormat fastq --seed %s -m -s %s" % (local_ref, read_length, seed, index))
result = subprocess.call( command )
if result == 0:
self.locations[ 'nt' ].append( [ key, desc, index ] )
else:
return False
os.remove( local_ref )
os.makedirs( 'cs' )
with WithChDir( 'cs' ):
if not os.path.exists( local_ref ):
os.symlink( os.path.relpath( self.fapath ), self.fafile )
for seed in [ 'F3', 'F4' ]:
key = '%s_%s_%s' % (genome, seed, read_length)
desc = '%s: seed=%s, read length=%s' % (genome, seed, read_length)
index = "%s_color_%s_%s.index" % (genome, seed, read_length)
command = shlex.split("PerM %s %s --readFormat csfastq --seed %s -m -s %s" % (local_ref, read_length, seed, index))
result = subprocess.call( command, stderr=self.logfile, stdout=self.logfile )
if result == 0:
self.locations[ 'cs' ].append( [ key, desc, index ] )
else:
return False
os.remove( local_ref )
temptar = tarfile.open( 'cs.tar', 'w' )
temptar.add( 'cs' )
temptar.close()
shutil.rmtree( 'cs' )
return True
def _bfast( self ):
"""Indexes bfast in color and nucleotide space for longer reads.
This preps for 40+bp sized reads, which is bfast's strength.
"""
dir_name_nt = 'nt'
dir_name_cs = 'cs'
window_size = 14
bfast_nt_masks = [
"1111111111111111111111",
"1111101110111010100101011011111",
"1011110101101001011000011010001111111",
"10111001101001100100111101010001011111",
"11111011011101111011111111",
"111111100101001000101111101110111",
"11110101110010100010101101010111111",
"111101101011011001100000101101001011101",
"1111011010001000110101100101100110100111",
"1111010010110110101110010110111011",
]
bfast_color_masks = [
"1111111111111111111111",
"111110100111110011111111111",
"10111111011001100011111000111111",
"1111111100101111000001100011111011",
"111111110001111110011111111",
"11111011010011000011000110011111111",
"1111111111110011101111111",
"111011000011111111001111011111",
"1110110001011010011100101111101111",
"111111001000110001011100110001100011111",
]
local_ref = self.fafile
os.makedirs( dir_name_nt )
os.makedirs( dir_name_cs )
if not os.path.exists( self.fafile ):
os.symlink( os.path.relpath( self.fapath ), self.fafile )
with WithChDir( dir_name_nt ):
if not os.path.exists( self.fafile ):
os.symlink( os.path.relpath( self.fapath ), self.fafile )
# nucleotide space
command = shlex.split( "bfast fasta2brg -f %s -A 0" % local_ref )
result = subprocess.call( command, stderr=self.logfile )
for i, mask in enumerate( bfast_nt_masks ):
command = shlex.split("bfast index -d 1 -n 4 -f %s -A 0 -m %s -w %s -i %s" %
( local_ref, mask, window_size, i + 1 ) )
result = subprocess.call( command, stderr=self.logfile, stdout=self.logfile )
os.remove( self.fafile )
if result != 0:
return False
else:
os.remove( self.fafile )
with WithChDir( dir_name_cs ):
if not os.path.exists( self.fafile ):
os.symlink( os.path.relpath( self.fapath ), self.fafile )
# colorspace
command = shlex.split( "bfast fasta2brg -f %s -A 1" % local_ref )
result = subprocess.call( command, stderr=self.logfile, stdout=self.logfile )
for i, mask in enumerate( bfast_color_masks ):
command = shlex.split( "bfast index -d 1 -n 4 -f %s -A 1 -m %s -w %s -i %s" %
( local_ref, mask, window_size, i + 1 ) )
result = subprocess.call( command, stderr=self.logfile, stdout=self.logfile )
if result != 0:
return False
else:
os.remove( self.fafile )
self.locations = None
return True
def _picard( self ):
local_ref = self.fafile
srma = '/Users/dave/srma.jar'
genome = os.path.splitext( self.fafile )[0]
if not os.path.exists( self.fafile ):
os.symlink( os.path.relpath( self.fapath ), self.fafile )
command = shlex.split( 'samtools faidx %s' % self.fafile )
subprocess.call( command, stderr=self.logfile )
os.rename( '%s.fai' % self.fafile, '%s.fai' % genome )
command = shlex.split( "java -cp %s net.sf.picard.sam.CreateSequenceDictionary R=%s O=%s/%s.dict URI=%s" \
% ( srma, local_ref, os.curdir, genome, local_ref ) )
result = subprocess.call( command, stderr=self.logfile, stdout=self.logfile )
if result != 0:
return False
else:
self.locations[ 'nt' ].append( self.fafile )
#os.remove( '%s.fai' % genome )
os.remove( self.fafile )
return True
def _sam( self ):
local_ref = self.fafile
local_file = os.path.splitext( self.fafile )[ 0 ]
if not os.path.exists( local_ref ):
os.symlink( os.path.relpath( self.fapath ), self.fafile )
command = shlex.split("samtools faidx %s" % local_ref)
result = subprocess.call( command, stderr=self.logfile )
if result != 0:
return False
else:
self.locations[ 'nt' ].append( local_ref )
os.remove( local_ref )
return True
class WithChDir():
def __init__( self, target ):
self.working = target
self.previous = os.getcwd()
def __enter__( self ):
os.chdir( self.working )
def __exit__( self, *args ):
os.chdir( self.previous )
if __name__ == "__main__":
# Parse command line.
parser = optparse.OptionParser()
(options, args) = parser.parse_args()
indexer, infile, outfile, working_dir = args
# Create archive.
idxobj = ManagedIndexer( outfile, infile, working_dir )
idxobj.run_indexer( indexer )
lib/galaxy/tools/imp_exp/__init__.py
+10 -2
View File
@@ -1,5 +1,6 @@
import os, shutil, logging, tempfile, simplejson
from galaxy import model
from galaxy.tools.parameters.basic import UnvalidatedValue
from galaxy.web.framework.helpers import to_unicode
from galaxy.model.item_attrs import UsesAnnotations
from galaxy.util.json import *
@@ -324,6 +325,8 @@ class JobExportHistoryArchiveWrapper( object, UsesHistory, UsesAnnotations ):
"annotation" : to_unicode( getattr( obj, 'annotation', '' ) ),
"tags" : get_item_tag_dict( obj ),
}
if isinstance( obj, UnvalidatedValue ):
return obj.__str__()
return simplejson.JSONEncoder.default( self, obj )
#
@@ -415,8 +418,13 @@ class JobExportHistoryArchiveWrapper( object, UsesHistory, UsesAnnotations ):
params_dict[ name ] = value
job_attrs[ 'params' ] = params_dict
# Get input, output datasets.
input_datasets = [ assoc.dataset.hid for assoc in job.input_datasets ]
# -- Get input, output datasets. --
input_datasets = []
for assoc in job.input_datasets:
# Optional data inputs will not have a dataset.
if assoc.dataset:
input_datasets.append( assoc.dataset.hid )
job_attrs[ 'input_datasets' ] = input_datasets
output_datasets = [ assoc.dataset.hid for assoc in job.output_datasets ]
job_attrs[ 'output_datasets' ] = output_datasets
lib/galaxy/tools/parameters/basic.py
+74 -43
View File
@@ -1056,26 +1056,35 @@ class DrillDownSelectToolParameter( SelectToolParameter ):
... </param>
... ''' ) )
>>> print p.get_html()
<div><ul class="toolParameterExpandableCollapsable">
<li><span class="toolParameterExpandableCollapsable">[+]</span><input type="checkbox" name="some_name" value="heading1"">Heading 1
<ul class="toolParameterExpandableCollapsable" default_state="collapsed">
<li><input type="checkbox" name="some_name" value="option1"">Option 1
</li>
<li><input type="checkbox" name="some_name" value="option2"">Option 2
</li>
<li><span class="toolParameterExpandableCollapsable">[+]</span><input type="checkbox" name="some_name" value="heading1"">Heading 1
<ul class="toolParameterExpandableCollapsable" default_state="collapsed">
<li><input type="checkbox" name="some_name" value="option3"">Option 3
</li>
<li><input type="checkbox" name="some_name" value="option4"">Option 4
</li>
</ul>
</li>
</ul>
</li>
<li><input type="checkbox" name="some_name" value="option5"">Option 5
</li>
</ul></div>
<div class="form-row drilldown-container" id="drilldown--736f6d655f6e616d65">
<div class="form-row-input">
<span class="form-toggle icon-button toggle-expand" id="drilldown--736f6d655f6e616d65-68656164696e6731-click"></span>
<input type="checkbox" name="some_name" value="heading1" >Heading 1
<div class="form-row" id="drilldown--736f6d655f6e616d65-68656164696e6731-container" style="float: left; margin-left: 1em;">
<div class="form-row-input">
<input type="checkbox" name="some_name" value="option1" >Option 1
</div>
<div class="form-row-input">
<input type="checkbox" name="some_name" value="option2" >Option 2
</div>
<div class="form-row-input">
<span class="form-toggle icon-button toggle-expand" id="drilldown--736f6d655f6e616d65-68656164696e6731-68656164696e6731-click"></span>
<input type="checkbox" name="some_name" value="heading1" >Heading 1
<div class="form-row" id="drilldown--736f6d655f6e616d65-68656164696e6731-68656164696e6731-container" style="float: left; margin-left: 1em;">
<div class="form-row-input">
<input type="checkbox" name="some_name" value="option3" >Option 3
</div>
<div class="form-row-input">
<input type="checkbox" name="some_name" value="option4" >Option 4
</div>
</div>
</div>
</div>
</div>
<div class="form-row-input">
<input type="checkbox" name="some_name" value="option5" >Option 5
</div>
</div>
>>> p = DrillDownSelectToolParameter( None, XML(
... '''
... <param name="some_name" type="drill_down" display="radio" hierarchy="recurse" multiple="false">
@@ -1093,26 +1102,35 @@ class DrillDownSelectToolParameter( SelectToolParameter ):
... </param>
... ''' ) )
>>> print p.get_html()
<div><ul class="toolParameterExpandableCollapsable">
<li><span class="toolParameterExpandableCollapsable">[+]</span><input type="radio" name="some_name" value="heading1"">Heading 1
<ul class="toolParameterExpandableCollapsable" default_state="collapsed">
<li><input type="radio" name="some_name" value="option1"">Option 1
</li>
<li><input type="radio" name="some_name" value="option2"">Option 2
</li>
<li><span class="toolParameterExpandableCollapsable">[+]</span><input type="radio" name="some_name" value="heading1"">Heading 1
<ul class="toolParameterExpandableCollapsable" default_state="collapsed">
<li><input type="radio" name="some_name" value="option3"">Option 3
</li>
<li><input type="radio" name="some_name" value="option4"">Option 4
</li>
</ul>
</li>
</ul>
</li>
<li><input type="radio" name="some_name" value="option5"">Option 5
</li>
</ul></div>
<div class="form-row drilldown-container" id="drilldown--736f6d655f6e616d65">
<div class="form-row-input">
<span class="form-toggle icon-button toggle-expand" id="drilldown--736f6d655f6e616d65-68656164696e6731-click"></span>
<input type="radio" name="some_name" value="heading1" >Heading 1
<div class="form-row" id="drilldown--736f6d655f6e616d65-68656164696e6731-container" style="float: left; margin-left: 1em;">
<div class="form-row-input">
<input type="radio" name="some_name" value="option1" >Option 1
</div>
<div class="form-row-input">
<input type="radio" name="some_name" value="option2" >Option 2
</div>
<div class="form-row-input">
<span class="form-toggle icon-button toggle-expand" id="drilldown--736f6d655f6e616d65-68656164696e6731-68656164696e6731-click"></span>
<input type="radio" name="some_name" value="heading1" >Heading 1
<div class="form-row" id="drilldown--736f6d655f6e616d65-68656164696e6731-68656164696e6731-container" style="float: left; margin-left: 1em;">
<div class="form-row-input">
<input type="radio" name="some_name" value="option3" >Option 3
</div>
<div class="form-row-input">
<input type="radio" name="some_name" value="option4" >Option 4
</div>
</div>
</div>
</div>
</div>
<div class="form-row-input">
<input type="radio" name="some_name" value="option5" >Option 5
</div>
</div>
>>> print p.options
[{'selected': False, 'name': 'Heading 1', 'value': 'heading1', 'options': [{'selected': False, 'name': 'Option 1', 'value': 'option1', 'options': []}, {'selected': False, 'name': 'Option 2', 'value': 'option2', 'options': []}, {'selected': False, 'name': 'Heading 1', 'value': 'heading1', 'options': [{'selected': False, 'name': 'Option 3', 'value': 'option3', 'options': []}, {'selected': False, 'name': 'Option 4', 'value': 'option4', 'options': []}]}]}, {'selected': False, 'name': 'Option 5', 'value': 'option5', 'options': []}]
"""
@@ -1133,7 +1151,9 @@ class DrillDownSelectToolParameter( SelectToolParameter ):
from_file = os.path.join( tool.app.config.tool_data_path, from_file )
elem = XML( "<root>%s</root>" % open( from_file ).read() )
self.is_dynamic = False
self.dynamic_options = None #backwards compatibility with SelectToolParameter's old dynamic options and late validation
self.dynamic_options = elem.get( 'dynamic_options' , None )
if self.dynamic_options:
self.is_dynamic = True
self.options = []
self.filtered = {}
if elem.find( 'filter' ):
@@ -1148,12 +1168,23 @@ class DrillDownSelectToolParameter( SelectToolParameter ):
if filter.get( 'value' ) not in self.filtered[filter.get( 'data_ref' )][filter.get( 'meta_key' )]:
self.filtered[filter.get( 'data_ref' )][filter.get( 'meta_key' )][filter.get( 'value' )] = []
recurse_option_elems( self.filtered[filter.get( 'data_ref' )][filter.get( 'meta_key' )][filter.get( 'value' )], filter.find( 'options' ).findall( 'option' ) )
else:
elif not self.dynamic_options:
recurse_option_elems( self.options, elem.find( 'options' ).findall( 'option' ) )
def _get_options_from_code( self, trans=None, value=None, other_values=None ):
assert self.dynamic_options, Exception( "dynamic_options was not specifed" )
call_other_values = { '__trans__': trans, '__value__': value }
if other_values:
call_other_values.update( other_values.dict )
return eval( self.dynamic_options, self.tool.code_namespace, call_other_values )
def get_options( self, trans=None, value=None, other_values={} ):
if self.is_dynamic:
options = []
if self.dynamic_options:
options = self._get_options_from_code( trans=trans, value=value, other_values=other_values )
else:
options = []
for filter_key, filter_value in self.filtered.iteritems():
dataset = other_values[filter_key]
if dataset.__class__.__name__.endswith( "DatasetFilenameWrapper" ): #this is a bad way to check for this, but problems importing class ( due to circular imports? )
lib/galaxy/util/inflection.py
+2 -2
View File
@@ -14,7 +14,7 @@ class Base:
def cond_plural(self, number_of_records, word) :
'''Returns the plural form of a word if first parameter is greater than 1'''
if number_of_records > 1 :
if number_of_records != 1:
return self.pluralize(word)
else :
return word
@@ -385,4 +385,4 @@ class Inflector:
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THIS SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THIS SOFTWARE.
# THIS SOFTWARE.
lib/galaxy/util/odict.py
+37 -40
View File
@@ -11,23 +11,22 @@ class odict(UserDict):
This dictionary class extends UserDict to record the order in which items are
added. Calling keys(), values(), items(), etc. will return results in this
order.
I've added iterkeys, itervalues, iteritems
"""
def __init__(self, dict = None):
def __init__( self, dict = None ):
self._keys = []
UserDict.__init__(self, dict)
UserDict.__init__( self, dict )
def __delitem__(self, key):
UserDict.__delitem__(self, key)
self._keys.remove(key)
def __delitem__( self, key ):
UserDict.__delitem__( self, key )
self._keys.remove( key )
def __setitem__(self, key, item):
UserDict.__setitem__(self, key, item)
if key not in self._keys: self._keys.append(key)
def __setitem__( self, key, item ):
UserDict.__setitem__( self, key, item )
if key not in self._keys:
self._keys.append( key )
def clear(self):
UserDict.clear(self)
def clear( self ):
UserDict.clear( self )
self._keys = []
def copy(self):
@@ -35,49 +34,43 @@ class odict(UserDict):
new.update( self )
return new
def items(self):
return zip(self._keys, self.values())
def items( self ):
return zip( self._keys, self.values() )
def keys(self):
def keys( self ):
return self._keys[:]
def popitem(self):
def popitem( self ):
try:
key = self._keys[-1]
except IndexError:
raise KeyError('dictionary is empty')
raise KeyError( 'dictionary is empty' )
val = self[ key ]
del self[ key ]
return ( key, val )
val = self[key]
del self[key]
def setdefault( self, key, failobj=None ):
if key not in self._keys:
self._keys.append( key )
return UserDict.setdefault( self, key, failobj )
return (key, val)
def update( self, dict ):
for ( key, val ) in dict.items():
self.__setitem__( key, val )
def setdefault(self, key, failobj = None):
if key not in self._keys: self._keys.append(key)
return UserDict.setdefault(self, key, failobj)
def values( self ):
return map( self.get, self._keys )
def update(self, dict):
UserDict.update(self, dict)
for key in dict.keys():
if key not in self._keys: self._keys.append(key)
def update(self, dict):
for (key,val) in dict.items():
self.__setitem__(key,val)
def values(self):
return map(self.get, self._keys)
def iterkeys(self):
def iterkeys( self ):
return iter( self._keys )
def itervalues(self):
def itervalues( self ):
for key in self._keys:
yield self.get(key)
yield self.get( key )
def iteritems(self):
def iteritems( self ):
for key in self._keys:
yield key, self.get(key)
yield key, self.get( key )
def __iter__( self ):
for key in self._keys:
@@ -86,3 +79,7 @@ class odict(UserDict):
def reverse( self ):
self._keys.reverse()
def insert( self, index, key, item ):
if key not in self._keys:
self._keys.insert( index, key )
UserDict.__setitem__( self, key, item )
lib/galaxy/util/shed_util.py
+246 -96
View File
@@ -1,18 +1,107 @@
import os, tempfile, shutil, subprocess, logging
import sys, os, tempfile, shutil, subprocess, logging, string, urllib2
from datetime import date, datetime, timedelta
from time import strftime
from time import strftime, gmtime
from galaxy import util
from galaxy.datatypes.checkers import *
from galaxy.util.json import *
from galaxy.tools.search import ToolBoxSearch
from galaxy.model.orm import *
from galaxy import eggs
import pkg_resources
pkg_resources.require( 'mercurial' )
from mercurial import ui, commands
pkg_resources.require( 'elementtree' )
from elementtree import ElementTree, ElementInclude
from elementtree.ElementTree import Element, SubElement
log = logging.getLogger( __name__ )
# Characters that must be html escaped
MAPPED_CHARS = { '>' :'&gt;',
'<' :'&lt;',
'"' : '&quot;',
'&' : '&amp;',
'\'' : '&apos;' }
VALID_CHARS = set( string.letters + string.digits + "'\"-=_.()/+*^,:?!#[]%\\$@;{}" )
class ShedCounter( object ):
def __init__( self, model ):
self.model = model
self.generation_time = strftime( "%b %d, %Y", gmtime() )
self.repositories = 0
self.new_repositories = 0
self.deleted_repositories = 0
self.invalid_tools = 0
self.valid_tools = 0
self.workflows = 0
self.proprietary_datatypes = 0
self.total_clones = 0
self.generate_statistics()
@property
def sa_session( self ):
"""Returns a SQLAlchemy session"""
return self.model.context
def generate_statistics( self ):
self.repositories = 0
self.new_repositories = 0
self.deleted_repositories = 0
self.invalid_tools = 0
self.valid_tools = 0
self.workflows = 0
self.proprietary_datatypes = 0
self.total_clones = 0
for repository in self.sa_session.query( self.model.Repository ):
self.repositories += 1
self.total_clones += repository.times_downloaded
is_deleted = repository.deleted
is_new = repository.is_new
if is_deleted and is_new:
self.deleted_repositories += 1
self.new_repositories += 1
elif is_deleted:
self.deleted_repositories += 1
elif is_new:
self.new_repositories += 1
else:
processed_guids = []
processed_invalid_tool_configs = []
processed_relative_workflow_paths = []
processed_datatypes = []
for downloadable_revision in repository.downloadable_revisions:
metadata = downloadable_revision.metadata
if 'tools' in metadata:
tool_dicts = metadata[ 'tools' ]
for tool_dict in tool_dicts:
if 'guid' in tool_dict:
guid = tool_dict[ 'guid' ]
if guid not in processed_guids:
self.valid_tools += 1
processed_guids.append( guid )
if 'invalid_tools' in metadata:
invalid_tool_configs = metadata[ 'invalid_tools' ]
for invalid_tool_config in invalid_tool_configs:
if invalid_tool_config not in processed_invalid_tool_configs:
self.invalid_tools += 1
processed_invalid_tool_configs.append( invalid_tool_config )
if 'datatypes' in metadata:
datatypes = metadata[ 'datatypes' ]
for datatypes_dict in datatypes:
if 'extension' in datatypes_dict:
extension = datatypes_dict[ 'extension' ]
if extension not in processed_datatypes:
self.proprietary_datatypes += 1
processed_datatypes.append( extension )
if 'workflows' in metadata:
workflows = metadata[ 'workflows' ]
for workflow_tup in workflows:
relative_path, exported_workflow_dict = workflow_tup
if relative_path not in processed_relative_workflow_paths:
self.workflows += 1
processed_relative_workflow_paths.append( relative_path )
self.generation_time = strftime( "%b %d, %Y", gmtime() )
def add_to_shed_tool_config( app, shed_tool_conf_dict, elem_list ):
# A tool shed repository is being installed so change the shed_tool_conf file. Parse the config file to generate the entire list
# of config_elems instead of using the in-memory list since it will be a subset of the entire list if one or more repositories have
@@ -129,13 +218,15 @@ def alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_
# The value of proprietary_path must be an absolute path due to job_working_directory.
elem.attrib[ 'proprietary_path' ] = os.path.abspath( relative_head )
elem.attrib[ 'proprietary_datatype_module' ] = proprietary_datatype_module
sniffers = datatypes_config_root.find( 'sniffers' )
else:
sniffers = None
fd, proprietary_datatypes_config = tempfile.mkstemp()
os.write( fd, '<?xml version="1.0"?>\n' )
os.write( fd, '<datatypes>\n' )
os.write( fd, '%s' % util.xml_to_string( registration ) )
os.write( fd, '%s' % util.xml_to_string( sniffers ) )
if sniffers:
os.write( fd, '%s' % util.xml_to_string( sniffers ) )
os.write( fd, '</datatypes>\n' )
os.close( fd )
os.chmod( proprietary_datatypes_config, 0644 )
@@ -179,31 +270,38 @@ def clean_tool_shed_url( tool_shed_url ):
# Eliminate the port, if any, since it will result in an invalid directory name.
return tool_shed_url.split( ':' )[ 0 ]
return tool_shed_url.rstrip( '/' )
def clone_repository( name, clone_dir, current_working_dir, repository_clone_url ):
log.debug( "Installing repository '%s'" % name )
if not os.path.exists( clone_dir ):
os.makedirs( clone_dir )
log.debug( 'Cloning %s' % repository_clone_url )
cmd = 'hg clone %s' % repository_clone_url
tmp_name = tempfile.NamedTemporaryFile().name
tmp_stderr = open( tmp_name, 'wb' )
os.chdir( clone_dir )
proc = subprocess.Popen( args=cmd, shell=True, stderr=tmp_stderr.fileno() )
returncode = proc.wait()
os.chdir( current_working_dir )
tmp_stderr.close()
return returncode, tmp_name
def copy_sample_loc_file( app, filename ):
"""Copy xxx.loc.sample to ~/tool-data/xxx.loc.sample and ~/tool-data/xxx.loc"""
head, sample_loc_file = os.path.split( filename )
loc_file = sample_loc_file.replace( '.sample', '' )
tool_data_path = os.path.abspath( app.config.tool_data_path )
def clone_repository( repository_clone_url, repository_file_dir, ctx_rev ):
"""
Clone the repository up to the specified changeset_revision. No subsequent revisions will be present in the cloned repository.
"""
commands.clone( get_configured_ui(),
repository_clone_url,
dest=repository_file_dir,
pull=True,
noupdate=False,
rev=[ ctx_rev ] )
def copy_sample_file( app, filename, dest_path=None ):
"""
Copy xxx.loc.sample to dest_path/xxx.loc.sample and dest_path/xxx.loc. The default value for dest_path is ~/tool-data.
"""
if dest_path is None:
dest_path = os.path.abspath( app.config.tool_data_path )
sample_file_path, sample_file_name = os.path.split( filename )
copied_file = sample_file_name.replace( '.sample', '' )
# It's ok to overwrite the .sample version of the file.
shutil.copy( os.path.abspath( filename ), os.path.join( tool_data_path, sample_loc_file ) )
# Only create the .loc file if it does not yet exist. We don't
# overwrite it in case it contains stuff proprietary to the local instance.
if not os.path.exists( os.path.join( tool_data_path, loc_file ) ):
shutil.copy( os.path.abspath( filename ), os.path.join( tool_data_path, loc_file ) )
shutil.copy( os.path.abspath( filename ), os.path.join( dest_path, sample_file_name ) )
# Only create the .loc file if it does not yet exist. We don't overwrite it in case it contains stuff proprietary to the local instance.
if not os.path.exists( os.path.join( dest_path, copied_file ) ):
shutil.copy( os.path.abspath( filename ), os.path.join( dest_path, copied_file ) )
def copy_sample_files( app, sample_files, sample_files_copied=None, dest_path=None ):
"""
Copy all files to dest_path in the local Galaxy environment that have not already been copied. Those that have been copied
are contained in sample_files_copied. The default value for dest_path is ~/tool-data.
"""
sample_files_copied = util.listify( sample_files_copied )
for filename in sample_files:
if filename not in sample_files_copied:
copy_sample_file( app, filename, dest_path=dest_path )
def create_repository_dict_for_proprietary_datatypes( tool_shed, name, owner, installed_changeset_revision, tool_dicts, converter_path=None, display_path=None ):
return dict( tool_shed=tool_shed,
repository_name=name,
@@ -212,7 +310,8 @@ def create_repository_dict_for_proprietary_datatypes( tool_shed, name, owner, in
tool_dicts=tool_dicts,
converter_path=converter_path,
display_path=display_path )
def create_or_update_tool_shed_repository( app, name, description, changeset_revision, repository_clone_url, metadata_dict, owner='', dist_to_shed=False ):
def create_or_update_tool_shed_repository( app, name, description, changeset_revision, ctx_rev, repository_clone_url, metadata_dict,
owner='', dist_to_shed=False ):
# The received value for dist_to_shed will be True if the InstallManager is installing a repository that contains tools or datatypes that used
# to be in the Galaxy distribution, but have been moved to the main Galaxy tool shed.
sa_session = app.model.context.current
@@ -225,6 +324,7 @@ def create_or_update_tool_shed_repository( app, name, description, changeset_rev
if tool_shed_repository:
tool_shed_repository.description = description
tool_shed_repository.changeset_revision = changeset_revision
tool_shed_repository.ctx_rev = ctx_rev
tool_shed_repository.metadata = metadata_dict
tool_shed_repository.includes_datatypes = includes_datatypes
tool_shed_repository.deleted = False
@@ -236,6 +336,7 @@ def create_or_update_tool_shed_repository( app, name, description, changeset_rev
owner=owner,
installed_changeset_revision=changeset_revision,
changeset_revision=changeset_revision,
ctx_rev=ctx_rev,
metadata=metadata_dict,
includes_datatypes=includes_datatypes,
dist_to_shed=dist_to_shed )
@@ -247,7 +348,7 @@ def generate_clone_url( trans, repository ):
tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository )
return '%s/repos/%s/%s' % ( tool_shed_url, repository.owner, repository.name )
def generate_datatypes_metadata( datatypes_config, metadata_dict ):
"""Update the received metadata_dict with changes that have been applied to the received datatypes_config."""
"""Update the received metadata_dict with information from the parsed datatypes_config."""
tree = ElementTree.parse( datatypes_config )
root = tree.getroot()
ElementInclude.include( root )
@@ -325,12 +426,21 @@ def generate_metadata( toolbox, relative_install_dir, repository_clone_url ):
if not ( check_binary( full_path ) or check_image( full_path ) or check_gzip( full_path )[ 0 ]
or check_bz2( full_path )[ 0 ] or check_zip( full_path ) ):
try:
tool = toolbox.load_tool( full_path )
# Make sure we're looking at a tool config and not a display application config or something else.
element_tree = util.parse_xml( full_path )
element_tree_root = element_tree.getroot()
is_tool = element_tree_root.tag == 'tool'
except Exception, e:
tool = None
if tool is not None:
tool_config = os.path.join( root, name )
metadata_dict = generate_tool_metadata( tool_config, tool, repository_clone_url, metadata_dict )
log.debug( "Error parsing %s, exception: %s" % ( full_path, str( e ) ) )
is_tool = False
if is_tool:
try:
tool = toolbox.load_tool( full_path )
except Exception, e:
tool = None
if tool is not None:
tool_config = os.path.join( root, name )
metadata_dict = generate_tool_metadata( tool_config, tool, repository_clone_url, metadata_dict )
# Find all exported workflows
elif name.endswith( '.ga' ):
relative_path = os.path.join( root, name )
@@ -356,30 +466,30 @@ def generate_tool_metadata( tool_config, tool, repository_clone_url, metadata_di
# Handle tool.requirements.
tool_requirements = []
for tr in tool.requirements:
name=tr.name
type=tr.type
if type == 'fabfile':
version = None
fabfile = tr.fabfile
method = tr.method
else:
version = tr.version
fabfile = None
method = None
requirement_dict = dict( name=name,
type=type,
version=version,
fabfile=fabfile,
method=method )
requirement_dict = dict( name=tr.name,
type=tr.type,
version=tr.version )
tool_requirements.append( requirement_dict )
# Handle tool.tests.
tool_tests = []
if tool.tests:
for ttb in tool.tests:
required_files = []
for required_file in ttb.required_files:
value, extra = required_file
required_files.append( ( value ) )
inputs = []
for input in ttb.inputs:
name, value, extra = input
inputs.append( ( name, value ) )
outputs = []
for output in ttb.outputs:
name, file_name, extra = output
outputs.append( ( name, os.path.split( file_name )[ 1 ] ) )
test_dict = dict( name=ttb.name,
required_files=ttb.required_files,
inputs=ttb.inputs,
outputs=ttb.outputs )
required_files=required_files,
inputs=inputs,
outputs=outputs )
tool_tests.append( test_dict )
tool_dict = dict( id=tool.id,
guid=guid,
@@ -570,6 +680,15 @@ def generate_workflow_metadata( relative_path, exported_workflow_dict, metadata_
else:
metadata_dict[ 'workflows' ] = [ ( relative_path, exported_workflow_dict ) ]
return metadata_dict
def get_configured_ui():
# Configure any desired ui settings.
_ui = ui.ui()
# The following will suppress all messages. This is
# the same as adding the following setting to the repo
# hgrc file' [ui] section:
# quiet = True
_ui.setconfig( 'ui', 'quiet', True )
return _ui
def get_converter_and_display_paths( registration_elem, relative_install_dir ):
"""Find the relative path to data type converters and display applications included in installed tool shed repositories."""
converter_path = None
@@ -612,6 +731,12 @@ def get_converter_and_display_paths( registration_elem, relative_install_dir ):
if converter_path and display_path:
break
return converter_path, display_path
def get_ctx_rev( tool_shed_url, name, owner, changeset_revision ):
url = '%s/repository/get_ctx_rev?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy&no_reset=true' % ( tool_shed_url, name, owner, changeset_revision )
response = urllib2.urlopen( url )
ctx_rev = response.read()
response.close()
return ctx_rev
def get_shed_tool_conf_dict( app, shed_tool_conf ):
"""
Return the in-memory version of the shed_tool_conf file, which is stored in the config_elems entry
@@ -768,27 +893,27 @@ def handle_missing_data_table_entry( app, tool_path, sample_files, repository_to
return repository_tools_tups
def handle_missing_index_file( app, tool_path, sample_files, repository_tools_tups ):
"""Inspect each tool to see if it has any input parameters that are dynamically generated select lists that depend on a .loc file."""
missing_files_handled = []
sample_files_copied = []
for index, repository_tools_tup in enumerate( repository_tools_tups ):
tup_path, guid, repository_tool = repository_tools_tup
params_with_missing_index_file = repository_tool.params_with_missing_index_file
for param in params_with_missing_index_file:
options = param.options
missing_head, missing_tail = os.path.split( options.missing_index_file )
if missing_tail not in missing_files_handled:
missing_file_path, missing_file_name = os.path.split( options.missing_index_file )
if missing_file_name not in sample_files_copied:
# The repository must contain the required xxx.loc.sample file.
for sample_file in sample_files:
sample_head, sample_tail = os.path.split( sample_file )
if sample_tail == '%s.sample' % missing_tail:
copy_sample_loc_file( app, sample_file )
sample_file_path, sample_file_name = os.path.split( sample_file )
if sample_file_name == '%s.sample' % missing_file_name:
copy_sample_file( app, sample_file )
if options.tool_data_table and options.tool_data_table.missing_index_file:
options.tool_data_table.handle_found_index_file( options.missing_index_file )
missing_files_handled.append( missing_tail )
sample_files_copied.append( options.missing_index_file )
break
# Reload the tool into the local list of repository_tools_tups.
repository_tool = app.toolbox.load_tool( os.path.join( tool_path, tup_path ), guid=guid )
repository_tools_tups[ index ] = ( tup_path, guid, repository_tool )
return repository_tools_tups
return repository_tools_tups, sample_files_copied
def handle_sample_tool_data_table_conf_file( app, filename ):
"""
Parse the incoming filename and add new entries to the in-memory
@@ -912,8 +1037,9 @@ def load_datatype_items( app, repository, relative_install_dir, deactivate=False
if display_path:
# Load or deactivate proprietary datatype display applications
app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict, deactivate=deactivate )
def load_repository_contents( trans, repository_name, description, owner, changeset_revision, tool_path, repository_clone_url,
relative_install_dir, current_working_dir, tmp_name, tool_shed=None, tool_section=None, shed_tool_conf=None ):
def load_repository_contents( trans, repository_name, description, owner, changeset_revision, ctx_rev, tool_path, repository_clone_url,
relative_install_dir, current_working_dir, tool_shed=None, tool_section=None, shed_tool_conf=None,
install_tool_dependencies=False ):
"""Generate the metadata for the installed tool shed repository, among other things."""
# It is critical that the installed repository is updated to the desired changeset_revision before metadata is set because the
# process for setting metadata uses the repository files on disk. This method is called when an admin is installing a new repository
@@ -926,6 +1052,7 @@ def load_repository_contents( trans, repository_name, description, owner, change
repository_name,
description,
changeset_revision,
ctx_rev,
repository_clone_url,
metadata_dict,
dist_to_shed=False )
@@ -937,8 +1064,9 @@ def load_repository_contents( trans, repository_name, description, owner, change
# Handle missing data table entries for tool parameters that are dynamically generated select lists.
repository_tools_tups = handle_missing_data_table_entry( trans.app, tool_path, sample_files, repository_tools_tups )
# Handle missing index files for tool parameters that are dynamically generated select lists.
repository_tools_tups = handle_missing_index_file( trans.app, tool_path, sample_files, repository_tools_tups )
# Handle tools that use fabric scripts to install dependencies.
repository_tools_tups, sample_files_copied = handle_missing_index_file( trans.app, tool_path, sample_files, repository_tools_tups )
# Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance.
copy_sample_files( trans.app, sample_files, sample_files_copied=sample_files_copied )
handle_tool_dependencies( current_working_dir, relative_install_dir, repository_tools_tups )
add_to_tool_panel( app=trans.app,
repository_name=repository_name,
@@ -949,11 +1077,6 @@ def load_repository_contents( trans, repository_name, description, owner, change
shed_tool_conf=shed_tool_conf,
tool_panel_dict=tool_panel_dict,
new_install=True )
# Remove the temporary file
try:
os.unlink( tmp_name )
except:
pass
if 'datatypes_config' in metadata_dict:
datatypes_config = os.path.abspath( metadata_dict[ 'datatypes_config' ] )
# Load data types required by tools.
@@ -987,18 +1110,12 @@ def panel_entry_per_tool( tool_section_dict ):
if k not in [ 'id', 'version', 'name' ]:
return True
return False
def pull_repository( current_working_dir, repo_files_dir, name ):
# Pull the latest possible contents to the repository.
log.debug( "Pulling latest updates to the repository named '%s'" % name )
cmd = 'hg pull'
tmp_name = tempfile.NamedTemporaryFile().name
tmp_stderr = open( tmp_name, 'wb' )
os.chdir( repo_files_dir )
proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() )
returncode = proc.wait()
os.chdir( current_working_dir )
tmp_stderr.close()
return returncode, tmp_name
def pull_repository( repo, repository_clone_url, ctx_rev ):
"""Pull changes from a remote repository to a local one."""
commands.pull( get_configured_ui(),
repo,
source=repository_clone_url,
rev=ctx_rev )
def remove_from_shed_tool_config( trans, shed_tool_conf_dict, guids_to_remove ):
# A tool shed repository is being uninstalled so change the shed_tool_conf file. Parse the config file to generate the entire list
# of config_elems instead of using the in-memory list since it will be a subset of the entire list if one or more repositories have
@@ -1096,17 +1213,50 @@ def remove_from_tool_panel( trans, repository, shed_tool_conf, uninstall ):
if uninstall:
# Write the current in-memory version of the integrated_tool_panel.xml file to disk.
trans.app.toolbox.write_integrated_tool_panel_config_file()
def update_repository( current_working_dir, repo_files_dir, changeset_revision ):
# Update the cloned repository to changeset_revision. It is imperative that the
# installed repository is updated to the desired changeset_revision before metadata
# is set because the process for setting metadata uses the repository files on disk.
log.debug( 'Updating cloned repository to revision "%s"' % changeset_revision )
cmd = 'hg update -r %s' % changeset_revision
tmp_name = tempfile.NamedTemporaryFile().name
tmp_stderr = open( tmp_name, 'wb' )
os.chdir( repo_files_dir )
proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() )
returncode = proc.wait()
os.chdir( current_working_dir )
tmp_stderr.close()
return returncode, tmp_name
def to_html_escaped( text ):
"""Translates the characters in text to html values"""
translated = []
for c in text:
if c in [ '\r\n', '\n', ' ', '\t' ] or c in VALID_CHARS:
translated.append( c )
elif c in MAPPED_CHARS:
translated.append( MAPPED_CHARS[ c ] )
else:
translated.append( 'X' )
return ''.join( translated )
def to_html_str( text ):
"""Translates the characters in text to sn html string"""
translated = []
for c in text:
if c in VALID_CHARS:
translated.append( c )
elif c in MAPPED_CHARS:
translated.append( MAPPED_CHARS[ c ] )
elif c == ' ':
translated.append( '&nbsp;' )
elif c == '\t':
translated.append( '&nbsp;&nbsp;&nbsp;&nbsp;' )
elif c == '\n':
translated.append( '<br/>' )
elif c not in [ '\r' ]:
translated.append( 'X' )
return ''.join( translated )
def update_repository( repo, ctx_rev=None ):
"""
Update the cloned repository to changeset_revision. It is critical that the installed repository is updated to the desired
changeset_revision before metadata is set because the process for setting metadata uses the repository files on disk.
"""
# TODO: We may have files on disk in the repo directory that aren't being tracked, so they must be removed.
# The codes used to show the status of files are as follows.
# M = modified
# A = added
# R = removed
# C = clean
# ! = deleted, but still tracked
# ? = not tracked
# I = ignored
# It would be nice if we could use mercurial's purge extension to remove untracked files. The problem is that
# purging is not supported by the mercurial API. See the deprecated update_for_browsing() method in common.py.
commands.update( get_configured_ui(),
repo,
rev=ctx_rev )
lib/galaxy/visualization/tracks/data_providers.py
+147 -25
View File
@@ -19,7 +19,7 @@ from galaxy.util.lrucache import LRUCache
from galaxy.visualization.tracks.summary import *
import galaxy_utils.sequence.vcf
from galaxy.datatypes.tabular import Vcf
from galaxy.datatypes.interval import Bed, Gff, Gtf, ENCODEPeak
from galaxy.datatypes.interval import Interval, Bed, Gff, Gtf, ENCODEPeak
from pysam import csamtools, ctabix
@@ -67,8 +67,7 @@ class TracksDataProvider( object ):
"""
Write data in region defined by chrom, start, and end to a file.
"""
# Override.
pass
raise Exception( "Unimplemented Function" )
def valid_chroms( self ):
"""
@@ -81,23 +80,19 @@ class TracksDataProvider( object ):
Returns true if dataset has data in the specified genome window, false
otherwise.
"""
# Override.
pass
raise Exception( "Unimplemented Function" )
def get_iterator( self, chrom, start, end ):
"""
Returns an iterator that provides data in the region chrom:start-end
"""
# Override.
pass
raise Exception( "Unimplemented Function" )
def process_data( self, iterator, start_val=0, max_vals=None, **kwargs ):
"""
Process data from an iterator to a format that can be provided to client.
"""
# Override.
pass
raise Exception( "Unimplemented Function" )
def get_data( self, chrom, start, end, start_val=0, max_vals=sys.maxint, **kwargs ):
"""
@@ -270,18 +265,96 @@ class TabixDataProvider( FilterableMixin, TracksDataProvider ):
out.close()
#
# -- BED data providers --
# -- Interval data providers --
#
class BedDataProvider( TracksDataProvider ):
class IntervalDataProvider( TracksDataProvider ):
"""
Abstract class that processes BED data from native format to payload format.
Processes BED data from native format to payload format.
Payload format: [ uid (offset), start, end, name, strand, thick_start, thick_end, blocks ]
"""
def get_iterator( self, chrom, start, end ):
raise "Unimplemented Method"
raise Exception( "Unimplemented Function" )
def process_data( self, iterator, start_val=0, max_vals=None, **kwargs ):
"""
Provides
"""
# Build data to return. Payload format is:
# [ <guid/offset>, <start>, <end>, <name>, <strand> ]
#
# First three entries are mandatory, others are optional.
#
filter_cols = from_json_string( kwargs.get( "filter_cols", "[]" ) )
no_detail = ( "no_detail" in kwargs )
rval = []
message = None
# Subtract one b/c columns are 1-based but indices are 0-based.
col_fn = lambda col: None if col is None else col - 1
start_col = self.original_dataset.metadata.startCol - 1
end_col = self.original_dataset.metadata.endCol - 1
strand_col = col_fn( self.original_dataset.metadata.strandCol )
name_col = col_fn( self.original_dataset.metadata.nameCol )
for count, line in enumerate( iterator ):
if count < start_val:
continue
if max_vals and count-start_val >= max_vals:
message = ERROR_MAX_VALS % ( max_vals, "features" )
break
feature = line.split()
length = len(feature)
# Unique id is just a hash of the line
payload = [ hash(line), int( feature[start_col] ), int( feature [end_col] ) ]
if no_detail:
rval.append( payload )
continue
# Name, strand.
if name_col:
payload.append( feature[name_col] )
if strand_col:
# Put empty name as placeholder.
if not name_col: payload.append( "" )
payload.append( feature[strand_col] )
# Score (filter data)
if length >= 5 and filter_cols and filter_cols[0] == "Score":
try:
payload.append( float( feature[4] ) )
except:
payload.append( feature[4] )
rval.append( payload )
return { 'data': rval, 'message': message }
def write_data_to_file( self, chrom, start, end, filename ):
raise Exception( "Unimplemented Function" )
class IntervalTabixDataProvider( TabixDataProvider, IntervalDataProvider ):
"""
Provides data from a BED file indexed via tabix.
"""
pass
#
# -- BED data providers --
#
class BedDataProvider( TracksDataProvider ):
"""
Processes BED data from native format to payload format.
Payload format: [ uid (offset), start, end, name, strand, thick_start, thick_end, blocks ]
"""
def get_iterator( self, chrom, start, end ):
raise Exception( "Unimplemented Method" )
def process_data( self, iterator, start_val=0, max_vals=None, **kwargs ):
"""
@@ -359,7 +432,7 @@ class BedTabixDataProvider( TabixDataProvider, BedDataProvider ):
Provides data from a BED file indexed via tabix.
"""
pass
class RawBedDataProvider( BedDataProvider ):
"""
Provide data from BED file.
@@ -936,9 +1009,9 @@ class IntervalIndexDataProvider( FilterableMixin, TracksDataProvider ):
return { 'data': results, 'message': message }
class GFFDataProvider( TracksDataProvider ):
class RawGFFDataProvider( TracksDataProvider ):
"""
Provide data from GFF file.
Provide data from GFF file that has not been indexed.
NOTE: this data provider does not use indices, and hence will be very slow
for large datasets.
@@ -951,7 +1024,7 @@ class GFFDataProvider( TracksDataProvider ):
"""
start, end = int( start ), int( end )
source = open( self.original_dataset.file_name )
def features_in_region_iter():
offset = 0
for feature in GFFReaderWrapper( source, fix_strand=True ):
@@ -961,7 +1034,7 @@ class GFFDataProvider( TracksDataProvider ):
yield feature, offset
offset += feature.raw_size
return features_in_region_iter()
def process_data( self, iterator, start_val=0, max_vals=None, **kwargs ):
"""
Process data from an iterator to a format that can be provided to client.
@@ -984,6 +1057,48 @@ class GFFDataProvider( TracksDataProvider ):
return { 'data': results, 'message': message }
class GtfTabixDataProvider( TabixDataProvider ):
"""
Returns data from GTF datasets that are indexed via tabix.
"""
def process_data( self, iterator, start_val=0, max_vals=None, **kwargs ):
# Loop through lines and group by transcript_id; each group is a feature.
# TODO: extend this code or use code in gff_util to process GFF/3 as well
# and then create a generic GFFDataProvider that can be used with both
# raw and tabix datasets.
features = {}
for count, line in enumerate( iterator ):
line_attrs = parse_gff_attributes( line.split('\t')[8] )
transcript_id = line_attrs[ 'transcript_id' ]
if transcript_id in features:
feature = features[ transcript_id ]
else:
feature = []
features[ transcript_id ] = feature
feature.append( GFFInterval( None, line.split( '\t') ) )
# Process data.
filter_cols = from_json_string( kwargs.get( "filter_cols", "[]" ) )
no_detail = ( "no_detail" in kwargs )
results = []
message = None
for count, intervals in enumerate( features.values() ):
if count < start_val:
continue
if count-start_val >= max_vals:
message = ERROR_MAX_VALS % ( max_vals, "reads" )
break
feature = GFFFeature( None, intervals=intervals )
payload = package_gff_feature( feature, no_detail=no_detail, filter_cols=filter_cols )
payload.insert( 0, feature.intervals[ 0 ].attributes[ 'transcript_id' ] )
results.append( payload )
return { 'data': results, 'message': message }
#
# -- ENCODE Peak data providers.
@@ -1113,7 +1228,13 @@ class ENCODEPeakTabixDataProvider( TabixDataProvider, ENCODEPeakDataProvider ):
# type. First key is converted dataset type; if result is another dict, second key
# is original dataset type. TODO: This needs to be more flexible.
dataset_type_name_to_data_provider = {
"tabix": { Vcf: VcfTabixDataProvider, Bed: BedTabixDataProvider, ENCODEPeak: ENCODEPeakTabixDataProvider, "default" : TabixDataProvider },
"tabix": {
Vcf: VcfTabixDataProvider,
Bed: BedTabixDataProvider,
Gtf: GtfTabixDataProvider,
ENCODEPeak: ENCODEPeakTabixDataProvider,
Interval: IntervalTabixDataProvider,
"default" : TabixDataProvider },
"interval_index": IntervalIndexDataProvider,
"bai": BamDataProvider,
"bam": SamDataProvider,
@@ -1172,16 +1293,17 @@ def package_gff_feature( feature, no_detail=False, filter_cols=[] ):
feature.end
]
# HACK: remove interval with name 'transcript' from feature.
# HACK: ignore interval with name 'transcript' from feature.
# Cufflinks puts this interval in each of its transcripts,
# and they mess up trackster by covering the feature's blocks.
# This interval will always be a feature's first interval,
# and the GFF's third column is its feature name.
# and the GFF's third column is its feature name.
feature_intervals = feature.intervals
if feature.intervals[0].fields[2] == 'transcript':
feature.intervals = feature.intervals[1:]
feature_intervals = feature.intervals[1:]
# Add blocks.
block_sizes = [ (interval.end - interval.start ) for interval in feature.intervals ]
block_starts = [ ( interval.start - feature.start ) for interval in feature.intervals ]
block_sizes = [ (interval.end - interval.start ) for interval in feature_intervals ]
block_starts = [ ( interval.start - feature.start ) for interval in feature_intervals ]
blocks = zip( block_sizes, block_starts )
payload.append( [ ( feature.start + block[1], feature.start + block[1] + block[0] ) for block in blocks ] )
lib/galaxy/web/api/datasets.py
+44
View File
@@ -0,0 +1,44 @@
"""
API operations on the contents of a dataset.
"""
import logging, os, string, shutil, urllib, re, socket
from cgi import escape, FieldStorage
from galaxy import util, datatypes, jobs, web, util
from galaxy.web.base.controller import *
from galaxy.util.sanitize_html import sanitize_html
from galaxy.model.orm import *
log = logging.getLogger( __name__ )
class DatasetsController( BaseAPIController, UsesHistoryDatasetAssociation ):
@web.expose_api
def index( self, trans, hda_id, **kwd ):
"""
GET /api/datasets
Lists datasets.
"""
pass
@web.expose_api
def show( self, trans, id, deleted='False', **kwd ):
"""
GET /api/datasets/{encoded_dataset_id}
Displays information about and/or content of a dataset.
"""
# Get HDA.
try:
hda = self.get_dataset( trans, id, check_ownership=True, check_accessible=True )
except Exception, e:
return str( e )
# Return information about HDA.
rval = None
try:
rval = hda.get_api_value()
except Exception, e:
rval = "Error in dataset API at listing contents"
log.error( rval + ": %s" % str(e) )
trans.response.status = 500
return rval
lib/galaxy/web/api/histories.py
+20 -23
View File
@@ -20,28 +20,27 @@ class HistoriesController( BaseAPIController, UsesHistory ):
GET /api/histories
GET /api/histories/deleted
Displays a collection (list) of histories.
"""
"""
rval = []
deleted = util.string_as_bool( deleted )
try:
query = trans.sa_session.query( trans.app.model.History ).filter_by( user=trans.user, deleted=deleted ).order_by(
desc(trans.app.model.History.table.c.update_time)).all()
except Exception, e:
rval = "Error in history API"
log.error( rval + ": %s" % str(e) )
trans.response.status = 500
if not rval:
try:
if trans.user:
query = trans.sa_session.query(trans.app.model.History ).filter_by( user=trans.user, deleted=deleted ).order_by(
desc(trans.app.model.History.table.c.update_time)).all()
for history in query:
item = history.get_api_value(value_mapper={'id':trans.security.encode_id})
item['url'] = url_for( 'history', id=trans.security.encode_id( history.id ) )
rval.append( item )
except Exception, e:
rval = "Error in history API at constructing return list"
log.error( rval + ": %s" % str(e) )
trans.response.status = 500
elif trans.galaxy_session.current_history:
#No user, this must be session authentication with an anonymous user.
history = trans.galaxy_session.current_history
item = history.get_api_value(value_mapper={'id':trans.security.encode_id})
item['url'] = url_for( 'history', id=trans.security.encode_id( history.id ) )
rval.append(item)
except Exception, e:
rval = "Error in history API"
log.error( rval + ": %s" % str(e) )
trans.response.status = 500
return rval
@web.expose_api
@@ -54,7 +53,7 @@ class HistoriesController( BaseAPIController, UsesHistory ):
history_id = id
params = util.Params( kwd )
deleted = util.string_as_bool( deleted )
def traverse( datasets ):
rval = {}
states = trans.app.model.Dataset.states
@@ -65,15 +64,13 @@ class HistoriesController( BaseAPIController, UsesHistory ):
if not item['deleted']:
rval[item['state']] = rval[item['state']] + 1
return rval
try:
history = self.get_history( trans, history_id, check_ownership=True, check_accessible=True, deleted=deleted )
except Exception, e:
return str( e )
try:
item = history.get_api_value(view='element', value_mapper={'id':trans.security.encode_id})
num_sets = len( [hda.id for hda in history.datasets if not hda.deleted] )
num_sets = len( [hda.id for hda in history.datasets if not hda.deleted] )
states = trans.app.model.Dataset.states
state = states.ERROR
if num_sets == 0:
@@ -87,7 +84,7 @@ class HistoriesController( BaseAPIController, UsesHistory ):
elif summary[states.QUEUED] > 0:
state = states.QUEUED
elif summary[states.OK] == num_sets:
state = states.OK
state = states.OK
item['contents_url'] = url_for( 'history_contents', history_id=history_id )
item['state_details'] = summary
item['state'] = state
@@ -108,7 +105,7 @@ class HistoriesController( BaseAPIController, UsesHistory ):
if payload.get( 'name', None ):
hist_name = util.restore_text( payload['name'] )
new_history = trans.app.model.History( user=trans.user, name=hist_name )
trans.sa_session.add( new_history )
trans.sa_session.flush()
item = new_history.get_api_value(view='element', value_mapper={'id':trans.security.encode_id})
@@ -124,8 +121,8 @@ class HistoriesController( BaseAPIController, UsesHistory ):
# a request body is optional here
purge = False
if kwd.get( 'payload', None ):
purge = util.string_as_bool( kwd['payload'].get( 'purge', False ) )
purge = util.string_as_bool( kwd['payload'].get( 'purge', False ) )
try:
history = self.get_history( trans, history_id, check_ownership=True, check_accessible=False, deleted=True )
except Exception, e:
lib/galaxy/web/api/history_contents.py
+6 -9
View File
@@ -1,11 +1,9 @@
"""
API operations on the contents of a history.
"""
import logging, os, string, shutil, urllib, re, socket
from cgi import escape, FieldStorage
from galaxy import util, datatypes, jobs, web, util
import logging
from galaxy import web
from galaxy.web.base.controller import *
from galaxy.util.sanitize_html import sanitize_html
from galaxy.model.orm import *
import pkg_resources
@@ -21,12 +19,11 @@ class HistoryContentsController( BaseAPIController, UsesHistoryDatasetAssociatio
"""
GET /api/histories/{encoded_history_id}/contents
Displays a collection (list) of history contents
"""
"""
try:
history = self.get_history( trans, history_id, check_ownership=True, check_accessible=True )
except Exception, e:
return str( e )
rval = []
try:
for dataset in history.datasets:
@@ -57,6 +54,8 @@ class HistoryContentsController( BaseAPIController, UsesHistoryDatasetAssociatio
return str( e )
try:
item = content.get_api_value( view='element' )
if trans.user_is_admin() or trans.app.config.expose_dataset_path:
item['file_name'] = content.file_name
if not item['deleted']:
# Problem: Method url_for cannot use the dataset controller
# Get the environment from DefaultWebTransaction and use default webapp mapper instead of webapp API mapper
@@ -67,7 +66,7 @@ class HistoryContentsController( BaseAPIController, UsesHistoryDatasetAssociatio
item = self.encode_all_ids( trans, item )
except Exception, e:
item = "Error in history API at listing dataset"
log.error( item + ": %s" % str(e) )
log.error( item + ": %s" % str(e) )
trans.response.status = 500
return item
@@ -77,7 +76,6 @@ class HistoryContentsController( BaseAPIController, UsesHistoryDatasetAssociatio
POST /api/libraries/{encoded_history_id}/contents
Creates a new history content item (file, aka HistoryDatasetAssociation).
"""
params = util.Params( payload )
from_ld_id = payload.get( 'from_ld_id', None )
try:
@@ -95,7 +93,6 @@ class HistoryContentsController( BaseAPIController, UsesHistoryDatasetAssociatio
except Exception, e:
return str( e )
hda = ld.library_dataset_dataset_association.to_history_dataset_association( history, add_to_history=True )
history.add_dataset( hda )
trans.sa_session.flush()
return hda.get_api_value()
else:
lib/galaxy/web/api/libraries.py
+39 -12
View File
@@ -7,18 +7,27 @@ from galaxy import util, datatypes, jobs, web, util
from galaxy.web.base.controller import *
from galaxy.util.sanitize_html import sanitize_html
from galaxy.model.orm import *
from paste.httpexceptions import *
log = logging.getLogger( __name__ )
class LibrariesController( BaseAPIController ):
@web.expose_api
def index( self, trans, **kwd ):
def index( self, trans, deleted='False', **kwd ):
"""
GET /api/libraries
GET /api/libraries/deleted
Displays a collection (list) of libraries.
"""
query = trans.sa_session.query( trans.app.model.Library ).filter( trans.app.model.Library.table.c.deleted == False )
query = trans.sa_session.query( trans.app.model.Library )
deleted = util.string_as_bool( deleted )
if deleted:
route = 'deleted_library'
query = query.filter( trans.app.model.Library.table.c.deleted == True )
else:
route = 'library'
query = query.filter( trans.app.model.Library.table.c.deleted == False )
current_user_role_ids = [ role.id for role in trans.get_current_user_roles() ]
library_access_action = trans.app.security_agent.permitted_actions.LIBRARY_ACCESS.action
restricted_library_ids = [ lp.library_id for lp in trans.sa_session.query( trans.model.LibraryPermissions ) \
@@ -32,31 +41,32 @@ class LibrariesController( BaseAPIController ):
rval = []
for library in query:
item = library.get_api_value()
item['url'] = url_for( 'library', id=trans.security.encode_id( library.id ) )
item['url'] = url_for( route, id=trans.security.encode_id( library.id ) )
item['id'] = trans.security.encode_id( item['id'] )
rval.append( item )
return rval
@web.expose_api
def show( self, trans, id, **kwd ):
def show( self, trans, id, deleted='False', **kwd ):
"""
GET /api/libraries/{encoded_library_id}
GET /api/libraries/deleted/{encoded_library_id}
Displays information about a library.
"""
library_id = id
deleted = util.string_as_bool( deleted )
params = util.Params( kwd )
try:
decoded_library_id = trans.security.decode_id( library_id )
except TypeError:
trans.response.status = 400
return "Malformed library id ( %s ) specified, unable to decode." % str( library_id )
raise HTTPBadRequest( detail='Malformed library id ( %s ) specified, unable to decode.' % id )
try:
library = trans.sa_session.query( trans.app.model.Library ).get( decoded_library_id )
assert library.deleted == deleted
except:
library = None
if not library or not ( trans.user_is_admin() or trans.app.security_agent.can_access_library( trans.get_current_user_roles(), library ) ):
trans.response.status = 400
return "Invalid library id ( %s ) specified." % str( library_id )
raise HTTPBadRequest( detail='Invalid library id ( %s ) specified.' % id )
item = library.get_api_value( view='element' )
#item['contents_url'] = url_for( 'contents', library_id=library_id )
item['contents_url'] = url_for( 'library_contents', library_id=library_id )
@@ -69,13 +79,11 @@ class LibrariesController( BaseAPIController ):
Creates a new library.
"""
if not trans.user_is_admin():
trans.response.status = 403
return "You are not authorized to create a new library."
raise HTTPForbidden( detail='You are not authorized to create a new library.' )
params = util.Params( payload )
name = util.restore_text( params.get( 'name', None ) )
if not name:
trans.response.status = 400
return "Missing required parameter 'name'."
raise HTTPBadRequest( detail="Missing required parameter 'name'." )
description = util.restore_text( params.get( 'description', '' ) )
synopsis = util.restore_text( params.get( 'synopsis', '' ) )
if synopsis in [ 'None', None ]:
@@ -91,3 +99,22 @@ class LibrariesController( BaseAPIController ):
rval['name'] = name
rval['id'] = encoded_id
return [ rval ]
@web.expose_api
def delete( self, trans, id, **kwd ):
if not trans.user_is_admin():
raise HTTPForbidden( detail='You are not authorized to delete libraries.' )
try:
decoded_id = trans.security.decode_id( id )
except TypeError:
raise HTTPBadRequest( detail='Malformed library id ( %s ) specified, unable to decode.' % id )
try:
library = trans.sa_session.query( trans.app.model.Library ).get( decoded_id )
except:
library = None
if not library:
raise HTTPBadRequest( detail='Invalid library id ( %s ) specified.' % id )
library.deleted = True
trans.sa_session.add( library )
trans.sa_session.flush()
return library.get_api_value( view='element', value_mapper={ 'id' : trans.security.encode_id } )
lib/galaxy/web/api/requests.py
+5 -7
View File
@@ -1,19 +1,17 @@
"""
API operations on a sample tracking system.
"""
import logging, os, string, shutil, urllib, re, socket
from cgi import escape, FieldStorage
from galaxy import util, datatypes, jobs, web, util
import logging
from galaxy import util, web
from galaxy.web.base.controller import *
from galaxy.util.sanitize_html import sanitize_html
from galaxy.model.orm import *
from galaxy.util.bunch import Bunch
log = logging.getLogger( __name__ )
class RequestsAPIController( BaseAPIController ):
update_types = Bunch( REQUEST = 'request_state' )
update_type_values = [v[1] for v in update_types.items()]
_update_types = Bunch( REQUEST = 'request_state' )
_update_type_values = [v[1] for v in _update_types.items()]
@web.expose_api
def index( self, trans, **kwd ):
"""
@@ -77,7 +75,7 @@ class RequestsAPIController( BaseAPIController ):
return "Missing required 'update_type' parameter. Please consult the API documentation for help."
else:
update_type = payload.pop( 'update_type' )
if update_type not in self.update_type_values:
if update_type not in self._update_type_values:
trans.response.status = 400
return "Invalid value for 'update_type' parameter ( %s ) specified. Please consult the API documentation for help." % update_type
try:
lib/galaxy/web/api/tools.py
+75
View File
@@ -0,0 +1,75 @@
from galaxy import config, tools, web, util
from galaxy.web.base.controller import BaseController, BaseAPIController
from galaxy.util.bunch import Bunch
messages = Bunch(
NO_TOOL = "no tool"
)
class ToolsController( BaseAPIController ):
"""
RESTful controller for interactions with tools.
"""
@web.expose_api
def index( self, trans, **kwds ):
"""
GET /api/tools: returns a list of tools defined by parameters
parameters:
in_panel - if true, tools are returned in panel structure,
including sections and labels
trackster - if true, only tools that are compatible with
Trackster are returned
"""
# Read params.
in_panel = util.string_as_bool( kwds.get( 'in_panel', 'True' ) )
trackster = util.string_as_bool( kwds.get( 'trackster', 'False' ) )
# Create return value.
return self.app.toolbox.to_dict( trans, in_panel=in_panel, trackster=trackster )
@web.json
def show( self, trans, id, **kwd ):
"""
GET /api/tools/{tool_id}
Returns tool information, including parameters and inputs.
"""
return self.app.toolbox.tools_by_id[ id ].to_dict( trans, for_display=True )
@web.expose_api
def create( self, trans, payload, **kwd ):
"""
POST /api/tools
Executes tool using specified inputs, creating new history-dataset
associations, which are returned.
"""
# TODO: set target history?
# -- Execute tool. --
# Get tool.
tool_id = payload[ 'id' ]
tool = trans.app.toolbox.get_tool( tool_id )
if not tool:
return { "message": { "type": "error", "text" : messages.NO_TOOL } }
# Set up inputs.
inputs = payload[ 'inputs' ]
# HACK: add run button so that tool.handle_input will run tool.
inputs['runtool_btn'] = 'Execute'
# TODO: encode data ids and decode ids.
params = util.Params( inputs, sanitize = False )
template, vars = tool.handle_input( trans, params.__dict__ )
# TODO: check for errors and ensure that output dataset(s) are available.
output_datasets = vars[ 'out_data' ].values()
rval = {
"outputs": []
}
outputs = rval[ "outputs" ]
for output in output_datasets:
outputs.append( output.get_api_value() )
return rval
lib/galaxy/web/api/workflows.py
+9 -5
View File
@@ -18,9 +18,9 @@ class WorkflowsAPIController(BaseAPIController):
def index(self, trans, **kwd):
"""
GET /api/workflows
Displays a collection of workflows
Displays a collection of workflows.
"""
# List parameters of a specific workflow
rval = []
for wf in trans.sa_session.query(trans.app.model.StoredWorkflow).filter_by(
user=trans.user, deleted=False).order_by(
@@ -38,10 +38,12 @@ class WorkflowsAPIController(BaseAPIController):
item['url'] = url_for('workflow', id=encoded_id)
rval.append(item)
return rval
@web.expose_api
def show(self, trans, id, **kwd):
"""
GET /api/workflows/{encoded_workflow_id}
Displays information needed to run a workflow from the command line.
"""
workflow_id = id
@@ -72,28 +74,29 @@ class WorkflowsAPIController(BaseAPIController):
# p = step.get_required_parameters()
item['inputs'] = inputs
return item
@web.expose_api
def create(self, trans, payload, **kwd):
"""
POST /api/workflows
We're not creating workflows from the api. Just execute for now.
However, we will import them if installed_repository_file is specified
"""
if 'workflow_id' not in payload:
# create new
if 'installed_repository_file' in payload:
workflow_controller = trans.webapp.controllers[ 'workflow' ]
result = workflow_controller.import_workflow( trans=trans,
result = workflow_controller.import_workflow( trans=trans,
cntrller='api',
**payload)
return result
trans.response.status = 403
return "Either workflow_id or installed_repository_file must be specified"
if 'installed_repository_file' in payload:
trans.response.status = 403
return "installed_repository_file may not be specified with workflow_id"
stored_workflow = trans.sa_session.query(self.app.model.StoredWorkflow).get(
trans.security.decode_id(payload['workflow_id']))
if stored_workflow.user != trans.user and not trans.user_is_admin():
@@ -215,3 +218,4 @@ class WorkflowsAPIController(BaseAPIController):
trans.sa_session.add( workflow_invocation )
trans.sa_session.flush()
return rval
lib/galaxy/web/base/controller.py
+4 -5
View File
@@ -322,7 +322,7 @@ class UsesVisualization( SharableItemSecurity ):
if hda_ldda == "hda":
dataset = self.get_dataset( trans, dataset_id, check_ownership=False, check_accessible=True )
else:
dataset = trans.sa_session.query( trans.app.model.LibraryDatasetDatasetAssociation ).get( trans.security.decode_id(dataset_id) )
dataset = trans.sa_session.query( trans.app.model.LibraryDatasetDatasetAssociation ).get( dataset_id )
try:
prefs = track_dict['prefs']
@@ -1325,8 +1325,7 @@ class Admin( object ):
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
if webapp == 'galaxy':
cloned_repositories = trans.sa_session.query( trans.model.ToolShedRepository ) \
.first()
cloned_repositories = trans.sa_session.query( trans.model.ToolShedRepository ).first()
return trans.fill_template( '/webapps/galaxy/admin/index.mako',
webapp=webapp,
cloned_repositories=cloned_repositories,
@@ -2381,8 +2380,8 @@ class Admin( object ):
deleted = []
msg = None
status = None
if not trans.app.config.get_bool( "enable_job_running", True ):
return trans.show_error_message( 'This Galaxy instance is not configured to run jobs. If using multiple servers, please directly access the job running instance to manage jobs.' )
if not self.app.config.job_manager != self.app.config.server_name:
return trans.show_error_message( 'This Galaxy instance is not the job manager. If using multiple servers, please directly access the job manager instance to manage jobs.' )
job_ids = util.listify( stop )
if job_ids and stop_msg in [ None, '' ]:
msg = 'Please enter an error message to display to the user describing why the job was terminated'
lib/galaxy/web/buildapp.py
+32 -30
View File
@@ -103,36 +103,38 @@ def app_factory( global_conf, **kwargs ):
webapp.add_route( '/u/:username/h/:slug', controller='history', action='display_by_username_and_slug' )
webapp.add_route( '/u/:username/w/:slug', controller='workflow', action='display_by_username_and_slug' )
webapp.add_route( '/u/:username/v/:slug', controller='visualization', action='display_by_username_and_slug' )
# If enabled, add the web API
if asbool( kwargs.get( 'enable_api', False ) ):
add_api_controllers( webapp, app )
webapp.api_mapper.resource( 'content',
'contents',
controller='library_contents',
name_prefix='library_',
path_prefix='/api/libraries/:library_id',
parent_resources=dict( member_name='library', collection_name='libraries' ) )
webapp.api_mapper.resource( 'content',
'contents',
controller='history_contents',
name_prefix='history_',
path_prefix='/api/histories/:history_id',
parent_resources=dict( member_name='history', collection_name='histories' ) )
webapp.api_mapper.resource( 'permission',
'permissions',
path_prefix='/api/libraries/:library_id',
parent_resources=dict( member_name='library', collection_name='libraries' ) )
webapp.api_mapper.resource( 'library', 'libraries', path_prefix='/api' )
webapp.api_mapper.resource( 'sample', 'samples', path_prefix='/api' )
webapp.api_mapper.resource( 'request', 'requests', path_prefix='/api' )
webapp.api_mapper.resource( 'form', 'forms', path_prefix='/api' )
webapp.api_mapper.resource( 'request_type', 'request_types', path_prefix='/api' )
webapp.api_mapper.resource( 'role', 'roles', path_prefix='/api' )
webapp.api_mapper.resource_with_deleted( 'quota', 'quotas', path_prefix='/api' )
webapp.api_mapper.resource_with_deleted( 'user', 'users', path_prefix='/api' )
webapp.api_mapper.resource( 'workflow', 'workflows', path_prefix='/api' )
webapp.api_mapper.resource_with_deleted( 'history', 'histories', path_prefix='/api' )
#webapp.api_mapper.connect( 'run_workflow', '/api/workflow/{workflow_id}/library/{library_id}', controller='workflows', action='run', workflow_id=None, library_id=None, conditions=dict(method=["GET"]) )
# Add the web API
add_api_controllers( webapp, app )
webapp.api_mapper.resource( 'content',
'contents',
controller='library_contents',
name_prefix='library_',
path_prefix='/api/libraries/:library_id',
parent_resources=dict( member_name='library', collection_name='libraries' ) )
webapp.api_mapper.resource( 'content',
'contents',
controller='history_contents',
name_prefix='history_',
path_prefix='/api/histories/:history_id',
parent_resources=dict( member_name='history', collection_name='histories' ) )
webapp.api_mapper.resource( 'permission',
'permissions',
path_prefix='/api/libraries/:library_id',
parent_resources=dict( member_name='library', collection_name='libraries' ) )
webapp.api_mapper.resource( 'dataset', 'datasets', path_prefix='/api' )
webapp.api_mapper.resource_with_deleted( 'library', 'libraries', path_prefix='/api' )
webapp.api_mapper.resource( 'sample', 'samples', path_prefix='/api' )
webapp.api_mapper.resource( 'request', 'requests', path_prefix='/api' )
webapp.api_mapper.resource( 'form', 'forms', path_prefix='/api' )
webapp.api_mapper.resource( 'request_type', 'request_types', path_prefix='/api' )
webapp.api_mapper.resource( 'role', 'roles', path_prefix='/api' )
webapp.api_mapper.resource_with_deleted( 'quota', 'quotas', path_prefix='/api' )
webapp.api_mapper.resource( 'tool', 'tools', path_prefix='/api' )
webapp.api_mapper.resource_with_deleted( 'user', 'users', path_prefix='/api' )
webapp.api_mapper.resource( 'workflow', 'workflows', path_prefix='/api' )
webapp.api_mapper.resource_with_deleted( 'history', 'histories', path_prefix='/api' )
#webapp.api_mapper.connect( 'run_workflow', '/api/workflow/{workflow_id}/library/{library_id}', controller='workflows', action='run', workflow_id=None, library_id=None, conditions=dict(method=["GET"]) )
webapp.finalize_config()
# Wrap the webapp in some useful middleware
lib/galaxy/web/controllers/admin_toolshed.py
+159 -149
View File
@@ -2,10 +2,15 @@ import urllib2
from galaxy.web.controllers.admin import *
from galaxy.util.json import from_json_string, to_json_string
from galaxy.util.shed_util import *
from galaxy import tools
from galaxy import eggs, tools
eggs.require( 'mercurial' )
from mercurial import hg
log = logging.getLogger( __name__ )
MAX_CONTENT_SIZE = 32768
class RepositoryListGrid( grids.Grid ):
class NameColumn( grids.TextColumn ):
def get_value( self, trans, grid, tool_shed_repository ):
@@ -144,7 +149,7 @@ class AdminToolshed( AdminGalaxy ):
def browse_tool_shed( self, trans, **kwd ):
tool_shed_url = kwd[ 'tool_shed_url' ]
galaxy_url = url_for( '/', qualified=True )
url = '%s/repository/browse_valid_repositories?galaxy_url=%s&webapp=galaxy' % ( tool_shed_url, galaxy_url )
url = '%srepository/browse_valid_repositories?galaxy_url=%s&webapp=galaxy&no_reset=true' % ( tool_shed_url, galaxy_url )
return trans.response.send_redirect( url )
@web.expose
@web.require_admin
@@ -162,7 +167,7 @@ class AdminToolshed( AdminGalaxy ):
# Send a request to the relevant tool shed to see if there are any updates.
repository = get_repository( trans, kwd[ 'id' ] )
tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository )
url = '%s/repository/check_for_updates?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \
url = '%s/repository/check_for_updates?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s&webapp=galaxy&no_reset=true' % \
( tool_shed_url, url_for( '/', qualified=True ), repository.name, repository.owner, repository.changeset_revision )
return trans.response.send_redirect( url )
@web.expose
@@ -213,14 +218,14 @@ class AdminToolshed( AdminGalaxy ):
def find_tools_in_tool_shed( self, trans, **kwd ):
tool_shed_url = kwd[ 'tool_shed_url' ]
galaxy_url = url_for( '/', qualified=True )
url = '%s/repository/find_tools?galaxy_url=%s&webapp=galaxy' % ( tool_shed_url, galaxy_url )
url = '%srepository/find_tools?galaxy_url=%s&webapp=galaxy&no_reset=true' % ( tool_shed_url, galaxy_url )
return trans.response.send_redirect( url )
@web.expose
@web.require_admin
def find_workflows_in_tool_shed( self, trans, **kwd ):
tool_shed_url = kwd[ 'tool_shed_url' ]
galaxy_url = url_for( '/', qualified=True )
url = '%s/repository/find_workflows?galaxy_url=%s&webapp=galaxy' % ( tool_shed_url, galaxy_url )
url = '%srepository/find_workflows?galaxy_url=%s&webapp=galaxy&no_reset=true' % ( tool_shed_url, galaxy_url )
return trans.response.send_redirect( url )
@web.expose
@web.require_admin
@@ -288,58 +293,45 @@ class AdminToolshed( AdminGalaxy ):
current_working_dir = os.getcwd()
installed_repository_names = []
for name, repo_info_tuple in repo_info_dict.items():
description, repository_clone_url, changeset_revision = repo_info_tuple
description, repository_clone_url, changeset_revision, ctx_rev = repo_info_tuple
clone_dir = os.path.join( tool_path, self.__generate_tool_path( repository_clone_url, changeset_revision ) )
relative_install_dir = os.path.join( clone_dir, name )
if os.path.exists( clone_dir ):
# Repository and revision has already been cloned.
message += 'Revision <b>%s</b> of repository <b>%s</b> was previously installed.<br/>' % ( changeset_revision, name )
else:
returncode, tmp_name = clone_repository( name, clone_dir, current_working_dir, repository_clone_url )
if returncode == 0:
returncode, tmp_name = update_repository( current_working_dir, relative_install_dir, changeset_revision )
if returncode == 0:
owner = get_repository_owner( clean_repository_clone_url( repository_clone_url ) )
tool_shed = clean_tool_shed_url( tool_shed_url )
tool_shed_repository, metadata_dict = load_repository_contents( trans,
repository_name=name,
description=description,
owner=owner,
changeset_revision=changeset_revision,
tool_path=tool_path,
repository_clone_url=repository_clone_url,
relative_install_dir=relative_install_dir,
current_working_dir=current_working_dir,
tmp_name=tmp_name,
tool_shed=tool_shed,
tool_section=tool_section,
shed_tool_conf=shed_tool_conf )
if 'tools' in metadata_dict:
# Get the tool_versions from the tool shed for each tool in the installed change set.
url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \
( tool_shed_url, name, owner, changeset_revision )
response = urllib2.urlopen( url )
text = response.read()
response.close()
if text:
tool_version_dicts = from_json_string( text )
handle_tool_versions( trans.app, tool_version_dicts, tool_shed_repository )
else:
message += "Version information for the tools included in the <b>%s</b> repository is missing. " % name
message += "Reset all of this repository's metadata in the tool shed, then set the installed tool versions "
message += "from the installed repository's <b>Repository Actions</b> menu. "
status = 'error'
installed_repository_names.append( name )
clone_repository( repository_clone_url, os.path.abspath( relative_install_dir ), ctx_rev )
owner = get_repository_owner( clean_repository_clone_url( repository_clone_url ) )
tool_shed = clean_tool_shed_url( tool_shed_url )
tool_shed_repository, metadata_dict = load_repository_contents( trans,
repository_name=name,
description=description,
owner=owner,
changeset_revision=changeset_revision,
ctx_rev=ctx_rev,
tool_path=tool_path,
repository_clone_url=repository_clone_url,
relative_install_dir=relative_install_dir,
current_working_dir=current_working_dir,
tool_shed=tool_shed,
tool_section=tool_section,
shed_tool_conf=shed_tool_conf )
if 'tools' in metadata_dict:
# Get the tool_versions from the tool shed for each tool in the installed change set.
url = '%srepository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy&no_reset=true' % \
( tool_shed_url, name, owner, changeset_revision )
response = urllib2.urlopen( url )
text = response.read()
response.close()
if text:
tool_version_dicts = from_json_string( text )
handle_tool_versions( trans.app, tool_version_dicts, tool_shed_repository )
else:
tmp_stderr = open( tmp_name, 'rb' )
message += '%s<br/>' % tmp_stderr.read()
tmp_stderr.close()
message += "Version information for the tools included in the <b>%s</b> repository is missing. " % name
message += "Reset all of this repository's metadata in the tool shed, then set the installed tool versions "
message += "from the installed repository's <b>Repository Actions</b> menu. "
status = 'error'
else:
tmp_stderr = open( tmp_name, 'rb' )
message += '%s<br/>' % tmp_stderr.read()
tmp_stderr.close()
status = 'error'
installed_repository_names.append( name )
if installed_repository_names:
installed_repository_names.sort()
num_repositories_installed = len( installed_repository_names )
@@ -367,6 +359,29 @@ class AdminToolshed( AdminGalaxy ):
shed_tool_conf = shed_tool_conf.replace( './', '', 1 )
shed_tool_conf_select_field = None
tool_panel_section_select_field = build_tool_panel_section_select_field( trans )
if includes_tools:
# If we're installing a single repository that contains a readme file, get it's contents to display.
decoded_repo_info_dict = tool_shed_decode( repo_info_dict )
if len( decoded_repo_info_dict ) == 1:
name = decoded_repo_info_dict.keys()[ 0 ]
repo_info_tuple = decoded_repo_info_dict[ name ]
description, repository_clone_url, changeset_revision, ctx_rev = repo_info_tuple
owner = get_repository_owner( clean_repository_clone_url( repository_clone_url ) )
url = '%srepository/get_readme?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy&no_reset=true' % ( tool_shed_url, name, owner, changeset_revision )
response = urllib2.urlopen( url )
raw_text = response.read()
response.close()
readme_text = ''
for i, line in enumerate( raw_text ):
readme_text = '%s%s' % ( readme_text, to_html_str( line ) )
if len( readme_text ) > MAX_CONTENT_SIZE:
large_str = '\nFile contents truncated because file size is larger than maximum viewing size of %s\n' % util.nice_size( MAX_CONTENT_SIZE )
readme_text = '%s%s' % ( readme_text, to_html_str( large_str ) )
break
else:
readme_text = ''
else:
readme_text = ''
return trans.fill_template( '/admin/tool_shed_repository/select_tool_panel_section.mako',
tool_shed_url=tool_shed_url,
repo_info_dict=repo_info_dict,
@@ -375,6 +390,7 @@ class AdminToolshed( AdminGalaxy ):
shed_tool_conf_select_field=shed_tool_conf_select_field,
tool_panel_section_select_field=tool_panel_section_select_field,
new_tool_panel_section=new_tool_panel_section,
readme_text=readme_text,
message=message,
status=status )
@web.expose
@@ -418,79 +434,82 @@ class AdminToolshed( AdminGalaxy ):
repository_clone_url = generate_clone_url( trans, repository )
clone_dir = os.path.join( tool_path, self.__generate_tool_path( repository_clone_url, repository.installed_changeset_revision ) )
relative_install_dir = os.path.join( clone_dir, repository.name )
returncode, tmp_name = clone_repository( repository.name, clone_dir, current_working_dir, repository_clone_url )
if returncode == 0:
returncode, tmp_name = update_repository( current_working_dir, relative_install_dir, repository.installed_changeset_revision )
if returncode == 0:
if repository.includes_tools:
# Get the location in the tool panel in which each tool was originally loaded.
metadata = repository.metadata
if 'tool_panel_section' in metadata:
tool_panel_dict = metadata[ 'tool_panel_section' ]
if not tool_panel_dict:
tool_panel_dict = generate_tool_panel_dict_for_new_install( metadata[ 'tools' ] )
tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository )
if not repository.ctx_rev:
# The ctx_rev column was introduced late, so may be null for some installed ToolShedRepositories.
ctx_rev = get_ctx_rev( tool_shed_url, repository.name, repository.owner, repository.installed_changeset_revision )
else:
ctx_rev = repository.ctx_rev
clone_repository( repository_clone_url, os.path.abspath( relative_install_dir ), ctx_rev )
if repository.includes_tools:
# Get the location in the tool panel in which each tool was originally loaded.
metadata = repository.metadata
if 'tool_panel_section' in metadata:
tool_panel_dict = metadata[ 'tool_panel_section' ]
if not tool_panel_dict:
tool_panel_dict = generate_tool_panel_dict_for_new_install( metadata[ 'tools' ] )
else:
tool_panel_dict = generate_tool_panel_dict_for_new_install( metadata[ 'tools' ] )
# TODO: Fix this to handle the case where the tools are distributed across in more than 1 ToolSection. The
# following assumes everything was loaded into 1 section (or no section) in the tool panel.
tool_section_dicts = tool_panel_dict[ tool_panel_dict.keys()[ 0 ] ]
tool_section_dict = tool_section_dicts[ 0 ]
original_section_id = tool_section_dict[ 'id' ]
original_section_name = tool_section_dict[ 'name' ]
if no_changes_checked:
if original_section_id in [ '' ]:
tool_section = None
else:
section_key = 'section_%s' % str( original_section_id )
if section_key in trans.app.toolbox.tool_panel:
tool_section = trans.app.toolbox.tool_panel[ section_key ]
else:
tool_panel_dict = generate_tool_panel_dict_for_new_install( metadata[ 'tools' ] )
# TODO: Fix this to handle the case where the tools are distributed across in more than 1 ToolSection. The
# following assumes everything was loaded into 1 section (or no section) in the tool panel.
tool_section_dicts = tool_panel_dict[ tool_panel_dict.keys()[ 0 ] ]
tool_section_dict = tool_section_dicts[ 0 ]
original_section_id = tool_section_dict[ 'id' ]
original_section_name = tool_section_dict[ 'name' ]
if no_changes_checked:
if original_section_id in [ '' ]:
tool_section = None
else:
section_key = 'section_%s' % str( original_section_id )
if section_key in trans.app.toolbox.tool_panel:
tool_section = trans.app.toolbox.tool_panel[ section_key ]
else:
# The section in which the tool was originally loaded used to be in the tool panel, but no longer is.
elem = Element( 'section' )
elem.attrib[ 'name' ] = original_section_name
elem.attrib[ 'id' ] = original_section_id
elem.attrib[ 'version' ] = ''
tool_section = tools.ToolSection( elem )
trans.app.toolbox.tool_panel[ section_key ] = tool_section
# The section in which the tool was originally loaded used to be in the tool panel, but no longer is.
elem = Element( 'section' )
elem.attrib[ 'name' ] = original_section_name
elem.attrib[ 'id' ] = original_section_id
elem.attrib[ 'version' ] = ''
tool_section = tools.ToolSection( elem )
trans.app.toolbox.tool_panel[ section_key ] = tool_section
else:
# The user elected to change the tool panel section to contain the tools.
new_tool_panel_section = kwd.get( 'new_tool_panel_section', '' )
tool_panel_section = kwd.get( 'tool_panel_section', '' )
if new_tool_panel_section:
section_id = new_tool_panel_section.lower().replace( ' ', '_' )
new_section_key = 'section_%s' % str( section_id )
if new_section_key in trans.app.toolbox.tool_panel:
# Appending a tool to an existing section in trans.app.toolbox.tool_panel
log.debug( "Appending to tool panel section: %s" % new_tool_panel_section )
tool_section = trans.app.toolbox.tool_panel[ new_section_key ]
else:
# The user elected to change the tool panel section to contain the tools.
new_tool_panel_section = kwd.get( 'new_tool_panel_section', '' )
tool_panel_section = kwd.get( 'tool_panel_section', '' )
if new_tool_panel_section:
section_id = new_tool_panel_section.lower().replace( ' ', '_' )
new_section_key = 'section_%s' % str( section_id )
if new_section_key in trans.app.toolbox.tool_panel:
# Appending a tool to an existing section in trans.app.toolbox.tool_panel
log.debug( "Appending to tool panel section: %s" % new_tool_panel_section )
tool_section = trans.app.toolbox.tool_panel[ new_section_key ]
else:
# Appending a new section to trans.app.toolbox.tool_panel
log.debug( "Loading new tool panel section: %s" % new_tool_panel_section )
elem = Element( 'section' )
elem.attrib[ 'name' ] = new_tool_panel_section
elem.attrib[ 'id' ] = section_id
elem.attrib[ 'version' ] = ''
tool_section = tools.ToolSection( elem )
trans.app.toolbox.tool_panel[ new_section_key ] = tool_section
elif tool_panel_section:
section_key = 'section_%s' % tool_panel_section
tool_section = trans.app.toolbox.tool_panel[ section_key ]
else:
tool_section = None
tool_shed_repository, metadata_dict = load_repository_contents( trans,
repository_name=repository.name,
description=repository.description,
owner=repository.owner,
changeset_revision=repository.installed_changeset_revision,
tool_path=tool_path,
repository_clone_url=repository_clone_url,
relative_install_dir=relative_install_dir,
current_working_dir=current_working_dir,
tmp_name=tmp_name,
tool_shed=repository.tool_shed,
tool_section=tool_section,
shed_tool_conf=shed_tool_conf )
repository.uninstalled = False
# Appending a new section to trans.app.toolbox.tool_panel
log.debug( "Loading new tool panel section: %s" % new_tool_panel_section )
elem = Element( 'section' )
elem.attrib[ 'name' ] = new_tool_panel_section
elem.attrib[ 'id' ] = section_id
elem.attrib[ 'version' ] = ''
tool_section = tools.ToolSection( elem )
trans.app.toolbox.tool_panel[ new_section_key ] = tool_section
elif tool_panel_section:
section_key = 'section_%s' % tool_panel_section
tool_section = trans.app.toolbox.tool_panel[ section_key ]
else:
tool_section = None
tool_shed_repository, metadata_dict = load_repository_contents( trans,
repository_name=repository.name,
description=repository.description,
owner=repository.owner,
changeset_revision=repository.installed_changeset_revision,
ctx_rev=ctx_rev,
tool_path=tool_path,
repository_clone_url=repository_clone_url,
relative_install_dir=relative_install_dir,
current_working_dir=current_working_dir,
tool_shed=repository.tool_shed,
tool_section=tool_section,
shed_tool_conf=shed_tool_conf )
repository.uninstalled = False
repository.deleted = False
trans.sa_session.add( repository )
trans.sa_session.flush()
@@ -545,7 +564,7 @@ class AdminToolshed( AdminGalaxy ):
# Get the tool_versions from the tool shed for each tool in the installed change set.
repository = get_repository( trans, kwd[ 'id' ] )
tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository )
url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \
url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy&no_reset=true' % \
( tool_shed_url, repository.name, repository.owner, repository.changeset_revision )
response = urllib2.urlopen( url )
text = response.read()
@@ -580,41 +599,32 @@ class AdminToolshed( AdminGalaxy ):
owner = params.get( 'owner', None )
changeset_revision = params.get( 'changeset_revision', None )
latest_changeset_revision = params.get( 'latest_changeset_revision', None )
latest_ctx_rev = params.get( 'latest_ctx_rev', None )
repository = get_repository_by_shed_name_owner_changeset_revision( trans.app, tool_shed_url, name, owner, changeset_revision )
if changeset_revision and latest_changeset_revision:
if changeset_revision and latest_changeset_revision and latest_ctx_rev:
if changeset_revision == latest_changeset_revision:
message = "The cloned tool shed repository named '%s' is current (there are no updates available)." % name
else:
current_working_dir = os.getcwd()
shed_tool_conf, tool_path, relative_install_dir = get_tool_panel_config_tool_path_install_dir( trans.app, repository )
if relative_install_dir:
repo_files_dir = os.path.join( relative_install_dir, name )
returncode, tmp_name = pull_repository( current_working_dir, repo_files_dir, name )
if returncode == 0:
returncode, tmp_name = update_repository( current_working_dir, repo_files_dir, latest_changeset_revision )
if returncode == 0:
# Update the repository metadata.
repository_clone_url = os.path.join( tool_shed_url, 'repos', owner, name )
tool_shed = clean_tool_shed_url( tool_shed_url )
metadata_dict = generate_metadata( trans.app.toolbox, relative_install_dir, repository_clone_url )
repository.metadata = metadata_dict
# Update the repository changeset_revision in the database.
repository.changeset_revision = latest_changeset_revision
repository.update_available = False
trans.sa_session.add( repository )
trans.sa_session.flush()
message = "The cloned repository named '%s' has been updated to change set revision '%s'." % \
( name, latest_changeset_revision )
else:
tmp_stderr = open( tmp_name, 'rb' )
message = tmp_stderr.read()
tmp_stderr.close()
status = 'error'
else:
tmp_stderr = open( tmp_name, 'rb' )
message = tmp_stderr.read()
tmp_stderr.close()
status = 'error'
repo_files_dir = os.path.abspath( os.path.join( relative_install_dir, name ) )
repo = hg.repository( get_configured_ui(), path=repo_files_dir )
repository_clone_url = os.path.join( tool_shed_url, 'repos', owner, name )
pull_repository( repo, repository_clone_url, latest_ctx_rev )
update_repository( repo, latest_ctx_rev )
# Update the repository metadata.
tool_shed = clean_tool_shed_url( tool_shed_url )
metadata_dict = generate_metadata( trans.app.toolbox, relative_install_dir, repository_clone_url )
repository.metadata = metadata_dict
# Update the repository changeset_revision in the database.
repository.changeset_revision = latest_changeset_revision
repository.ctx_rev = latest_ctx_rev
repository.update_available = False
trans.sa_session.add( repository )
trans.sa_session.flush()
message = "The cloned repository named '%s' has been updated to change set revision '%s'." % \
( name, latest_changeset_revision )
else:
message = "The directory containing the cloned repository named '%s' cannot be found." % name
status = 'error'
lib/galaxy/web/controllers/cloud.py
+4 -1
View File
@@ -6,10 +6,13 @@ BioCloudCentral Source: https://github.com/chapmanb/biocloudcentral
"""
import boto
import datetime
import logging
import time
from galaxy import eggs
import pkg_resources
pkg_resources.require('boto')
import boto
from galaxy import web
from galaxy.web.base.controller import BaseUIController
from boto.ec2.regioninfo import RegionInfo
lib/galaxy/web/controllers/data_admin.py
+175
View File
@@ -0,0 +1,175 @@
import sys, ftplib, json
from galaxy import model, util
from galaxy.jobs import transfer_manager
from galaxy.web.base.controller import *
from galaxy.web.framework.helpers import time_ago, iff, grids
from galaxy.model.orm import *
from library_common import get_comptypes, lucene_search, whoosh_search
# Older py compatibility
try:
set()
except:
from sets import Set as set
import logging
log = logging.getLogger( __name__ )
class DataAdmin( BaseUIController ):
jobstyles = dict(
done='panel-done-message',
waiting='state-color-waiting',
running='state-color-running',
downloaded='state-color-running',
new='state-color-new',
ok='panel-done-message',
error='panel-error-message'
)
@web.expose
@web.require_admin
def manage_data( self, trans, **kwd ):
dbkeys = trans.db_builds
return trans.fill_template( '/admin/data_admin/data_form.mako', dbkeys=dbkeys )
@web.expose
@web.require_admin
def download_build( self, trans, **kwd ):
"""Download a genome from a remote source and add it to the library."""
params = util.Params( kwd )
source = params.get('source', '')
longname = params.get('longname', None)
if not isinstance( params.get( 'indexers', None ), list ):
indexers = [ params.get( 'indexers', None ) ]
else:
indexers = params.get( 'indexers', None )
if indexers is not None:
if indexers == [None]:
indexers = None
url = None
liftover = None
newlift = []
dbkey = params.get( 'dbkey', None )
dbkeys = dict()
protocol = 'http'
if source == 'NCBI':
dbkey = params.get('dbkey', '')[0]
url = 'http://togows.dbcls.jp/entry/ncbi-nucleotide/%s.fasta' % dbkey
elif source == 'Broad':
dbkey = params.get('dbkey', '')[0]
url = 'ftp://ftp.broadinstitute.org/pub/seq/references/%s.fasta' % dbkey
elif source == 'UCSC':
longname = None
for build in trans.db_builds:
if dbkey[1] == build[0]:
dbkey = build[0]
longname = build[1]
break
assert dbkey is not '?', 'That build was not found'
ftp = ftplib.FTP('hgdownload.cse.ucsc.edu')
ftp.login('anonymous', 'user@example.com')
checker = []
liftover = []
newlift = []
try:
ftp.retrlines('NLST /goldenPath/%s/liftOver/*.chain.gz' % dbkey, liftover.append)
for chain in liftover:
fname = chain.split( '/' )[-1]
target = fname.replace( '.over.chain.gz', '' ).split( 'To' )[1]
target = target[0].lower() + target[1:]
newlift.append( [ chain, dbkey, target ] )
current = dbkey[0].upper() + dbkey[1:]
targetfile = '%sTo%s.over.chain.gz' % ( target, current )
newlift.append( [ '/goldenPath/%s/liftOver/%s' % ( target, targetfile ), target, dbkey ] )
except:
newlift = None
pass
ftp.retrlines('NLST /goldenPath/%s/bigZips/' % dbkey, checker.append)
for filename in [ dbkey, 'chromFa' ]:
for extension in [ '.tar.gz', '.tar.bz2', '.zip', '.fa.gz', '.fa.bz2' ]:
testfile = '/goldenPath/%s/bigZips/%s%s' % ( dbkey, filename, extension )
if testfile in checker:
url = 'ftp://hgdownload.cse.ucsc.edu%s' % testfile
break;
else:
continue
if url is None:
message = u'The genome %s was not found on the UCSC server.' % dbkey
status = u'error'
return trans.fill_template( '/admin/data_admin/data_form.mako',
message=message,
status=status )
elif source == 'Ensembl':
section = params.get('ensembl_section', '')
release1 = params.get('release_number', '')
organism = params.get('organism', '')
name = params.get('name', '')
longname = organism
dbkey = name
release2 = params.get('release2', '')
release2 = ".%s" % release2 if release2 else ""
if section == 'standard':
url = 'ftp://ftp.ensembl.org/pub/release-%s/fasta/%s/dna/%s.%s%s.dna.toplevel.fa.gz' % \
(release1, organism.lower(), organism, name, release2)
else:
url = 'ftp://ftp.ensemblgenomes.org/pub/%s/release-%s/fasta/%s/dna/%s.%s%s.dna.toplevel.fa.gz' % \
(section, release1, organism.lower(), organism, name, release2)
elif source == 'local':
url = 'http://127.0.0.1/%s.tar.gz' % dbkey
else:
raise ValueError
params = dict( protocol='http', name=dbkey, datatype='fasta', url=url, user=trans.user.id )
jobid = trans.app.job_manager.deferred_job_queue.plugins['GenomeTransferPlugin'].create_job( trans, url, dbkey, longname, indexers )
chainjob = []
if newlift is not None:
for chain in newlift:
liftover_url = u'ftp://hgdownload.cse.ucsc.edu%s' % chain[0]
from_genome = chain[1]
to_genome = chain[2]
destfile = liftover_url.split('/')[-1].replace('.gz', '')
chainjob.append( trans.app.job_manager.deferred_job_queue.plugins['LiftOverTransferPlugin'].create_job( trans, liftover_url, dbkey, from_genome, to_genome, destfile ) )
job = trans.app.job_manager.deferred_job_queue.plugins['GenomeTransferPlugin'].get_job_status( jobid )
job.params['liftover'] = chainjob
trans.app.model.context.current.add( job )
trans.app.model.context.current.flush()
return trans.response.send_redirect( web.url_for( controller='data_admin',
action='monitor_status',
job=jobid ) )
@web.expose
@web.require_admin
def monitor_status( self, trans, **kwd ):
params = util.Params( kwd )
jobid = params.get( 'job', '' )
chains = params.get( 'chains', [] )
jobs = self._get_jobs( jobid, trans )
return trans.fill_template( '/admin/data_admin/download_status.mako', mainjob=jobid, jobs=jobs )
@web.expose
@web.require_admin
def ajax_statusupdate( self, trans, **kwd ):
sa_session = trans.app.model.context.current
jobs = []
params = util.Params( kwd )
jobid = params.get( 'jobid', '' )
jobs = self._get_jobs( jobid, trans )
return trans.fill_template( '/admin/data_admin/ajax_statusupdate.mako', mainjob=jobid, jobs=jobs )
def _get_jobs( self, jobid, trans ):
jobs = []
job = trans.app.job_manager.deferred_job_queue.plugins['GenomeTransferPlugin'].get_job_status( jobid )
sa_session = trans.app.model.context.current
idxjobs = sa_session.query( model.GenomeIndexToolData ).filter_by( deferred_job_id=job.id, transfer_job_id=job.transfer_job.id ).all()
if job.params[ 'liftover' ] is not None:
for jobid in job.params[ 'liftover' ]:
lo_job = trans.app.job_manager.deferred_job_queue.plugins['LiftOverTransferPlugin'].get_job_status( jobid )
jobs.append( dict( jobid=lo_job.id, state=lo_job.state, type='Download liftOver' ) )
for idxjob in idxjobs:
jobentry = sa_session.query( model.Job ).filter_by( id=idxjob.job_id ).first()
jobs.append( dict( jobid=jobentry.id, state=jobentry.state, type='Index Genome' ) )
jobs.append( dict ( jobid=job.id, state=job.state, type='Main Job' ) )
jobs.append( dict ( jobid=job.transfer_job.id, state=job.transfer_job.state, type='Download Genome' ) )
for je in jobs:
je[ 'style' ] = self.jobstyles[ je[ 'state' ] ]
return jobs
lib/galaxy/web/controllers/dataset.py
+75 -208
View File
@@ -1,17 +1,22 @@
import logging, os, string, shutil, re, socket, mimetypes, urllib, tempfile, zipfile, glob, sys
import logging
import mimetypes
import os
import string
import sys
import tempfile
import urllib
import zipfile
from galaxy.web.base.controller import *
from galaxy.web.framework.helpers import time_ago, iff, grids
from galaxy import util, datatypes, jobs, web, model
from cgi import escape, FieldStorage
from galaxy import util, datatypes, web, model
from galaxy.datatypes.display_applications.util import encode_dataset_user, decode_dataset_user
from galaxy.util.sanitize_html import sanitize_html
from galaxy.util import inflector
from galaxy.model.item_attrs import *
from galaxy.model import LibraryDatasetDatasetAssociation, HistoryDatasetAssociation
from galaxy.web.framework.helpers import to_unicode
import pkg_resources;
import pkg_resources;
pkg_resources.require( "Paste" )
import paste.httpexceptions
@@ -32,7 +37,7 @@ try:
except RuntimeError:
log.exception( "Compression error when testing zip compression. This option will be disabled for library downloads." )
except (TypeError, zipfile.LargeZipFile): # ZIP64 is only in Python2.5+. Remove TypeError when 2.4 support is dropped
log.warning( 'Max zip file size is 2GB, ZIP64 not supported' )
log.warning( 'Max zip file size is 2GB, ZIP64 not supported' )
comptypes.append( 'zip' )
try:
os.unlink( tmpf )
@@ -53,7 +58,7 @@ This is in reference to dataset id ${dataset_id} from history id ${history_id}
-----------------------------------------------------------------------------
You should be able to view the history containing the related history item
${hid}: ${history_item_name}
${hid}: ${history_item_name}
by logging in as a Galaxy admin user to the Galaxy instance referenced above
and pointing your browser to the following link.
@@ -90,7 +95,7 @@ class HistoryDatasetAssociationListGrid( grids.Grid ):
class HistoryColumn( grids.GridColumn ):
def get_value( self, trans, grid, hda):
return hda.history.name
class StatusColumn( grids.GridColumn ):
def get_value( self, trans, grid, hda ):
if hda.deleted:
@@ -111,19 +116,19 @@ class HistoryDatasetAssociationListGrid( grids.Grid ):
template='/dataset/grid.mako'
default_sort_key = "-update_time"
columns = [
grids.TextColumn( "Name", key="name",
grids.TextColumn( "Name", key="name",
# Link name to dataset's history.
link=( lambda item: iff( item.history.deleted, None, dict( operation="switch", id=item.id ) ) ), filterable="advanced", attach_popup=True ),
HistoryColumn( "History", key="history",
HistoryColumn( "History", key="history",
link=( lambda item: iff( item.history.deleted, None, dict( operation="switch_history", id=item.id ) ) ) ),
grids.IndividualTagsColumn( "Tags", key="tags", model_tag_association_class=model.HistoryDatasetAssociationTagAssociation, filterable="advanced", grid_name="HistoryDatasetAssocationListGrid" ),
StatusColumn( "Status", key="deleted", attach_popup=False ),
grids.GridColumn( "Last Updated", key="update_time", format=time_ago ),
]
columns.append(
grids.MulticolFilterColumn(
"Search",
cols_to_filter=[ columns[0], columns[2] ],
columns.append(
grids.MulticolFilterColumn(
"Search",
cols_to_filter=[ columns[0], columns[2] ],
key="free-text-search", visible=False, filterable="standard" )
)
operations = [
@@ -136,17 +141,17 @@ class HistoryDatasetAssociationListGrid( grids.Grid ):
num_rows_per_page = 50
def build_initial_query( self, trans, **kwargs ):
# Show user's datasets that are not deleted, not in deleted histories, and not hidden.
# To filter HDAs by user, need to join model class/HDA and History table so that it is
# possible to filter by user. However, for dictionary-based filtering to work, need a
# To filter HDAs by user, need to join model class/HDA and History table so that it is
# possible to filter by user. However, for dictionary-based filtering to work, need a
# primary table for the query.
return trans.sa_session.query( self.model_class ).select_from( self.model_class.table.join( model.History.table ) ) \
.filter( model.History.user == trans.user ) \
.filter( self.model_class.deleted==False ) \
.filter( model.History.deleted==False ) \
.filter( self.model_class.visible==True )
class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHistoryDatasetAssociation, UsesItemRatings ):
stored_list_grid = HistoryDatasetAssociationListGrid()
@web.expose
@@ -202,7 +207,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
job_stdout=job.stdout,
job_info=job.info,
job_traceback=job.traceback,
email=email,
email=email,
message=message )
frm = to_address
# Check email a bit
@@ -219,130 +224,45 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
return trans.show_ok_message( "Your error report has been sent" )
except Exception, e:
return trans.show_error_message( "An error occurred sending the report by email: %s" % str( e ) )
@web.expose
def default(self, trans, dataset_id=None, **kwd):
return 'This link may not be followed from within Galaxy.'
@web.expose
def archive_composite_dataset( self, trans, data=None, **kwd ):
# save a composite object into a compressed archive for downloading
params = util.Params( kwd )
valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
outfname = data.name[0:150]
outfname = ''.join(c in valid_chars and c or '_' for c in outfname)
if (params.do_action == None):
params.do_action = 'zip' # default
msg = util.restore_text( params.get( 'msg', '' ) )
messagetype = params.get( 'messagetype', 'done' )
if not data:
msg = "You must select at least one dataset"
messagetype = 'error'
else:
error = False
try:
if (params.do_action == 'zip'):
# Can't use mkstemp - the file must not exist first
tmpd = tempfile.mkdtemp()
tmpf = os.path.join( tmpd, 'library_download.' + params.do_action )
if ziptype == '64':
archive = zipfile.ZipFile( tmpf, 'w', zipfile.ZIP_DEFLATED, True )
else:
archive = zipfile.ZipFile( tmpf, 'w', zipfile.ZIP_DEFLATED )
archive.add = lambda x, y: archive.write( x, y.encode('CP437') )
elif params.do_action == 'tgz':
archive = util.streamball.StreamBall( 'w|gz' )
elif params.do_action == 'tbz':
archive = util.streamball.StreamBall( 'w|bz2' )
except (OSError, zipfile.BadZipFile):
error = True
log.exception( "Unable to create archive for download" )
msg = "Unable to create archive for %s for download, please report this error" % outfname
messagetype = 'error'
if not error:
current_user_roles = trans.get_current_user_roles()
ext = data.extension
path = data.file_name
fname = os.path.split(path)[-1]
efp = data.extra_files_path
htmlname = os.path.splitext(outfname)[0]
if not htmlname.endswith(ext):
htmlname = '%s_%s' % (htmlname,ext)
archname = '%s.html' % htmlname # fake the real nature of the html file
try:
archive.add(data.file_name,archname)
except IOError:
error = True
log.exception( "Unable to add composite parent %s to temporary library download archive" % data.file_name)
msg = "Unable to create archive for download, please report this error"
messagetype = 'error'
for root, dirs, files in os.walk(efp):
for fname in files:
fpath = os.path.join(root,fname)
rpath = os.path.relpath(fpath,efp)
try:
archive.add( fpath,rpath )
except IOError:
error = True
log.exception( "Unable to add %s to temporary library download archive" % rpath)
msg = "Unable to create archive for download, please report this error"
messagetype = 'error'
continue
if not error:
if params.do_action == 'zip':
archive.close()
tmpfh = open( tmpf )
# CANNOT clean up - unlink/rmdir was always failing because file handle retained to return - must rely on a cron job to clean up tmp
trans.response.set_content_type( "application/x-zip-compressed" )
trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s.zip"' % outfname
return tmpfh
else:
trans.response.set_content_type( "application/x-tar" )
outext = 'tgz'
if params.do_action == 'tbz':
outext = 'tbz'
trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s.%s"' % (outfname,outext)
archive.wsgi_status = trans.response.wsgi_status()
archive.wsgi_headeritems = trans.response.wsgi_headeritems()
return archive.stream
return trans.show_error_message( msg )
@web.expose
def get_metadata_file(self, trans, hda_id, metadata_name):
""" Allows the downloading of metadata files associated with datasets (eg. bai index for bam files) """
data = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( trans.security.decode_id( hda_id ) )
if not data or not trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), data.dataset ):
return trans.show_error_message( "You are not allowed to access this dataset" )
valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
fname = ''.join(c in valid_chars and c or '_' for c in data.name)[0:150]
file_ext = data.metadata.spec.get(metadata_name).get("file_ext", metadata_name)
trans.response.headers["Content-Type"] = "application/octet-stream"
trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy%s-[%s].%s"' % (data.hid, fname, file_ext)
return open(data.metadata.get(metadata_name).file_name)
def _check_dataset(self, trans, dataset_id):
def _check_dataset(self, trans, hda_id):
# DEPRECATION: We still support unencoded ids for backward compatibility
try:
data = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( trans.security.decode_id( dataset_id ) )
data = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( trans.security.decode_id( hda_id) )
if data is None:
raise ValueError( 'Invalid reference dataset id: %s.' % dataset_id )
raise ValueError( 'Invalid reference dataset id: %s.' % hda_id)
except:
try:
data = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( int( dataset_id ) )
data = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( int( hda_id ) )
except:
data = None
if not data:
raise paste.httpexceptions.HTTPRequestRangeNotSatisfiable( "Invalid reference dataset id: %s." % str( dataset_id ) )
raise paste.httpexceptions.HTTPRequestRangeNotSatisfiable( "Invalid reference dataset id: %s." % str( hda_id ) )
if not trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), data.dataset ):
return trans.show_error_message( "You are not allowed to access this dataset" )
if data.state == trans.model.Dataset.states.UPLOAD:
return trans.show_error_message( "Please wait until this dataset finishes uploading before attempting to view it." )
return data
@web.expose
@web.json
def transfer_status(self, trans, dataset_id, filename=None):
@@ -352,7 +272,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
if isinstance( data, basestring ):
return data
log.debug( "Checking transfer status for dataset %s..." % data.dataset.id )
# Pulling files in extra_files_path into cache is not handled via this
# method but that's primarily because those files are typically linked to
# through tool's output page anyhow so tying a JavaScript event that will
@@ -361,63 +281,11 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
return True
else:
return trans.app.object_store.file_ready(data.dataset)
@web.expose
def display(self, trans, dataset_id=None, preview=False, filename=None, to_ext=None, **kwd):
"""Catches the dataset id and displays file contents as directed"""
composite_extensions = trans.app.datatypes_registry.get_composite_extensions( )
composite_extensions.append('html') # for archiving composite datatypes
def display(self, trans, dataset_id=None, preview=False, filename=None, to_ext=None, chunk=None, **kwd):
data = self._check_dataset(trans, dataset_id)
if isinstance( data, basestring ):
return data
if filename and filename != "index":
# For files in extra_files_path
file_path = trans.app.object_store.get_filename(data.dataset, extra_dir='dataset_%s_files' % data.dataset.id, alt_name=filename)
if os.path.exists( file_path ):
if os.path.isdir( file_path ):
return trans.show_error_message( "Directory listing is not allowed." ) #TODO: Reconsider allowing listing of directories?
mime, encoding = mimetypes.guess_type( file_path )
if not mime:
try:
mime = trans.app.datatypes_registry.get_mimetype_by_extension( ".".split( file_path )[-1] )
except:
mime = "text/plain"
trans.response.set_content_type( mime )
return open( file_path )
else:
return trans.show_error_message( "Could not find '%s' on the extra files path %s." % ( filename, file_path ) )
trans.response.set_content_type(data.get_mime())
trans.log_event( "Display dataset id: %s" % str( dataset_id ) )
if to_ext or isinstance(data.datatype, datatypes.binary.Binary): # Saving the file, or binary file
if data.extension in composite_extensions:
return self.archive_composite_dataset( trans, data, **kwd )
else:
trans.response.headers['Content-Length'] = int( os.stat( data.file_name ).st_size )
if not to_ext:
to_ext = data.extension
valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
fname = ''.join(c in valid_chars and c or '_' for c in data.name)[0:150]
trans.response.set_content_type( "application/octet-stream" ) #force octet-stream so Safari doesn't append mime extensions to filename
trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy%s-[%s].%s"' % (data.hid, fname, to_ext)
return open( data.file_name )
if not os.path.exists( data.file_name ):
raise paste.httpexceptions.HTTPNotFound( "File Not Found (%s)." % data.file_name )
max_peek_size = 1000000 # 1 MB
if isinstance(data.datatype, datatypes.images.Html):
max_peek_size = 10000000 # 10 MB for html
if not preview or isinstance(data.datatype, datatypes.images.Image) or os.stat( data.file_name ).st_size < max_peek_size:
if trans.app.config.sanitize_all_html and trans.response.get_content_type() == "text/html":
# Sanitize anytime we respond with plain text/html content.
return sanitize_html(open( data.file_name ).read())
return open( data.file_name )
else:
trans.response.set_content_type( "text/html" )
return trans.stream_template_mako( "/dataset/large_file.mako",
truncated_data = open( data.file_name ).read(max_peek_size),
data = data )
return data.datatype.display_data(trans, data, preview, filename, to_ext, chunk, **kwd)
@web.expose
def edit(self, trans, dataset_id=None, filename=None, hid=None, **kwd):
@@ -443,7 +311,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
# TODO: hid handling
data = history.datasets[ int( hid ) - 1 ]
id = None
elif dataset_id is not None:
elif dataset_id is not None:
id = trans.app.security.decode_id( dataset_id )
data = trans.sa_session.query( self.app.model.HistoryDatasetAssociation ).get( id )
else:
@@ -463,7 +331,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
# permission. In this case, we'll reset this permission to the hda user's private role.
manage_permissions_action = trans.app.security_agent.get_action( trans.app.security_agent.permitted_actions.DATASET_MANAGE_PERMISSIONS.action )
permissions = { manage_permissions_action : [ trans.app.security_agent.get_private_user_role( data.history.user ) ] }
trans.app.security_agent.set_dataset_permission( data.dataset, permissions )
trans.app.security_agent.set_dataset_permission( data.dataset, permissions )
if trans.app.security_agent.can_access_dataset( current_user_roles, data.dataset ):
if data.state == trans.model.Dataset.states.UPLOAD:
return trans.show_error_message( "Please wait until this dataset finishes uploading before attempting to edit its metadata." )
@@ -600,7 +468,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
refresh_frames=refresh_frames )
else:
return trans.show_error_message( "You do not have permission to edit this dataset's ( id: %s ) information." % str( dataset_id ) )
@web.expose
@web.require_login( "see all available datasets" )
def list( self, trans, **kwargs ):
@@ -610,7 +478,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
if 'operation' in kwargs:
operation = kwargs['operation'].lower()
hda_ids = util.listify( kwargs.get( 'id', [] ) )
# Display no message by default
status, message = None, None
@@ -630,15 +498,15 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
if hdas:
if operation == "switch" or operation == "switch_history":
# Switch to a history that the HDA resides in.
# Convert hda to histories.
histories = []
for hda in hdas:
histories.append( hda.history )
# Use history controller to switch the history. TODO: is this reasonable?
status, message = trans.webapp.controllers['history']._list_switch( trans, histories )
# Current history changed, refresh history frame; if switching to a dataset, set hda seek.
trans.template_context['refresh_frames'] = ['history']
if operation == "switch":
@@ -648,35 +516,35 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
# Copy a dataset to the current history.
target_histories = [ trans.get_history() ]
status, message = self._copy_datasets( trans, hda_ids, target_histories )
# Current history changed, refresh history frame.
trans.template_context['refresh_frames'] = ['history']
# Render the list view
return self.stored_list_grid( trans, status=status, message=message, **kwargs )
@web.expose
def imp( self, trans, dataset_id=None, **kwd ):
""" Import another user's dataset via a shared URL; dataset is added to user's current history. """
msg = ""
# Set referer message.
referer = trans.request.referer
if referer is not "":
referer_message = "<a href='%s'>return to the previous page</a>" % referer
else:
referer_message = "<a href='%s'>go to Galaxy's start page</a>" % url_for( '/' )
# Error checking.
if not dataset_id:
return trans.show_error_message( "You must specify a dataset to import. You can %s." % referer_message, use_panels=True )
# Do import.
cur_history = trans.get_history( create=True )
status, message = self._copy_datasets( trans, [ dataset_id ], [ cur_history ], imported=True )
message = "Dataset imported. <br>You can <a href='%s'>start using the dataset</a> or %s." % ( url_for('/'), referer_message )
return trans.show_message( message, type=status, use_panels=True )
@web.expose
@web.json
@web.require_login( "use Galaxy datasets" )
@@ -685,7 +553,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
dataset = self.get_dataset( trans, id, False, True )
return_dict = { "name" : dataset.name, "link" : url_for( action="display_by_username_and_slug", username=dataset.history.user.username, slug=trans.security.encode_id( dataset.id ) ) }
return return_dict
@web.expose
def get_embed_html_async( self, trans, id ):
""" Returns HTML for embedding a dataset in a page. """
@@ -698,7 +566,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
def set_accessible_async( self, trans, id=None, accessible=False ):
""" Does nothing because datasets do not have an importable/accessible attribute. This method could potentially set another attribute. """
return
@web.expose
@web.require_login( "rate items" )
@web.json
@@ -713,7 +581,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
dataset_rating = self.rate_item( rate_item, trans.get_user(), dataset, rating )
return self.get_ave_item_rating_data( trans.sa_session, dataset )
@web.expose
def display_by_username_and_slug( self, trans, username, slug, filename=None, preview=True ):
""" Display dataset by username and slug; because datasets do not yet have slugs, the slug is the dataset's id. """
@@ -722,10 +590,10 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
# Filename used for composite types.
if filename:
return self.display( trans, dataset_id=slug, filename=filename)
truncated, dataset_data = self.get_data( dataset, preview )
dataset.annotation = self.get_item_annotation_str( trans.sa_session, dataset.history.user, dataset )
# If data is binary or an image, stream without template; otherwise, use display template.
# TODO: figure out a way to display images in display template.
if isinstance(dataset.datatype, datatypes.binary.Binary) or isinstance(dataset.datatype, datatypes.images.Image) or isinstance(dataset.datatype, datatypes.images.Html):
@@ -741,12 +609,12 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
else:
user_item_rating = 0
ave_item_rating, num_ratings = self.get_ave_item_rating_data( trans.sa_session, dataset )
return trans.fill_template_mako( "/dataset/display.mako", item=dataset, item_data=dataset_data, truncated=truncated,
user_item_rating = user_item_rating, ave_item_rating=ave_item_rating, num_ratings=num_ratings )
else:
raise web.httpexceptions.HTTPNotFound()
@web.expose
def get_item_content_async( self, trans, id ):
""" Returns item content in HTML format. """
@@ -758,7 +626,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
# Get annotation.
dataset.annotation = self.get_item_annotation_str( trans.sa_session, trans.user, dataset )
return trans.stream_template_mako( "/dataset/item_content.mako", item=dataset, item_data=dataset_data, truncated=truncated )
@web.expose
def annotate_async( self, trans, id, new_annotation=None, **kwargs ):
dataset = self.get_dataset( trans, id, False, True )
@@ -770,7 +638,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
self.add_item_annotation( trans.sa_session, trans.get_user(), dataset, new_annotation )
trans.sa_session.flush()
return new_annotation
@web.expose
def get_annotation_async( self, trans, id ):
dataset = self.get_dataset( trans, id, False, True )
@@ -841,7 +709,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
if app_action in [ 'data', 'param' ]:
assert action_param, "An action param must be provided for a data or param action"
#data is used for things with filenames that could be passed off to a proxy
#in case some display app wants all files to be in the same 'directory',
#in case some display app wants all files to be in the same 'directory',
#data can be forced to param, but not the other way (no filename for other direction)
#get param name from url param name
try:
@@ -904,8 +772,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
if job.state in [ self.app.model.Job.states.QUEUED, self.app.model.Job.states.RUNNING, self.app.model.Job.states.NEW ]:
# Are *all* of the job's other output datasets deleted?
if job.check_if_output_datasets_deleted():
job.mark_deleted( self.app.config.get_bool( 'enable_job_running', True ),
self.app.config.get_bool( 'track_jobs_in_database', False ) )
job.mark_deleted( self.app.config.track_jobs_in_database )
self.app.job_manager.job_stop_queue.put( job.id )
trans.sa_session.flush()
except Exception, e:
@@ -961,7 +828,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
trans.log_event( "Dataset id %s has been unhidden" % str(id) )
return True
return False
def _purge( self, trans, dataset_id ):
message = None
status = 'done'
@@ -1038,7 +905,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
return "OK"
else:
raise Exception( message )
@web.expose
def unhide( self, trans, dataset_id, filename ):
if self._unhide( trans, dataset_id ):
@@ -1071,7 +938,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
"""
Show the parameters used for an HDA
"""
def source_dataset_chain( dataset, lst ):
try:
cp_from_ldda = dataset.copied_from_library_dataset_dataset_association
@@ -1085,13 +952,13 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
except:
pass
return lst
hda = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( trans.security.decode_id( dataset_id ) )
if not hda:
raise paste.httpexceptions.HTTPRequestRangeNotSatisfiable( "Invalid reference dataset id: %s." % str( dataset_id ) )
if not trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), hda.dataset ):
return trans.show_error_message( "You are not allowed to access this dataset" )
# Get the associated job, if any. If this hda was copied from another,
# we need to find the job that created the origial hda
params_objects = None
@@ -1103,7 +970,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
job = None
for assoc in job_hda.creating_job_associations:
job = assoc.job
break
break
if job:
# Get the tool object
try:
@@ -1114,10 +981,10 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
params_objects = job.get_param_values( trans.app )
except:
pass
inherit_chain = source_dataset_chain(hda, [])
return trans.fill_template( "show_params.mako", inherit_chain=inherit_chain, history=trans.get_history(), hda=hda, tool=tool, params_objects=params_objects )
@web.expose
def copy_datasets( self, trans, source_history=None, source_dataset_ids="", target_history_id=None, target_history_ids="", new_history_name="", do_copy=False, **kwd ):
params = util.Params( kwd )
@@ -1176,7 +1043,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
if history in target_histories:
refresh_frames = ['history']
trans.sa_session.flush()
hist_names_str = ", ".join( ['<a href="%s" target="_top">%s</a>' %
hist_names_str = ", ".join( ['<a href="%s" target="_top">%s</a>' %
( url_for( controller="history", action="switch_to_history", \
hist_id=trans.security.encode_id( hist.id ) ), hist.name ) \
for hist in target_histories ] )
@@ -1187,7 +1054,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
source_datasets = history.visible_datasets
target_histories = [history]
if user:
target_histories = user.active_histories
target_histories = user.active_histories
return trans.fill_template( "/dataset/copy_view.mako",
source_history = history,
current_history = trans.get_history(),
@@ -1205,7 +1072,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
""" Helper method for copying datasets. """
user = trans.get_user()
done_msg = error_msg = ""
invalid_datasets = 0
if not dataset_ids or not target_histories:
error_msg = "You must provide both source datasets and target histories."
@@ -1230,7 +1097,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist
done_msg = "%i dataset%s copied to %i histor%s." % \
( num_datasets_copied, iff( num_datasets_copied == 1, "", "s"), len( target_histories ), iff( len ( target_histories ) == 1, "y", "ies") )
trans.sa_session.refresh( history )
if error_msg != "":
status = ERROR
message = error_msg
lib/galaxy/web/controllers/requests_admin.py
+6 -8
View File
@@ -199,6 +199,11 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitions ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
sample_id = params.get( 'sample_id', None )
try:
sample = trans.sa_session.query( trans.model.Sample ).get( trans.security.decode_id ( sample_id ) )
except:
return invalid_id_redirect( trans, 'requests_admin', sample_id, 'sample' )
if 'operation' in kwd:
operation = kwd[ 'operation' ].lower()
sample_dataset_id = params.get( 'id', None )
@@ -269,11 +274,6 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitions ):
**kwd ) )
# Render the grid view
sample_id = params.get( 'sample_id', None )
try:
sample = trans.sa_session.query( trans.model.Sample ).get( trans.security.decode_id ( sample_id ) )
except:
return invalid_id_redirect( trans, 'requests_admin', sample_id, 'sample' )
request_id = trans.security.encode_id( sample.request.id )
library_id = trans.security.encode_id( sample.library.id )
self.datatx_grid.title = 'Manage "%s" datasets' % sample.name
@@ -493,6 +493,7 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitions ):
return trans.response.send_redirect( web.url_for( controller='requests_admin',
action='select_datasets_to_transfer',
request_id=trans.security.encode_id( request.id ),
external_service_id=trans.security.encode_id( external_service.id ),
status=status,
message=message ) )
def __create_sample_datasets( self, trans, sample, selected_datasets_to_transfer, external_service ):
@@ -624,9 +625,6 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitions ):
or not scp_configs.get( 'user_name', '' ) \
or not scp_configs.get( 'password', '' ):
err_msg += "Error in external service login information. "
# Make sure web API is enabled and API key exists
if not trans.app.config.enable_api:
err_msg += "The 'enable_api = True' setting is not correctly set in the Galaxy config file. "
if not trans.user.api_keys:
err_msg += "Set your API Key in your User Preferences to transfer datasets. "
# Check if library_import_dir is set
lib/galaxy/web/controllers/requests_common.py
+3
View File
@@ -1011,6 +1011,8 @@ class RequestsCommon( BaseUIController, UsesFormDefinitions ):
sample = trans.sa_session.query( trans.model.Sample ).get( trans.security.decode_id( sample_id ) )
except:
return invalid_id_redirect( trans, cntrller, sample_id, 'sample' )
external_service_id = params.get( 'external_service_id', None )
external_service = trans.sa_session.query( trans.model.ExternalService ).get( trans.security.decode_id( external_service_id ) )
# See if a library and folder have been set for this sample.
if is_admin and not sample.library or not sample.folder:
status = 'error'
@@ -1043,6 +1045,7 @@ class RequestsCommon( BaseUIController, UsesFormDefinitions ):
return trans.fill_template( '/requests/common/view_sample_datasets.mako',
cntrller=cntrller,
title=title,
external_service=external_service,
sample=sample,
sample_datasets=sample_datasets,
transfer_status=transfer_status,
lib/galaxy/web/controllers/root.py
+1 -1
View File
@@ -250,7 +250,7 @@ class RootController( BaseUIController, UsesHistory, UsesAnnotations ):
valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
fname = data.name
fname = ''.join(c in valid_chars and c or '_' for c in fname)[0:150]
trans.response.headers["Content-Disposition"] = "attachment; filename=GalaxyHistoryItem-%s-[%s]%s" % (data.hid, fname, toext)
trans.response.headers["Content-Disposition"] = 'attachment; filename="GalaxyHistoryItem-%s-[%s]%s"' % (data.hid, fname, toext)
trans.log_event( "Display dataset id: %s" % str(id) )
try:
return open( data.file_name )
lib/galaxy/web/controllers/tool_runner.py
+2
View File
@@ -58,6 +58,8 @@ class ToolRunner( BaseUIController ):
log.error( "index called with tool id '%s' but no such tool exists", tool_id )
trans.log_event( "Tool id '%s' does not exist" % tool_id )
return "Tool '%s' does not exist, kwd=%s " % (tool_id, kwd)
if tool.require_login and not trans.user:
return trans.response.send_redirect( url_for( controller='user', action='login', cntrller='user', message="You must be logged in to use this tool.", status="info", redirect=url_for( controller='/tool_runner', action='index', tool_id=tool_id, **kwd ) ) )
params = util.Params( kwd, sanitize = False ) #Sanitize parameters when substituting into command line via input wrappers
#do param translation here, used by datasource tools
if tool.input_translator:
lib/galaxy/web/controllers/tracks.py
+57 -37
View File
@@ -486,7 +486,7 @@ class TracksController( BaseUIController, UsesVisualization, UsesHistoryDatasetA
data = None
# TODO: for raw data requests, map dataset type to provider using dict in data_providers.py
if isinstance( dataset.datatype, Gff ):
data = GFFDataProvider( original_dataset=dataset ).get_data( chrom, low, high, **kwargs )
data = RawGFFDataProvider( original_dataset=dataset ).get_data( chrom, low, high, **kwargs )
data[ 'dataset_type' ] = 'interval_index'
data[ 'extra_info' ] = None
elif isinstance( dataset.datatype, Bed ):
@@ -932,6 +932,7 @@ class TracksController( BaseUIController, UsesVisualization, UsesHistoryDatasetA
# Set input datasets for tool. If running on region, extract and use subset
# when possible.
#
location = "%s:%i-%i" % ( chrom, low, high )
for jida in original_job.input_datasets:
# If param set previously by config actions, do nothing.
if jida.name in params_set:
@@ -943,48 +944,67 @@ class TracksController( BaseUIController, UsesVisualization, UsesHistoryDatasetA
elif run_on_region and hasattr( input_dataset.datatype, 'get_track_type' ):
# Dataset is indexed and hence a subset can be extracted and used
# as input.
track_type, data_sources = input_dataset.datatype.get_track_type()
data_source = data_sources[ 'data' ]
converted_dataset = input_dataset.get_converted_dataset( trans, data_source )
deps = input_dataset.get_converted_dataset_deps( trans, data_source )
# Create new HDA for input dataset's subset.
new_dataset = trans.app.model.HistoryDatasetAssociation( extension=input_dataset.ext, \
dbkey=input_dataset.dbkey, \
create_dataset=True, \
sa_session=trans.sa_session,
name="Subset [%s:%i-%i] of data %i" % \
( chrom, low, high, input_dataset.hid ),
visible=False )
target_history.add_dataset( new_dataset )
trans.sa_session.add( new_dataset )
trans.app.security_agent.set_all_dataset_permissions( new_dataset.dataset, hda_permissions )
# Write subset of data to new dataset
data_provider_class = get_data_provider( original_dataset=input_dataset )
data_provider = data_provider_class( original_dataset=input_dataset,
converted_dataset=converted_dataset,
dependencies=deps )
trans.app.object_store.create( new_dataset.dataset )
data_provider.write_data_to_file( chrom, low, high, new_dataset.file_name )
# TODO: (a) size not working; (b) need to set peek.
new_dataset.set_size()
new_dataset.info = "Data subset for trackster"
new_dataset.set_dataset_state( trans.app.model.Dataset.states.OK )
# Set metadata.
if trans.app.config.set_metadata_externally:
trans.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( trans.app.datatypes_registry.set_external_metadata_tool, trans, incoming = { 'input1':new_dataset }, overwrite=False )
# Look for subset.
subset_dataset_association = trans.sa_session.query( trans.app.model.HistoryDatasetAssociationSubset ) \
.filter_by( hda=input_dataset, location=location ) \
.first()
if subset_dataset_association:
# Data subset exists.
subset_dataset = subset_dataset_association.subset
else:
message = 'Attributes updated'
new_dataset.set_meta()
new_dataset.datatype.after_setting_metadata( new_dataset )
# Need to create subset.
track_type, data_sources = input_dataset.datatype.get_track_type()
data_source = data_sources[ 'data' ]
converted_dataset = input_dataset.get_converted_dataset( trans, data_source )
deps = input_dataset.get_converted_dataset_deps( trans, data_source )
# Create new HDA for input dataset's subset.
new_dataset = trans.app.model.HistoryDatasetAssociation( extension=input_dataset.ext, \
dbkey=input_dataset.dbkey, \
create_dataset=True, \
sa_session=trans.sa_session,
name="Subset [%s] of data %i" % \
( location, input_dataset.hid ),
visible=False )
target_history.add_dataset( new_dataset )
trans.sa_session.add( new_dataset )
trans.app.security_agent.set_all_dataset_permissions( new_dataset.dataset, hda_permissions )
# Write subset of data to new dataset
data_provider_class = get_data_provider( original_dataset=input_dataset )
data_provider = data_provider_class( original_dataset=input_dataset,
converted_dataset=converted_dataset,
dependencies=deps )
trans.app.object_store.create( new_dataset.dataset )
data_provider.write_data_to_file( chrom, low, high, new_dataset.file_name )
# TODO: (a) size not working; (b) need to set peek.
new_dataset.set_size()
new_dataset.info = "Data subset for trackster"
new_dataset.set_dataset_state( trans.app.model.Dataset.states.OK )
# Set metadata.
# TODO: set meta internally if dataset is small enough?
if trans.app.config.set_metadata_externally:
trans.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( trans.app.datatypes_registry.set_external_metadata_tool,
trans, incoming = { 'input1':new_dataset },
overwrite=False, job_params={ "source" : "trackster" } )
else:
message = 'Attributes updated'
new_dataset.set_meta()
new_dataset.datatype.after_setting_metadata( new_dataset )
# Add HDA subset association.
subset_association = trans.app.model.HistoryDatasetAssociationSubset( hda=input_dataset, subset=new_dataset, location=location )
trans.sa_session.add( subset_association )
subset_dataset = new_dataset
trans.sa_session.flush()
# Add dataset to tool's parameters.
if not set_param_value( tool_params, jida.name, new_dataset ):
if not set_param_value( tool_params, jida.name, subset_dataset ):
return to_json_string( { "error" : True, "message" : "error setting parameter %s" % jida.name } )
#
lib/galaxy/web/controllers/user.py
+148 -94
View File
@@ -10,8 +10,7 @@ from random import choice
from galaxy.web.form_builder import *
from galaxy.util.json import from_json_string, to_json_string
from galaxy.web.framework.helpers import iff
from galaxy.security.validate_user_input import validate_email, validate_publicname, validate_password
from galaxy.openid.providers import OpenIDProvider
from galaxy.security.validate_user_input import validate_email, validate_publicname, validate_password, transform_publicname
log = logging.getLogger( __name__ )
@@ -30,7 +29,7 @@ class UserOpenIDGrid( grids.Grid ):
default_filter = { "openid" : "All" }
default_sort_key = "-create_time"
columns = [
grids.TextColumn( "OpenID URL", key="openid" ),
grids.TextColumn( "OpenID URL", key="openid", link=( lambda x: dict( action='openid_auth', login_button="Login", openid_url=x.openid if not x.provider else '', openid_provider=x.provider, auto_associate=True ) ) ),
grids.GridColumn( "Created", key="create_time", format=time_ago ),
]
operations = [
@@ -48,32 +47,30 @@ class User( BaseUIController, UsesFormDefinitions ):
return trans.fill_template( '/user/index.mako', cntrller=cntrller, webapp=webapp )
@web.expose
def openid_auth( self, trans, webapp='galaxy', **kwd ):
'''Handles user request to access an OpenID provider'''
if not trans.app.config.enable_openid:
return trans.show_error_message( 'OpenID authentication is not enabled in this instance of Galaxy' )
message = 'Unspecified failure authenticating via OpenID'
status = kwd.get( 'status', 'done' )
openid_url = kwd.get( 'openid_url', '' )
openid_provider = kwd.get( 'openid_provider', '' )
referer = kwd.get( 'referer', trans.request.referer )
if not openid_provider or openid_url:
openid_provider = trans.app.openid_providers.NO_PROVIDER_ID #empty fields cause validation errors
redirect = kwd.get( 'redirect', '' ).strip()
auto_associate = util.string_as_bool( kwd.get( 'auto_associate', False ) )
use_panels = util.string_as_bool( kwd.get( 'use_panels', False ) )
action = 'login'
if auto_associate:
action = 'openid_manage'
if not referer:
referer = url_for( '/' )
consumer = trans.app.openid_manager.get_consumer( trans )
openid_provider_obj = None
if not openid_url and openid_provider and trans.app.openid_providers.get( openid_provider ):
openid_provider_obj = trans.app.openid_providers.get( openid_provider )
elif openid_url:
openid_provider_obj = OpenIDProvider( openid_url, openid_url, openid_url ) #for manually entered links use the link for id, name and url
elif openid_provider:
message = 'Invalid OpenID provider specified: %s' % ( openid_provider )
if openid_url:
openid_provider_obj = trans.app.openid_providers.new_provider_from_identifier( openid_url )
else:
openid_provider_obj = trans.app.openid_providers.get( openid_provider )
if not openid_url and openid_provider == trans.app.openid_providers.NO_PROVIDER_ID:
message = 'An OpenID provider was not specified'
process_url = trans.request.base.rstrip( '/' ) + url_for( controller='user', action='openid_process', referer=referer, auto_associate=auto_associate, openid_provider=openid_provider )
if openid_provider_obj is not None:
elif openid_provider_obj:
if not redirect:
redirect = ' '
process_url = trans.request.base.rstrip( '/' ) + url_for( controller='user', action='openid_process', redirect=redirect, openid_provider=openid_provider, auto_associate=auto_associate ) #None of these values can be empty, or else a verification error will occur
request = None
try:
request = consumer.begin( openid_provider_obj.op_endpoint_url )
@@ -87,84 +84,100 @@ class User( BaseUIController, UsesFormDefinitions ):
redirect_url = request.redirectURL(
trans.request.base, process_url )
trans.app.openid_manager.persist_session( trans, consumer )
trans.response.send_redirect( redirect_url )
return
return trans.response.send_redirect( redirect_url )
else:
form = request.htmlMarkup( trans.request.base, process_url, form_tag_attrs={'id':'openid_message','target':'_top'} )
trans.app.openid_manager.persist_session( trans, consumer )
return form
return trans.response.send_redirect( url_for( controller='user',
action=action,
redirect=redirect,
use_panels=use_panels,
message=message,
status='error' ) )
@web.expose
def openid_process( self, trans, webapp='galaxy', **kwd ):
'''Handle's response from OpenID Providers'''
if not trans.app.config.enable_openid:
return trans.show_error_message( 'OpenID authentication is not enabled in this instance of Galaxy' )
auto_associate = util.string_as_bool( kwd.get( 'auto_associate', False ) )
action = 'login'
if auto_associate:
if trans.user:
action = 'openid_manage'
if trans.app.config.support_url is not None:
contact = '<a href="%s">support</a>' % trans.app.config.support_url
else:
contact = 'support'
message = 'Verification failed for an unknown reason. Please contact support for assistance.'
message = 'Verification failed for an unknown reason. Please contact %s for assistance.' % ( contact )
status = 'error'
consumer = trans.app.openid_manager.get_consumer( trans )
info = consumer.complete( kwd, trans.request.url )
display_identifier = info.getDisplayIdentifier()
redirect_url = kwd.get( 'referer', url_for( '/' ) )
openid_provider = kwd.get( 'openid_provider', '' )
redirect = kwd.get( 'redirect', '' ).strip()
openid_provider = kwd.get( 'openid_provider', None )
if info.status == trans.app.openid_manager.FAILURE and display_identifier:
message = "Login via OpenID failed. The technical reason for this follows, please include this message in your email if you need to %s to resolve this problem: %s" % ( contact, info.message )
return trans.response.send_redirect( url_for( controller='user',
action=action,
use_panels=True,
redirect=redirect,
message=message,
status='error' ) )
elif info.status == trans.app.openid_manager.SUCCESS:
if info.endpoint.canonicalID:
display_identifier = info.endpoint.canonicalID
user_openid = trans.sa_session.query( trans.app.model.UserOpenID ).filter( trans.app.model.UserOpenID.table.c.openid == display_identifier ).first()
openid_provider_obj = trans.app.openid_providers.get( openid_provider )
user_openid = trans.sa_session.query( trans.app.model.UserOpenID ).filter( trans.app.model.UserOpenID.table.c.openid == display_identifier ).first()
if not openid_provider_obj and user_openid and user_openid.provider:
openid_provider_obj = trans.app.openid_providers.get( user_openid.provider )
if not openid_provider_obj:
openid_provider_obj = OpenIDProvider( display_identifier, display_identifier, display_identifier )
openid_provider_obj = trans.app.openid_providers.new_provider_from_identifier( display_identifier )
if not user_openid:
user_openid = trans.app.model.UserOpenID( session=trans.galaxy_session, openid=display_identifier )
elif not user_openid.user and user_openid.session.id != trans.galaxy_session.id:
if not user_openid.user:
user_openid.session = trans.galaxy_session
elif user_openid.user and not auto_associate:
trans.handle_user_login( user_openid.user, webapp )
trans.log_event( "User logged in via OpenID: %s" % display_identifier )
openid_provider_obj.post_authentication( trans, trans.app.openid_manager, info )
trans.response.send_redirect( redirect_url )
return
if auto_associate and trans.user:
# The user is already logged in and requested association from
# the user prefs as opposed to using the OpenID form on the
# login page.
if not user_openid.provider and openid_provider:
user_openid.provider = openid_provider
if trans.user:
if user_openid.user and user_openid.user.id != trans.user.id:
message = "The OpenID <strong>%s</strong> is already associated with another Galaxy account, <strong>%s</strong>. Please disassociate it from that account before attempting to associate it with a new account." % ( display_identifier, user_openid.user.email )
status = "error"
elif user_openid.user and user_openid.user.id == trans.user.id:
message = "The OpenID <strong>%s</strong> is already associated with your Galaxy account, <strong>%s</strong>." % ( display_identifier, trans.user.email )
status = "warning"
else:
user_openid.user_id = trans.user.id
trans.sa_session.add( user_openid )
trans.sa_session.flush()
trans.log_event( "User associated OpenID: %s" % display_identifier )
message = "The OpenID <strong>%s</strong> has been associated with your Galaxy account, <strong>%s</strong>." % ( display_identifier, trans.user.email )
status = "done"
elif not user_openid.user or user_openid.user == trans.user:
if openid_provider_obj.id:
user_openid.provider = openid_provider_obj.id
user_openid.session = trans.galaxy_session
if not openid_provider_obj.never_associate_with_user:
if not auto_associate and ( user_openid.user and user_openid.user.id == trans.user.id ):
message = "The OpenID <strong>%s</strong> is already associated with your Galaxy account, <strong>%s</strong>." % ( display_identifier, trans.user.email )
status = "warning"
else:
message = "The OpenID <strong>%s</strong> has been associated with your Galaxy account, <strong>%s</strong>." % ( display_identifier, trans.user.email )
status = "done"
user_openid.user = trans.user
trans.sa_session.add( user_openid )
trans.sa_session.flush()
trans.log_event( "User associated OpenID: %s" % display_identifier )
else:
message = "The OpenID <strong>%s</strong> cannot be used to log into your Galaxy account, but any post authentication actions have been performed." % ( openid_provider_obj.name )
status ="info"
openid_provider_obj.post_authentication( trans, trans.app.openid_manager, info )
trans.response.send_redirect( url_for( controller='user',
if redirect:
message = '%s<br>Click <a href="%s"><strong>here</strong></a> to return to the page you were previously viewing.' % ( message, redirect )
if redirect and status != "error":
return trans.response.send_redirect( redirect )
return trans.response.send_redirect( url_for( controller='user',
action='openid_manage',
use_panels=True,
redirect=redirect,
message=message,
status=status ) )
return
elif user_openid.user:
trans.handle_user_login( user_openid.user, webapp )
trans.log_event( "User logged in via OpenID: %s" % display_identifier )
openid_provider_obj.post_authentication( trans, trans.app.openid_manager, info )
if not redirect:
redirect = url_for( '/' )
return trans.response.send_redirect( redirect )
trans.sa_session.add( user_openid )
trans.sa_session.flush()
message = "OpenID authentication was successful, but you need to associate your OpenID with a Galaxy account."
@@ -179,10 +192,11 @@ class User( BaseUIController, UsesFormDefinitions ):
email = sreg_resp.get( sreg_email_name, '' )
except AttributeError:
email = ''
trans.response.send_redirect( url_for( controller='user',
#OpenID success, but user not logged in, and not previously associated
return trans.response.send_redirect( url_for( controller='user',
action='openid_associate',
openid_provider=openid_provider,
use_panels=True,
redirect=redirect,
username=username,
email=email,
message=message,
@@ -198,10 +212,12 @@ class User( BaseUIController, UsesFormDefinitions ):
return trans.response.send_redirect( url_for( controller='user',
action=action,
use_panels=True,
redirect=redirect,
message=message,
status=status ) )
@web.expose
def openid_associate( self, trans, cntrller='user', webapp='galaxy', **kwd ):
'''Associates a user with an OpenID log in'''
if not trans.app.config.enable_openid:
return trans.show_error_message( 'OpenID authentication is not enabled in this instance of Galaxy' )
use_panels = util.string_as_bool( kwd.get( 'use_panels', False ) )
@@ -209,9 +225,7 @@ class User( BaseUIController, UsesFormDefinitions ):
status = kwd.get( 'status', 'done' )
email = kwd.get( 'email', '' )
username = kwd.get( 'username', '' )
referer = kwd.get( 'referer', trans.request.referer )
openid_provider = kwd.get( 'openid_provider', '' )
openid_provider_obj = trans.app.openid_providers.get( openid_provider )
redirect = kwd.get( 'redirect', '' ).strip()
params = util.Params( kwd )
is_admin = cntrller == 'admin' and trans.user_is_admin()
openids = trans.galaxy_session.openids
@@ -223,18 +237,38 @@ class User( BaseUIController, UsesFormDefinitions ):
if kwd.get( 'login_button', False ):
message, status, user, success = self.__validate_login( trans, webapp, **kwd )
if success:
openid_objs = []
for openid in openids:
openid.user = user
trans.sa_session.add( openid )
openid_provider_obj = trans.app.openid_providers.get( openid.provider )
if not openid_provider_obj or not openid_provider_obj.never_associate_with_user:
openid.user = user
trans.sa_session.add( openid )
trans.log_event( "User associated OpenID: %s" % openid.openid )
if openid_provider_obj and openid_provider_obj.has_post_authentication_actions():
openid_objs.append( openid_provider_obj )
trans.sa_session.flush()
for openid in openids:
trans.log_event( "User associated OpenID: %s" % openid.openid )
redirect_url = referer
if not redirect_url:
redirect_url = url_for( '/' )
if openid_provider_obj:
return trans.response.send_redirect( url_for( controller='user', action='openid_auth', openid_provider=openid_provider, referer=redirect_url ) )
return trans.response.send_redirect( redirect_url )
if len( openid_objs ) == 1:
return trans.response.send_redirect( url_for( controller='user', action='openid_auth', openid_provider=openid_objs[0].id, redirect=redirect, auto_associate=True ) )
elif openid_objs:
message = 'You have authenticated with several OpenID providers, please click the following links to execute the post authentication actions. '
message = "%s<br/><ul>" % ( message )
for openid in openid_objs:
message = '%s<li><a href="%s" target="_blank">%s</a></li>' % ( message, url_for( controller='user', action='openid_auth', openid_provider=openid.id, redirect=redirect, auto_associate=True ), openid.name )
message = "%s</ul>" % ( message )
return trans.response.send_redirect( url_for( controller='user',
action='openid_manage',
use_panels=use_panels,
redirect=redirect,
message=message,
status='info' ) )
if redirect:
return trans.response.send_redirect( redirect )
return trans.response.send_redirect( url_for( controller='user',
action='openid_manage',
use_panels=use_panels,
redirect=redirect,
message=message,
status='info' ) )
if kwd.get( 'create_user_button', False ):
password = kwd.get( 'password', '' )
confirm = kwd.get( 'confirm', '' )
@@ -253,21 +287,40 @@ class User( BaseUIController, UsesFormDefinitions ):
subscribe_checked,
**kwd )
if success:
trans.handle_user_login( user, webapp )
trans.log_event( "User created a new account" )
trans.log_event( "User logged in" )
openid_objs = []
for openid in openids:
openid.user = user
trans.sa_session.add( openid )
openid_provider_obj = trans.app.openid_providers.get( openid.provider )
if not openid_provider_obj:
openid_provider_obj = trans.app.openid_providers.new_provider_from_identifier( openid.identifier )
if not openid_provider_obj.never_associate_with_user:
openid.user = user
trans.sa_session.add( openid )
trans.log_event( "User associated OpenID: %s" % openid.openid )
if openid_provider_obj.has_post_authentication_actions():
openid_objs.append( openid_provider_obj )
trans.sa_session.flush()
for openid in openids:
trans.log_event( "User associated OpenID: %s" % openid.openid )
redirect_url = referer
if not redirect_url:
redirect_url = url_for( '/' )
if openid_provider_obj:
return trans.response.send_redirect( url_for( controller='user', action='openid_auth', openid_provider=openid_provider, referer=redirect_url ) )
return trans.response.send_redirect( redirect_url )
if len( openid_objs ) == 1:
return trans.response.send_redirect( url_for( controller='user', action='openid_auth', openid_provider=openid_objs[0].id, redirect=redirect, auto_associate=True ) )
elif openid_objs:
message = 'You have authenticated with several OpenID providers, please click the following links to execute the post authentication actions. '
message = "%s<br/><ul>" % ( message )
for openid in openid_objs:
message = '%s<li><a href="%s" target="_blank">%s</a></li>' % ( message, url_for( controller='user', action='openid_auth', openid_provider=openid.id, redirect=redirect, auto_associate=True ), openid.name )
message = "%s</ul>" % ( message )
return trans.response.send_redirect( url_for( controller='user',
action='openid_manage',
use_panels=True,
redirect=redirect,
message=message,
status='info' ) )
if redirect:
return trans.response.send_redirect( redirect )
return trans.response.send_redirect( url_for( controller='user',
action='openid_manage',
use_panels=use_panels,
redirect=redirect,
message=message,
status='info' ) )
else:
message = error
status = 'error'
@@ -288,11 +341,10 @@ class User( BaseUIController, UsesFormDefinitions ):
email=email,
password='',
confirm='',
username=username,
username=transform_publicname( trans, username ),
header='',
use_panels=use_panels,
redirect_url='',
referer='',
redirect=redirect,
refresh_frames=[],
message=message,
status=status,
@@ -301,11 +353,11 @@ class User( BaseUIController, UsesFormDefinitions ):
user_type_fd_id_select_field=user_type_fd_id_select_field,
user_type_form_definition=user_type_form_definition,
widgets=widgets,
openids=openids,
openid_provider=openid_provider )
openids=openids )
@web.expose
@web.require_login( 'manage OpenIDs' )
def openid_disassociate( self, trans, webapp='galaxy', **kwd ):
'''Disassociates a user with an OpenID'''
if not trans.app.config.enable_openid:
return trans.show_error_message( 'OpenID authentication is not enabled in this instance of Galaxy' )
params = util.Params( kwd )
@@ -338,7 +390,7 @@ class User( BaseUIController, UsesFormDefinitions ):
trans.log_event( "User disassociated OpenID: %s" % deleted_url )
message = '%s OpenIDs were disassociated from your Galaxy account.' % len( ids )
status = 'done'
trans.response.send_redirect( url_for( controller='user',
return trans.response.send_redirect( url_for( controller='user',
action='openid_manage',
use_panels=use_panels,
message=message,
@@ -346,29 +398,31 @@ class User( BaseUIController, UsesFormDefinitions ):
@web.expose
@web.require_login( 'manage OpenIDs' )
def openid_manage( self, trans, webapp='galaxy', **kwd ):
'''Manage OpenIDs for user'''
if not trans.app.config.enable_openid:
return trans.show_error_message( 'OpenID authentication is not enabled in this instance of Galaxy' )
use_panels = kwd.get( 'use_panels', False )
if 'operation' in kwd:
operation = kwd['operation'].lower()
if operation == "delete":
trans.response.send_redirect( url_for( controller='user',
return trans.response.send_redirect( url_for( controller='user',
action='openid_disassociate',
use_panels=use_panels,
id=kwd['id'] ) )
kwd['referer'] = url_for( controller='user', action='openid_manage', use_panels=True )
kwd['redirect'] = kwd.get( 'redirect', url_for( controller='user', action='openid_manage', use_panels=True ) ).strip()
kwd['openid_providers'] = trans.app.openid_providers
return self.user_openid_grid( trans, **kwd )
@web.expose
def login( self, trans, webapp='galaxy', redirect_url='', refresh_frames=[], **kwd ):
referer = kwd.get( 'referer', trans.request.referer )
'''Handle Galaxy Log in'''
redirect = kwd.get( 'redirect', trans.request.referer ).strip()
use_panels = util.string_as_bool( kwd.get( 'use_panels', False ) )
message = kwd.get( 'message', '' )
status = kwd.get( 'status', 'done' )
header = ''
user = None
email = kwd.get( 'email', '' )
openid_provider = kwd.get( 'openid_provider', '' )
if kwd.get( 'login_button', False ):
if webapp == 'galaxy' and not refresh_frames:
if trans.app.config.require_login:
@@ -376,8 +430,8 @@ class User( BaseUIController, UsesFormDefinitions ):
else:
refresh_frames = [ 'masthead', 'history' ]
message, status, user, success = self.__validate_login( trans, webapp, **kwd )
if success and referer and not referer.startswith( trans.request.base + url_for( controller='user', action='logout' ) ):
redirect_url = referer
if success and redirect and not redirect.startswith( trans.request.base + url_for( controller='user', action='logout' ) ):
redirect_url = redirect
elif success:
redirect_url = url_for( '/' )
if not user and trans.app.config.require_login:
@@ -399,7 +453,7 @@ class User( BaseUIController, UsesFormDefinitions ):
header=header,
use_panels=use_panels,
redirect_url=redirect_url,
referer=referer,
redirect=redirect,
refresh_frames=refresh_frames,
message=message,
status=status,
@@ -410,7 +464,7 @@ class User( BaseUIController, UsesFormDefinitions ):
status = kwd.get( 'status', 'done' )
email = kwd.get( 'email', '' )
password = kwd.get( 'password', '' )
referer = kwd.get( 'referer', trans.request.referer )
redirect = kwd.get( 'redirect', trans.request.referer ).strip()
success = False
user = trans.sa_session.query( trans.app.model.User ).filter( trans.app.model.User.table.c.email==email ).first()
if not user:
@@ -430,7 +484,7 @@ class User( BaseUIController, UsesFormDefinitions ):
if webapp == 'galaxy':
trans.log_event( "User logged in" )
message = 'You are now logged in as %s.<br>You can <a target="_top" href="%s">go back to the page you were visiting</a> or <a target="_top" href="%s">go to the home page</a>.' % \
( user.email, referer, url_for( '/' ) )
( user.email, redirect, url_for( '/' ) )
if trans.app.config.require_login:
message += ' <a target="_top" href="%s">Click here</a> to continue to the home page.' % web.url_for( '/static/welcome.html' )
success = True
@@ -470,7 +524,7 @@ class User( BaseUIController, UsesFormDefinitions ):
username = util.restore_text( params.get( 'username', '' ) )
subscribe = params.get( 'subscribe', '' )
subscribe_checked = CheckboxField.is_checked( subscribe )
referer = kwd.get( 'referer', trans.request.referer )
redirect = kwd.get( 'redirect', trans.request.referer ).strip()
is_admin = cntrller == 'admin' and trans.user_is_admin
if not trans.app.config.allow_user_creation and not trans.user_is_admin():
message = 'User registration is disabled. Please contact your Galaxy administrator for an account.'
@@ -527,14 +581,14 @@ class User( BaseUIController, UsesFormDefinitions ):
email=email,
password=password,
confirm=confirm,
username=username,
username=transform_publicname( trans, username ),
subscribe_checked=subscribe_checked,
user_type_fd_id_select_field=user_type_fd_id_select_field,
user_type_form_definition=user_type_form_definition,
widgets=widgets,
webapp=webapp,
use_panels=use_panels,
referer=referer,
redirect=redirect,
redirect_url=redirect_url,
refresh_frames=refresh_frames,
message=message,
lib/galaxy/web/controllers/workflow.py
+4 -11
View File
@@ -1119,7 +1119,7 @@ class WorkflowController( BaseUIController, Sharable, UsesStoredWorkflow, UsesAn
valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
sname = stored.name
sname = ''.join(c in valid_chars and c or '_' for c in sname)[0:150]
trans.response.headers["Content-Disposition"] = "attachment; filename=Galaxy-Workflow-%s.ga" % ( sname )
trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy-Workflow-%s.ga"' % ( sname )
trans.response.set_content_type( 'application/galaxy-archive' )
return stored_dict
@web.expose
@@ -1160,7 +1160,7 @@ class WorkflowController( BaseUIController, Sharable, UsesStoredWorkflow, UsesAn
import_button = True
if tool_shed_url and not import_button:
# Use urllib (send another request to the tool shed) to retrieve the workflow.
workflow_url = '%s/workflow/import_workflow?repository_metadata_id=%s&workflow_name=%s&webapp=%s&open_for_url=true' % \
workflow_url = '%s/workflow/import_workflow?repository_metadata_id=%s&workflow_name=%s&webapp=%s&open_for_url=true&no_reset=true' % \
( tool_shed_url, repository_metadata_id, tool_shed_encode( workflow_name ), webapp )
response = urllib2.urlopen( workflow_url )
workflow_text = response.read()
@@ -1354,11 +1354,6 @@ class WorkflowController( BaseUIController, Sharable, UsesStoredWorkflow, UsesAn
tool = trans.app.toolbox.get_tool( job.tool_id )
param_values = job.get_param_values( trans.app )
associations = cleanup_param_values( tool.inputs, param_values )
# Doing it this way breaks dynamic parameters, backed out temporarily.
# def extract_callback( input, value, prefixed_name, prefixed_label ):
# if isinstance( value, UnvalidatedValue ):
# return str( value )
# visit_input_values( tool.inputs, param_values, extract_callback )
step = model.WorkflowStep()
step.type = 'tool'
step.tool_id = job.tool_id
@@ -2059,13 +2054,11 @@ def cleanup_param_values( inputs, values ):
group_values = values[key]
for i, rep_values in enumerate( group_values ):
rep_index = rep_values['__index__']
prefix = "%s_%d|" % ( key, rep_index )
cleanup( prefix, input.inputs, group_values[i] )
cleanup( "%s%s_%d|" % (prefix, key, rep_index ), input.inputs, group_values[i] )
elif isinstance( input, Conditional ):
group_values = values[input.name]
current_case = group_values['__current_case__']
prefix = "%s|" % ( key )
cleanup( prefix, input.cases[current_case].inputs, group_values )
cleanup( "%s%s|" % ( prefix, key ), input.cases[current_case].inputs, group_values )
cleanup( "", inputs, values )
return associations
lib/galaxy/web/form_builder.py
+86 -63
View File
@@ -6,6 +6,7 @@ import logging, sys, os, time
from cgi import escape
from galaxy.util import restore_text, relpath, nice_size
from galaxy.web import url_for
from binascii import hexlify
log = logging.getLogger(__name__)
@@ -143,7 +144,7 @@ class FileField(BaseField):
def get_html( self, prefix="" ):
value_text = ""
if self.value:
value_text = ' value="%s"' % self.value
value_text = ' value="%s"' % escape( str( self.value ), quote=True )
ajax_text = ""
if self.ajax:
ajax_text = ' galaxy-ajax-upload="true"'
@@ -278,7 +279,7 @@ class SelectField(BaseField):
if self.refresh_on_change:
self.refresh_on_change_text = ' refresh_on_change="true"'
if self.refresh_on_change_values:
self.refresh_on_change_text = '%s refresh_on_change_values="%s"' % ( self.refresh_on_change_text, ",".join( self.refresh_on_change_values ) )
self.refresh_on_change_text = '%s refresh_on_change_values="%s"' % ( self.refresh_on_change_text, escape( ",".join( self.refresh_on_change_values ), quote=True ) )
else:
self.refresh_on_change_text = ''
def add_option( self, text, value, selected = False ):
@@ -305,7 +306,7 @@ class SelectField(BaseField):
if selected:
selected_text = " checked='checked'"
rval.append( '<div%s><input type="checkbox" name="%s%s" value="%s" id="%s"%s%s><label class="inline" for="%s">%s</label></div>' % \
( style, prefix, self.name, escaped_value, uniq_id, selected_text, self.get_disabled_str( disabled ), uniq_id, text ) )
( style, prefix, self.name, escaped_value, uniq_id, selected_text, self.get_disabled_str( disabled ), uniq_id, escape( str( text ), quote=True ) ) )
ctr += 1
return "\n".join( rval )
def get_html_radio( self, prefix="", disabled=False ):
@@ -350,7 +351,7 @@ class SelectField(BaseField):
last_selected_value = value
else:
selected_text = ""
rval.append( '<option value="%s"%s>%s</option>' % ( escape( str( value ), quote=True ), selected_text, text ) )
rval.append( '<option value="%s"%s>%s</option>' % ( escape( str( value ), quote=True ), selected_text, escape( str( text ), quote=True ) ) )
if last_selected_value:
last_selected_value = ' last_selected_value="%s"' % escape( str( last_selected_value ), quote=True )
rval.insert( 0, '<select name="%s%s"%s%s%s%s%s>' % \
@@ -391,48 +392,66 @@ class DrillDownField( BaseField ):
>>> t = DrillDownField( "foo", multiple=True, display="checkbox", options=[{'name': 'Heading 1', 'value': 'heading1', 'options': [{'name': 'Option 1', 'value': 'option1', 'options': []}, {'name': 'Option 2', 'value': 'option2', 'options': []}, {'name': 'Heading 1', 'value': 'heading1', 'options': [{'name': 'Option 3', 'value': 'option3', 'options': []}, {'name': 'Option 4', 'value': 'option4', 'options': []}]}]}, {'name': 'Option 5', 'value': 'option5', 'options': []}] )
>>> print t.get_html()
<div><ul class="toolParameterExpandableCollapsable">
<li><span class="toolParameterExpandableCollapsable">[+]</span><input type="checkbox" name="foo" value="heading1"">Heading 1
<ul class="toolParameterExpandableCollapsable" default_state="collapsed">
<li><input type="checkbox" name="foo" value="option1"">Option 1
</li>
<li><input type="checkbox" name="foo" value="option2"">Option 2
</li>
<li><span class="toolParameterExpandableCollapsable">[+]</span><input type="checkbox" name="foo" value="heading1"">Heading 1
<ul class="toolParameterExpandableCollapsable" default_state="collapsed">
<li><input type="checkbox" name="foo" value="option3"">Option 3
</li>
<li><input type="checkbox" name="foo" value="option4"">Option 4
</li>
</ul>
</li>
</ul>
</li>
<li><input type="checkbox" name="foo" value="option5"">Option 5
</li>
</ul></div>
<div class="form-row drilldown-container" id="drilldown--666f6f">
<div class="form-row-input">
<span class="form-toggle icon-button toggle-expand" id="drilldown--666f6f-68656164696e6731-click"></span>
<input type="checkbox" name="foo" value="heading1" >Heading 1
<div class="form-row" id="drilldown--666f6f-68656164696e6731-container" style="float: left; margin-left: 1em;">
<div class="form-row-input">
<input type="checkbox" name="foo" value="option1" >Option 1
</div>
<div class="form-row-input">
<input type="checkbox" name="foo" value="option2" >Option 2
</div>
<div class="form-row-input">
<span class="form-toggle icon-button toggle-expand" id="drilldown--666f6f-68656164696e6731-68656164696e6731-click"></span>
<input type="checkbox" name="foo" value="heading1" >Heading 1
<div class="form-row" id="drilldown--666f6f-68656164696e6731-68656164696e6731-container" style="float: left; margin-left: 1em;">
<div class="form-row-input">
<input type="checkbox" name="foo" value="option3" >Option 3
</div>
<div class="form-row-input">
<input type="checkbox" name="foo" value="option4" >Option 4
</div>
</div>
</div>
</div>
</div>
<div class="form-row-input">
<input type="checkbox" name="foo" value="option5" >Option 5
</div>
</div>
>>> t = DrillDownField( "foo", multiple=False, display="radio", options=[{'name': 'Heading 1', 'value': 'heading1', 'options': [{'name': 'Option 1', 'value': 'option1', 'options': []}, {'name': 'Option 2', 'value': 'option2', 'options': []}, {'name': 'Heading 1', 'value': 'heading1', 'options': [{'name': 'Option 3', 'value': 'option3', 'options': []}, {'name': 'Option 4', 'value': 'option4', 'options': []}]}]}, {'name': 'Option 5', 'value': 'option5', 'options': []}] )
>>> print t.get_html()
<div><ul class="toolParameterExpandableCollapsable">
<li><span class="toolParameterExpandableCollapsable">[+]</span><input type="radio" name="foo" value="heading1"">Heading 1
<ul class="toolParameterExpandableCollapsable" default_state="collapsed">
<li><input type="radio" name="foo" value="option1"">Option 1
</li>
<li><input type="radio" name="foo" value="option2"">Option 2
</li>
<li><span class="toolParameterExpandableCollapsable">[+]</span><input type="radio" name="foo" value="heading1"">Heading 1
<ul class="toolParameterExpandableCollapsable" default_state="collapsed">
<li><input type="radio" name="foo" value="option3"">Option 3
</li>
<li><input type="radio" name="foo" value="option4"">Option 4
</li>
</ul>
</li>
</ul>
</li>
<li><input type="radio" name="foo" value="option5"">Option 5
</li>
</ul></div>
<div class="form-row drilldown-container" id="drilldown--666f6f">
<div class="form-row-input">
<span class="form-toggle icon-button toggle-expand" id="drilldown--666f6f-68656164696e6731-click"></span>
<input type="radio" name="foo" value="heading1" >Heading 1
<div class="form-row" id="drilldown--666f6f-68656164696e6731-container" style="float: left; margin-left: 1em;">
<div class="form-row-input">
<input type="radio" name="foo" value="option1" >Option 1
</div>
<div class="form-row-input">
<input type="radio" name="foo" value="option2" >Option 2
</div>
<div class="form-row-input">
<span class="form-toggle icon-button toggle-expand" id="drilldown--666f6f-68656164696e6731-68656164696e6731-click"></span>
<input type="radio" name="foo" value="heading1" >Heading 1
<div class="form-row" id="drilldown--666f6f-68656164696e6731-68656164696e6731-container" style="float: left; margin-left: 1em;">
<div class="form-row-input">
<input type="radio" name="foo" value="option3" >Option 3
</div>
<div class="form-row-input">
<input type="radio" name="foo" value="option4" >Option 4
</div>
</div>
</div>
</div>
</div>
<div class="form-row-input">
<input type="radio" name="foo" value="option5" >Option 5
</div>
</div>
"""
def __init__( self, name, multiple=None, display=None, refresh_on_change=False, options = [], value = [], refresh_on_change_values = [] ):
self.name = name
@@ -440,7 +459,7 @@ class DrillDownField( BaseField ):
self.options = options
if value and not isinstance( value, list ):
value = [ value ]
else:
elif not value:
value = []
self.value = value
if display == "checkbox":
@@ -466,30 +485,34 @@ class DrillDownField( BaseField ):
if option['options']:
new_parents = list( parent_options ) + [ option['value'] ]
find_expanded_options( expanded_options, option['options'], new_parents )
def recurse_options( html, options, expanded_options = [] ):
def recurse_options( html, options, base_id, expanded_options = [] ):
for option in options:
escaped_option_value = escape( str( option['value'] ), quote=True )
selected = ( option['value'] in self.value )
if selected: selected = ' checked'
else: selected = ''
if option['options']:
default_state = 'collapsed'
default_icon = '[+]'
if option['value'] in expanded_options:
default_state = 'expanded'
default_icon = '[-]'
html.append( '<li><span class="toolParameterExpandableCollapsable">%s</span><input type="%s" name="%s%s" value="%s"%s">%s' % ( default_icon, self.display, prefix, self.name, escape(str(option['value']), quote=True), selected, option['name']) )
html.append( '<ul class="toolParameterExpandableCollapsable" default_state="%s">' % default_state )
recurse_options( html, option['options'], expanded_options )
html.append( '</ul>')
if selected:
selected = ' checked'
else:
html.append( '<li><input type="%s" name="%s%s" value="%s"%s">%s' % ( self.display, prefix, self.name, escape(str(option['value']), quote=True), selected, option['name']) )
html.append( '</li>' )
selected = ''
span_class = 'form-toggle icon-button toggle'
if option['value'] not in expanded_options:
span_class = "%s-expand" % ( span_class )
html.append( '<div class="form-row-input">')
drilldown_group_id = "%s-%s" % ( base_id, hexlify( option['value'] ) )
if option['options']:
html.append( '<span class="%s" id="%s-click"></span>' % ( span_class, drilldown_group_id ) )
html.append( '<input type="%s" name="%s%s" value="%s" %s>%s' % ( self.display, prefix, self.name, escaped_option_value, selected, option['name']) )
if option['options']:
html.append( '<div class="form-row" id="%s-container" style="float: left; margin-left: 1em;">' % ( drilldown_group_id ) )
recurse_options( html, option['options'], drilldown_group_id, expanded_options )
html.append( '</div>')
html.append( '</div>')
drilldown_id = "drilldown-%s-%s" % ( hexlify( prefix ), hexlify( self.name ) )
rval = []
rval.append( '<div><ul class="toolParameterExpandableCollapsable">' )
rval.append( '<div class="form-row drilldown-container" id="%s">' % ( drilldown_id ) )
expanded_options = []
find_expanded_options( expanded_options, self.options )
recurse_options( rval, self.options, expanded_options )
rval.append( '</ul></div>' )
recurse_options( rval, self.options, drilldown_id, expanded_options )
rval.append( '</div>' )
return '\n'.join( rval )
class AddressField(BaseField):
lib/galaxy/web/framework/__init__.py
+115 -30
View File
@@ -9,6 +9,7 @@ pkg_resources.require( "Cheetah" )
from Cheetah.Template import Template
import base
import pickle
from functools import wraps
from galaxy import util
from galaxy.exceptions import MessageException
from galaxy.util.json import to_json_string, from_json_string
@@ -62,6 +63,7 @@ def expose( func ):
return func
def json( func ):
@wraps(func)
def decorator( self, trans, *args, **kwargs ):
trans.response.set_content_type( "text/javascript" )
return simplejson.dumps( func( self, trans, *args, **kwargs ) )
@@ -71,6 +73,7 @@ def json( func ):
return decorator
def json_pretty( func ):
@wraps(func)
def decorator( self, trans, *args, **kwargs ):
trans.response.set_content_type( "text/javascript" )
return simplejson.dumps( func( self, trans, *args, **kwargs ), indent=4, sort_keys=True )
@@ -81,37 +84,46 @@ def json_pretty( func ):
def require_login( verb="perform this action", use_panels=False, webapp='galaxy' ):
def argcatcher( func ):
@wraps(func)
def decorator( self, trans, *args, **kwargs ):
if trans.get_user():
return func( self, trans, *args, **kwargs )
else:
return trans.show_error_message(
'You must be <a target="galaxy_main" href="%s">logged in</a> to %s.'
% ( url_for( controller='user', action='login', webapp=webapp ), verb ), use_panels=use_panels )
% ( url_for( controller='user', action='login', webapp=webapp ), verb ), use_panels=use_panels )
return decorator
return argcatcher
def expose_api( func ):
@wraps(func)
def decorator( self, trans, *args, **kwargs ):
def error( environ, start_response ):
start_response( error_status, [('Content-type', 'text/plain')] )
return error_message
error_status = '403 Forbidden'
if 'key' not in kwargs:
error_message = 'No API key provided with request, please consult the API documentation.'
return error
try:
provided_key = trans.sa_session.query( trans.app.model.APIKeys ).filter( trans.app.model.APIKeys.table.c.key == kwargs['key'] ).one()
except NoResultFound:
error_message = 'Provided API key is not valid.'
return error
if provided_key.user.deleted:
error_message = 'User account is deactivated, please contact an administrator.'
return error
newest_key = provided_key.user.api_keys[0]
if newest_key.key != provided_key.key:
error_message = 'Provided API key has expired.'
return error
## If there is a user, we've authenticated a session.
if not trans.user and isinstance(trans.galaxy_session, Bunch):
# If trans.user is already set, don't check for a key.
# This happens when we're authenticating using session instead of an API key.
# The Bunch clause is used to prevent the case where there's no user, but there is a real session.
# DBTODO: This needs to be fixed when merging transaction types.
if 'key' not in kwargs:
error_message = 'No API key provided with request, please consult the API documentation.'
return error
try:
provided_key = trans.sa_session.query( trans.app.model.APIKeys ).filter( trans.app.model.APIKeys.table.c.key == kwargs['key'] ).one()
except NoResultFound:
error_message = 'Provided API key is not valid.'
return error
if provided_key.user.deleted:
error_message = 'User account is deactivated, please contact an administrator.'
return error
newest_key = provided_key.user.api_keys[0]
if newest_key.key != provided_key.key:
error_message = 'Provided API key has expired.'
return error
trans.set_user( provided_key.user )
if trans.request.body:
try:
payload = util.recursively_stringify_dictionary_keys( simplejson.loads( trans.request.body ) )
@@ -121,7 +133,6 @@ def expose_api( func ):
error_message = 'Your request did not appear to be valid JSON, please consult the API documentation'
return error
trans.response.set_content_type( "application/json" )
trans.set_user( provided_key.user )
# Perform api_run_as processing, possibly changing identity
if 'run_as' in kwargs:
if not trans.user_can_do_run_as():
@@ -139,7 +150,6 @@ def expose_api( func ):
except:
trans.response.status = 400
return "That user does not exist."
try:
if trans.debug:
return simplejson.dumps( func( self, trans, *args, **kwargs ), indent=4, sort_keys=True )
@@ -156,6 +166,7 @@ def expose_api( func ):
return decorator
def require_admin( func ):
@wraps(func)
def decorator( self, trans, *args, **kwargs ):
if not trans.user_is_admin():
msg = "You must be an administrator to access this feature."
@@ -202,10 +213,10 @@ class WebApplication( base.WebApplication ):
return base.WebApplication.make_body_iterable( self, trans, body )
def transaction_chooser( self, environ, galaxy_app, session_cookie ):
if 'is_api_request' in environ:
return GalaxyWebAPITransaction( environ, galaxy_app, self )
return GalaxyWebAPITransaction( environ, galaxy_app, self, session_cookie )
else:
return GalaxyWebUITransaction( environ, galaxy_app, self, session_cookie )
class GalaxyWebTransaction( base.DefaultWebTransaction ):
"""
Encapsulates web transaction specific state for the Galaxy application
@@ -262,7 +273,7 @@ class GalaxyWebTransaction( base.DefaultWebTransaction ):
except:
action.user = None
try:
action.session_id = self.galaxy_session.id
action.session_id = self.galaxy_session.id
except:
action.session_id = None
self.sa_session.add( action )
@@ -317,7 +328,7 @@ class GalaxyWebTransaction( base.DefaultWebTransaction ):
tstamp = time.localtime ( time.time() + 3600 * 24 * age )
self.response.cookies[name]['expires'] = time.strftime( '%a, %d-%b-%Y %H:%M:%S GMT', tstamp )
self.response.cookies[name]['version'] = version
def _ensure_valid_session( self, session_cookie ):
def _ensure_valid_session( self, session_cookie, create=True):
"""
Ensure that a valid Galaxy session exists and is available as
trans.session (part of initialization)
@@ -822,7 +833,7 @@ class FormBuilder( object ):
def add_select( self, name, label, value=None, options=[], error=None, help=None, use_label=True ):
self.inputs.append( SelectInput( name, label, value=value, options=options, error=error, help=help, use_label=use_label ) )
return self
class FormInput( object ):
"""
Simple class describing a form input element
@@ -837,15 +848,89 @@ class FormInput( object ):
self.use_label = use_label
class GalaxyWebAPITransaction( GalaxyWebTransaction ):
def __init__( self, environ, app, webapp ):
"""
TODO: Unify this with WebUITransaction, since we allow session auth now.
Enable functionality of 'create' parameter in parent _ensure_valid_session
"""
def __init__( self, environ, app, webapp, session_cookie):
GalaxyWebTransaction.__init__( self, environ, app, webapp )
self.__user = None
self._ensure_valid_session( None )
self._ensure_valid_session( session_cookie )
def _ensure_valid_session( self, session_cookie ):
self.galaxy_session = Bunch()
self.galaxy_session.history = self.galaxy_session.current_history = Bunch()
self.galaxy_session.history.genome_build = None
self.galaxy_session.is_api = True
#Check to see if there is an existing session. Never create a new one.
# Try to load an existing session
secure_id = self.get_cookie( name=session_cookie )
galaxy_session = None
prev_galaxy_session = None
user_for_new_session = None
invalidate_existing_session = False
# Track whether the session has changed so we can avoid calling flush
# in the most common case (session exists and is valid).
galaxy_session_requires_flush = False
if secure_id:
# Decode the cookie value to get the session_key
session_key = self.security.decode_guid( secure_id )
try:
# Make sure we have a valid UTF-8 string
session_key = session_key.encode( 'utf8' )
except UnicodeDecodeError:
# We'll end up creating a new galaxy_session
session_key = None
if session_key:
# Retrieve the galaxy_session id via the unique session_key
galaxy_session = self.sa_session.query( self.app.model.GalaxySession ) \
.filter( and_( self.app.model.GalaxySession.table.c.session_key==session_key,
self.app.model.GalaxySession.table.c.is_valid==True ) ) \
.first()
if galaxy_session:
# If remote user is in use it can invalidate the session, so we need to to check some things now.
if self.app.config.use_remote_user:
assert "HTTP_REMOTE_USER" in self.environ, \
"use_remote_user is set but no HTTP_REMOTE_USER variable"
remote_user_email = self.environ[ 'HTTP_REMOTE_USER' ]
# An existing session, make sure correct association exists
if galaxy_session.user is None:
# No user, associate
galaxy_session.user = self.get_or_create_remote_user( remote_user_email )
galaxy_session_requires_flush = True
elif galaxy_session.user.email != remote_user_email:
# Session exists but is not associated with the correct remote user
log.warning( "User logged in as '%s' externally, but has a cookie as '%s' invalidating session",
remote_user_email, galaxy_session.user.email )
galaxy_session = None
else:
if galaxy_session.user and galaxy_session.user.external:
# Remote user support is not enabled, but there is an existing
# session with an external user, invalidate
invalidate_existing_session = True
log.warning( "User '%s' is an external user with an existing session, invalidating session since external auth is disabled",
galaxy_session.user.email )
galaxy_session = None
elif galaxy_session.user is not None and galaxy_session.user.deleted:
invalidate_existing_session = True
log.warning( "User '%s' is marked deleted, invalidating session" % galaxy_session.user.email )
galaxy_session = None
# No relevant cookies, or couldn't find, or invalid, so create a new session
if galaxy_session:
self.galaxy_session = galaxy_session
self.user = galaxy_session.user
# Do we need to flush the session?
if galaxy_session_requires_flush:
self.sa_session.add( galaxy_session )
# FIXME: If prev_session is a proper relation this would not
# be needed.
if prev_galaxy_session:
self.sa_session.add( prev_galaxy_session )
self.sa_session.flush()
# If the old session was invalid, get a new history with our new session
if not galaxy_session:
#Failed to find a session. Set up fake stuff for API transaction
self.user = None
self.galaxy_session = Bunch()
self.galaxy_session.history = self.galaxy_session.current_history = Bunch()
self.galaxy_session.history.genome_build = None
self.galaxy_session.is_api = True
def get_user( self ):
"""Return the current user (the expose_api decorator ensures that it is set)."""
return self.__user
lib/galaxy/web/framework/helpers/__init__.py
+17 -3
View File
@@ -45,14 +45,28 @@ def css( *args ):
"""
return "\n".join( [ stylesheet_link_tag( "/static/style/" + name + ".css?v=%s" % server_starttime ) for name in args ] )
def js( *args ):
def js_helper( prefix, *args ):
"""
Take a list of javascript names (no extension) and return appropriate
Take a prefix and list of javascript names and return appropriate
string of script tags.
Cache-bust with time that server started running on
"""
return "\n".join( [ javascript_include_tag( "/static/scripts/" + name + ".js?v=%s" % server_starttime ) for name in args ] )
return "\n".join( [ javascript_include_tag( prefix + name + ".js?v=%s" % server_starttime ) for name in args ] )
def js( *args ):
"""
Take a prefix and list of javascript names and return appropriate
string of script tags.
"""
return js_helper( '/static/scripts/', *args )
def templates( *args ):
"""
Take a list of template names (no extension) and return appropriate
string of script tags.
"""
return js_helper( '/static/scripts/templates/compiled/', *args )
# Hashes
lib/galaxy/webapps/community/app.py
+3 -1
View File
@@ -43,6 +43,8 @@ class UniverseApplication( object ):
# Load security policy
self.security_agent = self.model.security_agent
self.quota_agent = galaxy.quota.NoQuotaAgent( self.model )
self.openid_providers = OpenIDProviders() #TODO: Add OpenID support
# TODO: Add OpenID support
self.openid_providers = OpenIDProviders()
self.shed_counter = self.model.shed_counter
def shutdown( self ):
pass
lib/galaxy/webapps/community/config.py
+7 -2
View File
@@ -39,8 +39,6 @@ class Configuration( object ):
self.file_path = resolve_path( kwargs.get( "file_path", "database/files" ), self.root )
self.new_file_path = resolve_path( kwargs.get( "new_file_path", "database/tmp" ), self.root )
self.cookie_path = kwargs.get( "cookie_path", "/" )
# web API
self.enable_api = string_as_bool( kwargs.get( 'enable_api', False ) )
self.enable_quotas = string_as_bool( kwargs.get( 'enable_quotas', False ) )
self.datatypes_config = kwargs.get( 'datatypes_config_file', 'datatypes_conf.xml' )
self.test_conf = resolve_path( kwargs.get( "test_conf", "" ), self.root )
@@ -84,6 +82,12 @@ class Configuration( object ):
self.screencasts_url = kwargs.get( 'screencasts_url', None )
self.log_events = False
self.cloud_controller_instance = False
self.server_name = ''
self.job_manager = ''
self.default_job_handlers = []
self.job_handlers = []
self.tool_handlers = []
self.tool_runners = []
# Proxy features
self.apache_xsendfile = kwargs.get( 'apache_xsendfile', False )
self.nginx_x_accel_redirect_base = kwargs.get( 'nginx_x_accel_redirect_base', False )
@@ -94,6 +98,7 @@ class Configuration( object ):
self.global_conf_parser = global_conf_parser
if global_conf and "__file__" in global_conf:
global_conf_parser.read(global_conf['__file__'])
self.running_functional_tests = string_as_bool( kwargs.get( 'running_functional_tests', False ) )
def get( self, key, default ):
return self.config_dict.get( key, default )
def get_bool( self, key, default ):
lib/galaxy/webapps/community/controllers/admin.py
+43 -1
View File
@@ -3,9 +3,14 @@ from galaxy.webapps.community import model
from galaxy.model.orm import *
from galaxy.web.framework.helpers import time_ago, iff, grids
from galaxy.util import inflector
from galaxy.util.shed_util import get_configured_ui
from common import *
from repository import RepositoryListGrid, CategoryListGrid
from galaxy import eggs
eggs.require('mercurial')
from mercurial import hg
import logging
log = logging.getLogger( __name__ )
@@ -321,7 +326,7 @@ class RepositoryMetadataListGrid( grids.Grid ):
def get_value( self, trans, grid, repository_metadata ):
repository = repository_metadata.repository
repo = hg.repository( get_configured_ui(), repository.repo_path )
ctx = get_changectx_for_changeset( trans, repo, repository_metadata.changeset_revision )
ctx = get_changectx_for_changeset( repo, repository_metadata.changeset_revision )
return "%s:%s" % ( str( ctx.rev() ), repository_metadata.changeset_revision )
class ToolsColumn( grids.TextColumn ):
def get_value( self, trans, grid, repository_metadata ):
@@ -457,6 +462,31 @@ class AdminController( BaseUIController, Admin ):
status=status ) )
@web.expose
@web.require_admin
def reset_all_repository_metadata( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
if 'reset_all_repository_metadata_button' in kwd:
count = 0
for repository in trans.sa_session.query( trans.model.Repository ) \
.filter( trans.model.Repository.table.c.deleted == False ):
try:
reset_all_repository_metadata( trans, trans.security.encode_id( repository.id ) )
log.debug( "Reset metadata on repository %s" % repository.name )
count += 1
except Exception, e:
log.debug( "Error attempting to reset metadata on repository %s: %s" % ( repository.name, str( e ) ) )
message = "Reset metadata on %d repositories" % count
trans.response.send_redirect( web.url_for( controller='admin',
action='browse_repository_metadata',
webapp='community',
message=util.sanitize_text( message ),
status=status ) )
return trans.fill_template( '/webapps/community/admin/reset_all_repository_metadata.mako',
message=message,
status=status )
@web.expose
@web.require_admin
def browse_repositories( self, trans, **kwd ):
# We add params to the keyword dict in this method in order to rename the param
# with an "f-" prefix, simulating filtering by clicking a search link. We have
@@ -527,6 +557,18 @@ class AdminController( BaseUIController, Admin ):
return self.repository_list_grid( trans, **kwd )
@web.expose
@web.require_admin
def regenerate_statistics( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
if 'regenerate_statistics_button' in kwd:
trans.app.shed_counter.generate_statistics()
message = "Successfully regenerated statistics"
return trans.fill_template( '/webapps/community/admin/statistics.mako',
message=message,
status=status )
@web.expose
@web.require_admin
def delete_repository( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
lib/galaxy/webapps/community/controllers/common.py
+114 -73
View File
@@ -5,12 +5,15 @@ from galaxy.datatypes.checkers import *
from galaxy.tools import *
from galaxy.util.json import from_json_string, to_json_string
from galaxy.util.hash_util import *
from galaxy.util.shed_util import copy_sample_loc_file, generate_datatypes_metadata, generate_tool_metadata, generate_workflow_metadata
from galaxy.util.shed_util import handle_sample_tool_data_table_conf_file
from galaxy.util.shed_util import copy_sample_file, get_configured_ui, generate_datatypes_metadata, generate_tool_metadata, generate_workflow_metadata
from galaxy.util.shed_util import handle_sample_tool_data_table_conf_file, to_html_escaped, to_html_str, update_repository
from galaxy.web.base.controller import *
from galaxy.webapps.community import model
from galaxy.model.orm import *
from galaxy.model.item_attrs import UsesItemRatings
from galaxy import eggs
eggs.require('mercurial')
from mercurial import hg, ui, commands
log = logging.getLogger( __name__ )
@@ -140,7 +143,7 @@ def get_revision_label( trans, repository, changeset_revision ):
changeset rev and the changeset revision string.
"""
repo = hg.repository( get_configured_ui(), repository.repo_path )
ctx = get_changectx_for_changeset( trans, repo, changeset_revision )
ctx = get_changectx_for_changeset( repo, changeset_revision )
if ctx:
return "%s:%s" % ( str( ctx.rev() ), changeset_revision )
else:
@@ -201,18 +204,18 @@ def check_tool_input_params( trans, name, tool, sample_files, invalid_files ):
can_set_metadata = False
correction_msg = "This file requires an entry in the tool_data_table_conf.xml file. "
correction_msg += "Upload a file named tool_data_table_conf.xml.sample to the repository "
correction_msg += "that includes the required entry to resolve this issue.<br/>"
correction_msg += "that includes the required entry to correct this error.<br/>"
invalid_files.append( ( name, correction_msg ) )
if options.index_file or options.missing_index_file:
# Make sure the repository contains the required xxx.loc.sample file.
index_file = options.index_file or options.missing_index_file
index_head, index_tail = os.path.split( index_file )
index_file_path, index_file_name = os.path.split( index_file )
sample_found = False
for sample_file in sample_files:
sample_head, sample_tail = os.path.split( sample_file )
if sample_tail == '%s.sample' % index_tail:
copy_sample_loc_file( trans.app, sample_file )
options.index_file = index_tail
sample_file_path, sample_file_name = os.path.split( sample_file )
if sample_file_name == '%s.sample' % index_file_name:
copy_sample_file( trans.app, sample_file )
options.index_file = index_file_name
options.missing_index_file = None
if options.tool_data_table:
options.tool_data_table.missing_index_file = None
@@ -221,7 +224,7 @@ def check_tool_input_params( trans, name, tool, sample_files, invalid_files ):
if not sample_found:
can_set_metadata = False
correction_msg = "This file refers to a file named <b>%s</b>. " % str( index_file )
correction_msg += "Upload a file named <b>%s.sample</b> to the repository to correct this error." % str( index_tail )
correction_msg += "Upload a file named <b>%s.sample</b> to the repository to correct this error." % str( index_file_name )
invalid_files.append( ( name, correction_msg ) )
return can_set_metadata, invalid_files
def new_tool_metadata_required( trans, id, metadata_dict ):
@@ -280,16 +283,20 @@ def new_workflow_metadata_required( trans, id, metadata_dict ):
else:
# There is no saved repository metadata, so we need to create a new repository_metadata table record.
return True
# The received metadata_dict includes no metadata for workflows, so a new repository_metadata table
# record is not needed.
# The received metadata_dict includes no metadata for workflows, so a new repository_metadata table record is not needed.
return False
def generate_metadata_for_repository_tip( trans, id, ctx, changeset_revision, repo_dir ):
# Browse the repository tip files on disk to generate metadata. This is faster than
# the generate_metadata_for_changeset_revision() method below because fctx.data() does
# not have to be written to disk to load tools. also, since changeset_revision is the
# repository tip, we handle things like .loc.sample files here.
def generate_metadata_for_repository_tip( trans, id, ctx, changeset_revision, repo, repo_dir ):
"""
Browse the repository tip files on disk to generate metadata. This is faster than the
generate_metadata_for_changeset_revision() method below because fctx.data() does not have
to be written to disk to load tools. We also handle things like .loc.sample files and
invalid_tool_configs here, while they are ignored in older revisions.
"""
# If a push from the command line is occurring, update the repository files on disk before setting metadata.
update_repository( repo, str( ctx.rev() ) )
metadata_dict = {}
invalid_files = []
invalid_tool_configs = []
sample_files = []
datatypes_config = None
# Find datatypes_conf.xml if it exists.
@@ -319,18 +326,30 @@ def generate_metadata_for_repository_tip( trans, id, ctx, changeset_revision, re
if not ( check_binary( full_path ) or check_image( full_path ) or check_gzip( full_path )[ 0 ]
or check_bz2( full_path )[ 0 ] or check_zip( full_path ) ):
try:
tool = load_tool( trans, full_path )
valid = True
# Make sure we're looking at a tool config and not a display application config or something else.
element_tree = util.parse_xml( full_path )
element_tree_root = element_tree.getroot()
is_tool = element_tree_root.tag == 'tool'
except Exception, e:
valid = False
invalid_files.append( ( name, str( e ) ) )
if valid and tool is not None:
can_set_metadata, invalid_files = check_tool_input_params( trans, name, tool, sample_files, invalid_files )
if can_set_metadata:
# Update the list of metadata dictionaries for tools in metadata_dict.
tool_config = os.path.join( root, name )
repository_clone_url = generate_clone_url( trans, id )
metadata_dict = generate_tool_metadata( tool_config, tool, repository_clone_url, metadata_dict )
log.debug( "Error parsing %s, exception: %s" % ( full_path, str( e ) ) )
is_tool = False
if is_tool:
try:
tool = load_tool( trans, full_path )
valid = True
except Exception, e:
valid = False
invalid_files.append( ( name, str( e ) ) )
invalid_tool_configs.append( name )
if valid and tool is not None:
can_set_metadata, invalid_files = check_tool_input_params( trans, name, tool, sample_files, invalid_files )
if can_set_metadata:
# Update the list of metadata dictionaries for tools in metadata_dict.
tool_config = os.path.join( root, name )
repository_clone_url = generate_clone_url( trans, id )
metadata_dict = generate_tool_metadata( tool_config, tool, repository_clone_url, metadata_dict )
else:
invalid_tool_configs.append( name )
# Find all exported workflows
elif name.endswith( '.ga' ):
try:
@@ -344,6 +363,8 @@ def generate_metadata_for_repository_tip( trans, id, ctx, changeset_revision, re
metadata_dict = generate_workflow_metadata( relative_path, exported_workflow_dict, metadata_dict )
except Exception, e:
invalid_files.append( ( name, str( e ) ) )
if invalid_tool_configs:
metadata_dict[ 'invalid_tools' ] = invalid_tool_configs
return metadata_dict, invalid_files
def generate_metadata_for_changeset_revision( trans, id, ctx, changeset_revision, repo_dir ):
# Browse repository files within a change set to generate metadata.
@@ -388,21 +409,29 @@ def generate_metadata_for_changeset_revision( trans, id, ctx, changeset_revision
if not ( check_binary( tmp_filename ) or check_image( tmp_filename ) or check_gzip( tmp_filename )[ 0 ]
or check_bz2( tmp_filename )[ 0 ] or check_zip( tmp_filename ) ):
try:
tool = load_tool( trans, tmp_filename )
valid = True
except Exception, e:
invalid_files.append( ( filename, str( e ) ) )
valid = False
if valid and tool is not None:
# Update the list of metadata dictionaries for tools in metadata_dict. Note that filename
# here is the relative path to the config file within the change set context, something
# like filtering.xml, but when the change set was the repository tip, the value was
# something like database/community_files/000/repo_1/filtering.xml. This shouldn't break
# anything, but may result in a bit of confusion when maintaining the code / data over time.
# IMPORTANT NOTE: Here we are assuming that since the current change set is not the repository
# tip, we do not have to handle any .loc.sample files since they would have been handled previously.
repository_clone_url = generate_clone_url( trans, id )
metadata_dict = generate_tool_metadata( filename, tool, repository_clone_url, metadata_dict )
# Make sure we're looking at a tool config and not a display application config or something else.
element_tree = util.parse_xml( tmp_filename )
element_tree_root = element_tree.getroot()
is_tool = element_tree_root.tag == 'tool'
except:
is_tool = False
if is_tool:
try:
tool = load_tool( trans, tmp_filename )
valid = True
except Exception, e:
invalid_files.append( ( filename, str( e ) ) )
valid = False
if valid and tool is not None:
# Update the list of metadata dictionaries for tools in metadata_dict. Note that filename
# here is the relative path to the config file within the change set context, something
# like filtering.xml, but when the change set was the repository tip, the value was
# something like database/community_files/000/repo_1/filtering.xml. This shouldn't break
# anything, but may result in a bit of confusion when maintaining the code / data over time.
# IMPORTANT NOTE: Here we are assuming that since the current change set is not the repository
# tip, we do not have to handle any .loc.sample files since they would have been handled previously.
repository_clone_url = generate_clone_url( trans, id )
metadata_dict = generate_tool_metadata( filename, tool, repository_clone_url, metadata_dict )
try:
os.unlink( tmp_filename )
except:
@@ -425,12 +454,12 @@ def set_repository_metadata( trans, id, changeset_revision, content_alert_str=''
repository = get_repository( trans, id )
repo_dir = repository.repo_path
repo = hg.repository( get_configured_ui(), repo_dir )
ctx = get_changectx_for_changeset( trans, repo, changeset_revision )
ctx = get_changectx_for_changeset( repo, changeset_revision )
metadata_dict = {}
invalid_files = []
if ctx is not None:
if changeset_revision == repository.tip:
metadata_dict, invalid_files = generate_metadata_for_repository_tip( trans, id, ctx, changeset_revision, repo_dir )
metadata_dict, invalid_files = generate_metadata_for_repository_tip( trans, id, ctx, changeset_revision, repo, repo_dir )
else:
metadata_dict, invalid_files = generate_metadata_for_changeset_revision( trans, id, ctx, changeset_revision, repo_dir )
if metadata_dict:
@@ -439,10 +468,14 @@ def set_repository_metadata( trans, id, changeset_revision, content_alert_str=''
# Create a new repository_metadata table row.
repository_metadata = trans.model.RepositoryMetadata( repository.id, changeset_revision, metadata_dict )
trans.sa_session.add( repository_metadata )
trans.sa_session.flush()
# If this is the first record stored for this repository, see if we need to send any email alerts.
if len( repository.downloadable_revisions ) == 1:
handle_email_alerts( trans, repository, content_alert_str='', new_repo_alert=True, admin_only=False )
try:
trans.sa_session.flush()
# If this is the first record stored for this repository, see if we need to send any email alerts.
if len( repository.downloadable_revisions ) == 1:
handle_email_alerts( trans, repository, content_alert_str='', new_repo_alert=True, admin_only=False )
except TypeError, e:
message = "Unable to save metadata for this repository probably due to a tool config file that doesn't conform to the Cheetah template syntax."
status = 'error'
else:
repository_metadata = get_latest_repository_metadata( trans, id )
if repository_metadata:
@@ -450,9 +483,13 @@ def set_repository_metadata( trans, id, changeset_revision, content_alert_str=''
repository_metadata.changeset_revision = changeset_revision
repository_metadata.metadata = metadata_dict
trans.sa_session.add( repository_metadata )
trans.sa_session.flush()
try:
trans.sa_session.flush()
except TypeError, e:
message = "Unable to save metadata for this repository probably due to a tool config file that doesn't conform to the Cheetah template syntax."
status = 'error'
else:
# There are no tools in the repository, and we're setting metadat on the repository tip.
# There are no tools in the repository, and we're setting metadata on the repository tip.
repository_metadata = trans.model.RepositoryMetadata( repository.id, changeset_revision, metadata_dict )
trans.sa_session.add( repository_metadata )
trans.sa_session.flush()
@@ -514,9 +551,9 @@ def reset_all_repository_metadata( trans, id, **kwd ):
ancestor_metadata_dict = None
for changeset in repo.changelog:
current_changeset_revision = str( repo.changectx( changeset ) )
ctx = get_changectx_for_changeset( trans, repo, current_changeset_revision )
ctx = get_changectx_for_changeset( repo, current_changeset_revision )
if current_changeset_revision == repository.tip:
current_metadata_dict, invalid_files = generate_metadata_for_repository_tip( trans, id, ctx, current_changeset_revision, repo_dir )
current_metadata_dict, invalid_files = generate_metadata_for_repository_tip( trans, id, ctx, current_changeset_revision, repo, repo_dir )
else:
current_metadata_dict, invalid_files = generate_metadata_for_changeset_revision( trans, id, ctx, current_changeset_revision, repo_dir )
if current_metadata_dict:
@@ -538,8 +575,8 @@ def reset_all_repository_metadata( trans, id, **kwd ):
create_or_update_repository_metadata( trans, id, repository, ancestor_changeset_revision, ancestor_metadata_dict )
# Keep track of the changeset_revisions that we've persisted.
changeset_revisions.append( ancestor_changeset_revision )
ancestor_changeset_revision = None
ancestor_metadata_dict = None
ancestor_changeset_revision = current_changeset_revision
ancestor_metadata_dict = current_metadata_dict
else:
# We're either at the first change set in the change log or we have just created or updated
# a repository_metadata record. At this point we set the ancestor changeset to the current
@@ -722,12 +759,12 @@ def compare_datatypes( ancestor_datatypes, current_datatypes ):
# Currently the only way to differentiate datatypes is by name.
ancestor_datatype_dtype = ancestor_datatype[ 'dtype' ]
ancestor_datatype_extension = ancestor_datatype[ 'extension' ]
ancestor_datatype_mimetype = ancestor_datatype[ 'mimetype' ]
ancestor_datatype_mimetype = ancestor_datatype.get( 'mimetype', None )
found_in_current = False
for current_datatype in current_datatypes:
if current_datatype[ 'dtype' ] == ancestor_datatype_dtype and \
current_datatype[ 'extension' ] == ancestor_datatype_extension and \
current_datatype[ 'mimetype' ] == ancestor_datatype_mimetype:
current_datatype.get( 'mimetype', None ) == ancestor_datatype_mimetype:
found_in_current = True
break
if not found_in_current:
@@ -740,7 +777,7 @@ def compare_datatypes( ancestor_datatypes, current_datatypes ):
def get_repository_by_name( trans, name ):
"""Get a repository from the database via name"""
return trans.sa_session.query( trans.model.Repository ).filter_by( name=name ).one()
def get_changectx_for_changeset( trans, repo, changeset_revision, **kwd ):
def get_changectx_for_changeset( repo, changeset_revision, **kwd ):
"""Retrieve a specified changectx from a repository"""
for changeset in repo.changelog:
ctx = repo.changectx( changeset )
@@ -753,15 +790,6 @@ def change_set_is_malicious( trans, id, changeset_revision, **kwd ):
if repository_metadata:
return repository_metadata.malicious
return False
def get_configured_ui():
# Configure any desired ui settings.
_ui = ui.ui()
# The following will suppress all messages. This is
# the same as adding the following setting to the repo
# hgrc file' [ui] section:
# quiet = True
_ui.setconfig( 'ui', 'quiet', True )
return _ui
def get_user( trans, id ):
"""Get a user from the database by id"""
return trans.sa_session.query( trans.model.User ).get( trans.security.decode_id( id ) )
@@ -864,9 +892,10 @@ def check_file_contents( trans ):
return True
return False
def update_for_browsing( trans, repository, current_working_dir, commit_message='' ):
# Make a copy of a repository's files for browsing, remove from disk all files that
# are not tracked, and commit all added, modified or removed files that have not yet
# been committed.
# This method id deprecated, but we'll keep it around for a while in case we need it. The problem is that hg purge
# is not supported by the mercurial API.
# Make a copy of a repository's files for browsing, remove from disk all files that are not tracked, and commit all
# added, modified or removed files that have not yet been committed.
repo_dir = repository.repo_path
repo = hg.repository( get_configured_ui(), repo_dir )
# The following will delete the disk copy of only the files in the repository.
@@ -963,7 +992,7 @@ def load_tool_from_changeset_revision( trans, repository_id, changeset_revision,
# Get the tool config file name from the hgweb url, something like:
# /repos/test/convert_chars1/file/e58dcf0026c7/convert_characters.xml
old_tool_config_file_name = tool_config.split( '/' )[ -1 ]
ctx = get_changectx_for_changeset( trans, repo, changeset_revision )
ctx = get_changectx_for_changeset( repo, changeset_revision )
fctx = None
for filename in ctx:
filename_head, filename_tail = os.path.split( filename )
@@ -997,12 +1026,24 @@ def build_changeset_revision_select_field( trans, repository, selected_value=Non
"""
repo = hg.repository( get_configured_ui(), repository.repo_path )
options = []
changeset_tups = []
refresh_on_change_values = []
for repository_metadata in repository.downloadable_revisions:
changeset_revision = repository_metadata.changeset_revision
revision_label = get_revision_label( trans, repository, changeset_revision )
options.append( ( revision_label, changeset_revision ) )
ctx = get_changectx_for_changeset( repo, changeset_revision )
if ctx:
rev = '%04d' % ctx.rev()
label = "%s:%s" % ( str( ctx.rev() ), changeset_revision )
else:
rev = '-1'
label = "-1:%s" % changeset_revision
changeset_tups.append( ( rev, label, changeset_revision ) )
refresh_on_change_values.append( changeset_revision )
# Sort options by the revision label. Even though the downloadable_revisions query sorts by update_time,
# the changeset revisions may not be sorted correctly because setting metadata over time will reset update_time.
for changeset_tup in sorted( changeset_tups ):
# Display the latest revision first.
options.insert( 0, ( changeset_tup[1], changeset_tup[2] ) )
if add_id_to_name:
name = 'changeset_revision_%d' % repository.id
else:
lib/galaxy/webapps/community/controllers/hg.py
+18 -1
View File
@@ -1,5 +1,9 @@
import os, logging
from galaxy.web.base.controller import *
from galaxy.webapps.community.controllers.common import *
from galaxy import eggs
eggs.require('mercurial')
from mercurial.hgweb.hgwebdir_mod import hgwebdir
from mercurial.hgweb.request import wsgiapplication
@@ -10,7 +14,20 @@ class HgController( BaseUIController ):
def handle_request( self, trans, **kwd ):
# The os command that results in this method being called will look something like
# hg clone http://test@127.0.0.1:9009/repos/test/convert_characters1
return wsgiapplication( make_web_app )
cmd = kwd.get( 'cmd', None )
wsgi_app = wsgiapplication( make_web_app )
# Hack: Add a parameter to requests for which we do not want all repository metadata reset.
reset_metadata = not ( kwd.get( 'no_reset', False ) )
if cmd == 'listkeys' and reset_metadata:
# This possibly results from an "hg push" from the command line. When doing this, the following 7 commands, in order,
# will be retrieved from environ: between -> capabilities -> heads -> branchmap -> unbundle -> unbundle -> listkeys
path_info = kwd.get( 'path_info', None )
if path_info:
owner, name = path_info.split( '/' )
repository = get_repository_by_name_and_owner( trans, name, owner )
if repository:
reset_all_repository_metadata( trans, trans.security.encode_id( repository.id ) )
return wsgi_app
def make_web_app():
hgweb_config = "%s/hgweb.config" % os.getcwd()
lib/galaxy/webapps/community/controllers/repository.py
+214 -79
View File
@@ -10,19 +10,16 @@ from galaxy.webapps.community.model import directory_hash_id
from galaxy.web.framework.helpers import time_ago, iff, grids
from galaxy.util.json import from_json_string, to_json_string
from galaxy.model.orm import *
from galaxy.util.shed_util import get_configured_ui
from common import *
from galaxy import eggs
eggs.require('mercurial')
from mercurial import hg, ui, patch, commands
log = logging.getLogger( __name__ )
# Characters that must be html escaped
MAPPED_CHARS = { '>' :'&gt;',
'<' :'&lt;',
'"' : '&quot;',
'&' : '&amp;',
'\'' : '&apos;' }
MAX_CONTENT_SIZE = 32768
VALID_CHARS = set( string.letters + string.digits + "'\"-=_.()/+*^,:?!#[]%\\$@;{}" )
VALID_REPOSITORYNAME_RE = re.compile( "^[a-z0-9\_]+$" )
class CategoryListGrid( grids.Grid ):
@@ -407,6 +404,40 @@ class RepositoryController( BaseUIController, ItemRatings ):
# Render the list view
return self.valid_repository_list_grid( trans, **kwd )
@web.expose
def browse_invalid_tools( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
webapp = params.get( 'webapp', 'community' )
cntrller = params.get( 'cntrller', 'repository' )
is_admin = trans.user_is_admin()
invalid_tools_dict = odict()
if is_admin and cntrller == 'admin':
for repository in trans.sa_session.query( trans.model.Repository ) \
.filter( trans.model.Repository.table.c.deleted == False ) \
.order_by( trans.model.Repository.table.c.name ):
for downloadable_revision in repository.downloadable_revisions:
metadata = downloadable_revision.metadata
invalid_tools = metadata.get( 'invalid_tools', [] )
for invalid_tool_config in invalid_tools:
invalid_tools_dict[ invalid_tool_config ] = ( repository.id, repository.name, downloadable_revision.changeset_revision )
else:
for repository in trans.sa_session.query( trans.model.Repository ) \
.filter( and_( trans.model.Repository.table.c.deleted == False,
trans.model.Repository.table.c.user_id == trans.user.id ) ) \
.order_by( trans.model.Repository.table.c.name ):
for downloadable_revision in repository.downloadable_revisions:
metadata = downloadable_revision.metadata
invalid_tools = metadata.get( 'invalid_tools', [] )
for invalid_tool_config in invalid_tools:
invalid_tools_dict[ invalid_tool_config ] = ( repository.id, repository.name, downloadable_revision.changeset_revision )
return trans.fill_template( '/webapps/community/repository/browse_invalid_tools.mako',
cntrller=cntrller,
invalid_tools_dict=invalid_tools_dict,
webapp=webapp,
message=message,
status=status )
@web.expose
def find_workflows( self, trans, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
@@ -444,13 +475,16 @@ class RepositoryController( BaseUIController, ItemRatings ):
# This can only occur when there is a multi-select grid with check boxes and an operation,
# and the user clicked the operation button without checking any of the check boxes.
return trans.show_error_message( "No items were selected." )
workflow_names = [ item.lower() for item in util.listify( kwd.get( 'workflow_name', '' ) ) ]
exact_matches = params.get( 'exact_matches', '' )
exact_matches_checked = CheckboxField.is_checked( exact_matches )
match_tuples = []
ok = True
if workflow_names:
ok, match_tuples = self.__search_repository_metadata( trans, exact_matches_checked, workflow_names=workflow_names )
if 'find_workflows_button' in kwd:
workflow_names = [ item.lower() for item in util.listify( kwd.get( 'workflow_name', '' ) ) ]
exact_matches = params.get( 'exact_matches', '' )
exact_matches_checked = CheckboxField.is_checked( exact_matches )
match_tuples = []
ok = True
if workflow_names:
ok, match_tuples = self.__search_repository_metadata( trans, exact_matches_checked, workflow_names=workflow_names )
else:
ok, match_tuples = self.__search_repository_metadata( trans, exact_matches_checked, workflow_names=[], all_workflows=True )
if ok:
kwd[ 'match_tuples' ] = match_tuples
# Render the list view
@@ -475,6 +509,9 @@ class RepositoryController( BaseUIController, ItemRatings ):
else:
message = "No search performed - each field must contain the same number of comma-separated items."
status = "error"
else:
exact_matches_checked = False
workflow_names = []
exact_matches_check_box = CheckboxField( 'exact_matches', checked=exact_matches_checked )
return trans.fill_template( '/webapps/community/repository/find_workflows.mako',
webapp=webapp,
@@ -562,7 +599,7 @@ class RepositoryController( BaseUIController, ItemRatings ):
exact_matches_check_box=exact_matches_check_box,
message=message,
status=status )
def __search_repository_metadata( self, trans, exact_matches_checked, tool_ids='', tool_names='', tool_versions='', workflow_names='' ):
def __search_repository_metadata( self, trans, exact_matches_checked, tool_ids='', tool_names='', tool_versions='', workflow_names='', all_workflows=False ):
match_tuples = []
ok = True
for repository_metadata in trans.sa_session.query( model.RepositoryMetadata ):
@@ -630,6 +667,8 @@ class RepositoryController( BaseUIController, ItemRatings ):
for workflow_name in workflow_names:
if self.__in_workflow_dict( workflow_dict, exact_matches_checked, workflow_name ):
match_tuples.append( ( repository_metadata.repository_id, repository_metadata.changeset_revision ) )
elif all_workflows and 'workflows' in metadata:
match_tuples.append( ( repository_metadata.repository_id, repository_metadata.changeset_revision ) )
return ok, match_tuples
def __in_workflow_dict( self, workflow_dict, exact_matches_checked, workflow_name ):
workflow_dict_workflow_name = workflow_dict[ 'name' ].lower()
@@ -716,10 +755,14 @@ class RepositoryController( BaseUIController, ItemRatings ):
if not includes_tools and 'tools' in repository_metadata.metadata:
includes_tools = True
repository = get_repository( trans, trans.security.encode_id( repository_metadata.repository_id ) )
repository_id = trans.security.encode_id( repository.id )
# Get the changelog rev for this changeset_revision.
repo_dir = repository.repo_path
repo = hg.repository( get_configured_ui(), repo_dir )
changeset_revision = repository_metadata.changeset_revision
ctx = get_changectx_for_changeset( repo, changeset_revision )
repository_id = trans.security.encode_id( repository.id )
repository_clone_url = generate_clone_url( trans, repository_id )
repo_info_dict[ repository.name ] = ( repository.description, repository_clone_url, changeset_revision )
repo_info_dict[ repository.name ] = ( repository.description, repository_clone_url, changeset_revision, str( ctx.rev() ) )
return encode( repo_info_dict ), includes_tools
@web.expose
def preview_tools_in_changeset( self, trans, repository_id, **kwd ):
@@ -765,14 +808,47 @@ class RepositoryController( BaseUIController, ItemRatings ):
# Tell the caller if the repository includes Galaxy tools so the page
# enabling selection of the tool panel section can be displayed.
includes_tools = 'tools' in repository_metadata.metadata
# Get the changelog rev for this changeset_revision.
repo_dir = repository.repo_path
repo = hg.repository( get_configured_ui(), repo_dir )
ctx = get_changectx_for_changeset( repo, changeset_revision )
repo_info_dict = {}
repo_info_dict[ repository.name ] = ( repository.description, repository_clone_url, changeset_revision )
repo_info_dict[ repository.name ] = ( repository.description, repository_clone_url, changeset_revision, str( ctx.rev() ) )
encoded_repo_info_dict = encode( repo_info_dict )
# Redirect back to local Galaxy to perform install.
url = '%sadmin_toolshed/install_repository?tool_shed_url=%s&repo_info_dict=%s&includes_tools=%s' % \
( galaxy_url, url_for( '/', qualified=True ), encoded_repo_info_dict, str( includes_tools ) )
return trans.response.send_redirect( url )
@web.expose
def get_ctx_rev( self, trans, **kwd ):
"""Given a repository and changeset_revision, return the correct ctx.rev() value."""
repository_name = kwd[ 'name' ]
repository_owner = kwd[ 'owner' ]
changeset_revision = kwd[ 'changeset_revision' ]
repository = get_repository_by_name_and_owner( trans, repository_name, repository_owner )
repo_dir = repository.repo_path
repo = hg.repository( get_configured_ui(), repo_dir )
ctx = get_changectx_for_changeset( repo, changeset_revision )
if ctx:
return str( ctx.rev() )
return ''
@web.expose
def get_readme( self, trans, **kwd ):
"""If the received changeset_revision includes a file named readme (case ignored), return it's contents."""
repository_name = kwd[ 'name' ]
repository_owner = kwd[ 'owner' ]
changeset_revision = kwd[ 'changeset_revision' ]
repository = get_repository_by_name_and_owner( trans, repository_name, repository_owner )
repo_dir = repository.repo_path
for root, dirs, files in os.walk( repo_dir ):
for name in files:
if name.lower() in [ 'readme', 'readme.txt', 'read_me', 'read_me.txt', '%s.txt' % repository_name ]:
f = open( os.path.join( root, name ), 'r' )
text = f.read()
f.close()
return str( text )
return ''
@web.expose
def get_tool_versions( self, trans, **kwd ):
"""
For each valid /downloadable change set (up to the received changeset_revision) in the
@@ -793,7 +869,7 @@ class RepositoryController( BaseUIController, ItemRatings ):
tool_version_dicts.append( repository_metadata.tool_versions )
if current_changeset_revision == changeset_revision:
break
if repository_metadata.tool_versions:
if tool_version_dicts:
return to_json_string( tool_version_dicts )
return ''
@web.expose
@@ -810,6 +886,9 @@ class RepositoryController( BaseUIController, ItemRatings ):
changeset_revision = params.get( 'changeset_revision', None )
webapp = params.get( 'webapp', 'community' )
repository = get_repository_by_name_and_owner( trans, name, owner )
repo_dir = repository.repo_path
repo = hg.repository( get_configured_ui(), repo_dir )
latest_ctx = get_changectx_for_changeset( repo, changeset_revision )
from_update_manager = webapp == 'update_manager'
if from_update_manager:
update = 'true'
@@ -820,10 +899,10 @@ class RepositoryController( BaseUIController, ItemRatings ):
url += '&name=%s&owner=%s&changeset_revision=%s&latest_changeset_revision=' % \
( repository.name, repository.user.username, changeset_revision )
if changeset_revision == repository.tip:
# If changeset_revision is the repository tip, then
# we know there are no additional updates for the tools.
# If changeset_revision is the repository tip, we know there are no additional updates for the tools.
if from_update_manager:
return no_update
# Return the same value for changeset_revision and latest_changeset_revision.
url += repository.tip
else:
repository_metadata = get_repository_metadata_by_changeset_revision( trans,
@@ -834,15 +913,15 @@ class RepositoryController( BaseUIController, ItemRatings ):
# repository, then we know there are no additional updates for the tools.
if from_update_manager:
return no_update
url += changeset_revision
else:
# Return the same value for changeset_revision and latest_changeset_revision.
url += changeset_revision
else:
# The changeset_revision column in the repository_metadata table has been
# updated with a new changeset_revision value since the repository was cloned.
repo_dir = repository.repo_path
repo = hg.repository( get_configured_ui(), repo_dir )
# Load each tool in the repository's changeset_revision to generate a list of
# tool guids, since guids differentiate tools by id and version.
ctx = get_changectx_for_changeset( trans, repo, changeset_revision )
ctx = get_changectx_for_changeset( repo, changeset_revision )
if ctx is not None:
tool_guids = []
for filename in ctx:
@@ -859,12 +938,21 @@ class RepositoryController( BaseUIController, ItemRatings ):
if not ( check_binary( tmp_filename ) or check_image( tmp_filename ) or check_gzip( tmp_filename )[ 0 ]
or check_bz2( tmp_filename )[ 0 ] or check_zip( tmp_filename ) ):
try:
tool = load_tool( trans, tmp_filename )
valid = True
except:
valid = False
if valid and tool is not None:
tool_guids.append( generate_tool_guid( trans, repository, tool ) )
# Make sure we're looking at a tool config and not a display application config or something else.
element_tree = util.parse_xml( tmp_filename )
element_tree_root = element_tree.getroot()
is_tool = element_tree_root.tag == 'tool'
except Exception, e:
log.debug( "Error parsing %s, exception: %s" % ( tmp_filename, str( e ) ) )
is_tool = False
if is_tool:
try:
tool = load_tool( trans, tmp_filename )
valid = True
except:
valid = False
if valid and tool is not None:
tool_guids.append( generate_tool_guid( trans, repository, tool ) )
try:
os.unlink( tmp_filename )
except:
@@ -882,11 +970,12 @@ class RepositoryController( BaseUIController, ItemRatings ):
metadata_tool_guids.append( tool_dict[ 'guid' ] )
metadata_tool_guids.sort()
if tool_guids == metadata_tool_guids:
# We've found the repository_metadata record whose changeset_revision
# value has been updated.
# We've found the repository_metadata record whose changeset_revision value has been updated.
if from_update_manager:
return update
url += repository_metadata.changeset_revision
# Get the ctx_rev for the discovered changeset_revision.
latest_ctx = get_changectx_for_changeset( repo, repository_metadata.changeset_revision )
found = True
break
if not found:
@@ -896,10 +985,11 @@ class RepositoryController( BaseUIController, ItemRatings ):
return no_update
url += changeset_revision
else:
# There are not tools in the changeset_revision, so no tool updates are possible.
# There are no tools in the changeset_revision, so no tool updates are possible.
if from_update_manager:
return no_update
url += changeset_revision
url += '&latest_ctx_rev=%s' % str( latest_ctx.rev() )
return trans.response.send_redirect( url )
@web.expose
def browse_repositories( self, trans, **kwd ):
@@ -1123,13 +1213,10 @@ class RepositoryController( BaseUIController, ItemRatings ):
new_hgweb_config.flush()
shutil.move( tmp_fname, os.path.abspath( hgweb_config ) )
def __create_hgrc_file( self, repository ):
# At this point, an entry for the repository is required to be in the hgweb.config
# file so we can call repository.repo_path.
# Since we support both http and https, we set push_ssl to False to override
# the default (which is True) in the mercurial api.
# The hg purge extension purges all files and directories not being tracked by
# mercurial in the current repository. It'll remove unknown files and empty
# directories. This is used in the update_for_browsing() method.
# At this point, an entry for the repository is required to be in the hgweb.config file so we can call repository.repo_path.
# Since we support both http and https, we set push_ssl to False to override the default (which is True) in the mercurial api.
# The hg purge extension purges all files and directories not being tracked by mercurial in the current repository. It'll
# remove unknown files and empty directories. This is not currently used because it is not supported in the mercurial API.
repo = hg.repository( get_configured_ui(), path=repository.repo_path )
fp = repo.opener( 'hgrc', 'wb' )
fp.write( '[paths]\n' )
@@ -1153,7 +1240,7 @@ class RepositoryController( BaseUIController, ItemRatings ):
repo = hg.repository( get_configured_ui(), repository.repo_path )
current_working_dir = os.getcwd()
# Update repository files for browsing.
update_for_browsing( trans, repository, current_working_dir, commit_message=commit_message )
update_repository( repo )
is_malicious = change_set_is_malicious( trans, id, repository.tip )
return trans.fill_template( '/webapps/community/repository/browse_repository.mako',
repo=repo,
@@ -1269,7 +1356,7 @@ class RepositoryController( BaseUIController, ItemRatings ):
repo.commit( user=trans.user.username, text=commit_message )
handle_email_alerts( trans, repository )
# Update the repository files for browsing.
update_for_browsing( trans, repository, current_working_dir, commit_message=commit_message )
update_repository( repo )
# Get the new repository tip.
repo = hg.repository( get_configured_ui(), repo_dir )
if tip != repository.tip:
@@ -1567,7 +1654,7 @@ class RepositoryController( BaseUIController, ItemRatings ):
status = params.get( 'status', 'done' )
repository = get_repository( trans, id )
repo = hg.repository( get_configured_ui(), repository.repo_path )
ctx = get_changectx_for_changeset( trans, repo, ctx_str )
ctx = get_changectx_for_changeset( repo, ctx_str )
if ctx is None:
message = "Repository does not include changeset revision '%s'." % str( ctx_str )
status = 'error'
@@ -1581,7 +1668,7 @@ class RepositoryController( BaseUIController, ItemRatings ):
anchors = modified + added + removed + deleted + unknown + ignored + clean
diffs = []
for diff in patch.diff( repo, node1=ctx_parent.node(), node2=ctx.node() ):
diffs.append( self.to_html_escaped( diff ) )
diffs.append( to_html_escaped( diff ) )
is_malicious = change_set_is_malicious( trans, id, repository.tip )
return trans.fill_template( '/webapps/community/repository/view_changeset.mako',
repository=repository,
@@ -1802,6 +1889,82 @@ class RepositoryController( BaseUIController, ItemRatings ):
changeset_revision=changeset_revision,
message=message,
status='error' ) )
@web.expose
def load_invalid_tool( self, trans, repository_id, tool_config, changeset_revision, **kwd ):
params = util.Params( kwd )
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'error' )
webapp = params.get( 'webapp', 'community' )
repository = get_repository( trans, repository_id )
repo_dir = repository.repo_path
repo = hg.repository( get_configured_ui(), repo_dir )
ctx = get_changectx_for_changeset( repo, changeset_revision )
invalid_message = ''
if changeset_revision == repository.tip:
for root, dirs, files in os.walk( repo_dir ):
found = False
for name in files:
if name == tool_config:
tool_config_path = os.path.join( root, name )
found = True
break
if found:
break
metadata_dict, invalid_files = generate_metadata_for_repository_tip( trans, repository_id, ctx, changeset_revision, repo, repo_dir )
else:
for filename in ctx:
if filename == tool_config:
fctx = ctx[ filename ]
# Write the contents of datatypes_config.xml to a temporary file.
fh = tempfile.NamedTemporaryFile( 'w' )
tool_config_path = fh.name
fh.close()
fh = open( tool_config_path, 'w' )
fh.write( fctx.data() )
fh.close()
break
metadata_dict, invalid_files = generate_metadata_for_changeset_revision( trans, repository_id, ctx, changeset_revision, repo_dir )
for invalid_file_tup in invalid_files:
invalid_tool_config, invalid_msg = invalid_file_tup
if tool_config == invalid_tool_config:
invalid_message = invalid_msg
break
tool, message = load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_config_path )
tool_state = self.__new_state( trans )
is_malicious = change_set_is_malicious( trans, repository_id, repository.tip )
if changeset_revision != repository.tip:
try:
os.unlink( tool_config_path )
except:
pass
try:
if invalid_message:
message = invalid_message
return trans.fill_template( "/webapps/community/repository/tool_form.mako",
repository=repository,
changeset_revision=changeset_revision,
tool=tool,
tool_state=tool_state,
is_malicious=is_malicious,
webapp=webapp,
message=message,
status='error' )
except Exception, e:
message = "This tool is invalid because: %s." % str( e )
if webapp == 'galaxy':
return trans.response.send_redirect( web.url_for( controller='repository',
action='preview_tools_in_changeset',
repository_id=repository_id,
changeset_revision=changeset_revision,
message=message,
status='error' ) )
return trans.response.send_redirect( web.url_for( controller='repository',
action='browse_repositories',
operation='view_or_manage_repository',
id=repository_id,
changeset_revision=changeset_revision,
message=message,
status='error' ) )
def __new_state( self, trans, all_pages=False ):
"""
Create a new `DefaultToolState` for this tool. It will not be initialized
@@ -1919,20 +2082,20 @@ class RepositoryController( BaseUIController, ItemRatings ):
trans.response.headers['Pragma'] = 'no-cache'
trans.response.headers['Expires'] = '0'
if is_gzip( file_path ):
to_html = self.to_html_str( '\ngzip compressed file\n' )
to_html = to_html_str( '\ngzip compressed file\n' )
elif is_bz2( file_path ):
to_html = self.to_html_str( '\nbz2 compressed file\n' )
to_html = to_html_str( '\nbz2 compressed file\n' )
elif check_zip( file_path ):
to_html = self.to_html_str( '\nzip compressed file\n' )
to_html = to_html_str( '\nzip compressed file\n' )
elif check_binary( file_path ):
to_html = self.to_html_str( '\nBinary file\n' )
to_html = to_html_str( '\nBinary file\n' )
else:
to_html = ''
for i, line in enumerate( open( file_path ) ):
to_html = '%s%s' % ( to_html, self.to_html_str( line ) )
to_html = '%s%s' % ( to_html, to_html_str( line ) )
if len( to_html ) > MAX_CONTENT_SIZE:
large_str = '\nFile contents truncated because file size is larger than maximum viewing size of %s\n' % util.nice_size( MAX_CONTENT_SIZE )
to_html = '%s%s' % ( to_html, self.to_html_str( large_str ) )
to_html = '%s%s' % ( to_html, to_html_str( large_str ) )
break
return to_html
@web.expose
@@ -1941,34 +2104,6 @@ class RepositoryController( BaseUIController, ItemRatings ):
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
return trans.fill_template( '/webapps/community/repository/help.mako', message=message, status=status, **kwd )
def to_html_escaped( self, text ):
"""Translates the characters in text to html values"""
translated = []
for c in text:
if c in [ '\r\n', '\n', ' ', '\t' ] or c in VALID_CHARS:
translated.append( c )
elif c in MAPPED_CHARS:
translated.append( MAPPED_CHARS[ c ] )
else:
translated.append( 'X' )
return ''.join( translated )
def to_html_str( self, text ):
"""Translates the characters in text to sn html string"""
translated = []
for c in text:
if c in VALID_CHARS:
translated.append( c )
elif c in MAPPED_CHARS:
translated.append( MAPPED_CHARS[ c ] )
elif c == ' ':
translated.append( '&nbsp;' )
elif c == '\t':
translated.append( '&nbsp;&nbsp;&nbsp;&nbsp;' )
elif c == '\n':
translated.append( '<br/>' )
elif c not in [ '\r' ]:
translated.append( 'X' )
return ''.join( translated )
def __build_allow_push_select_field( self, trans, current_push_list, selected_value='none' ):
options = []
for user in trans.sa_session.query( trans.model.User ):
lib/galaxy/webapps/community/controllers/upload.py
+19 -33
View File
@@ -3,7 +3,10 @@ from galaxy.web.base.controller import *
from galaxy.model.orm import *
from galaxy.datatypes.checkers import *
from common import *
from galaxy.util.shed_util import handle_sample_tool_data_table_conf_file
from galaxy.util.shed_util import get_configured_ui, handle_sample_tool_data_table_conf_file
from galaxy import eggs
eggs.require('mercurial')
from mercurial import hg, ui, commands
log = logging.getLogger( __name__ )
@@ -119,14 +122,9 @@ class UploadController( BaseUIController ):
else:
content_alert_str = ''
commands.add( repo.ui, repo, full_path )
try:
commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message )
except Exception, e:
# I never have a problem with commands.commit on a Mac, but in the test/production
# tool shed environment, it occasionally throws a "TypeError: array item must be char"
# exception. If this happens, we'll try the following.
repo.dirstate.write()
repo.commit( user=trans.user.username, text=commit_message )
# Convert from unicode to prevent "TypeError: array item must be char"
full_path = full_path.encode( 'ascii', 'replace' )
commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message )
if full_path.endswith( 'tool_data_table_conf.xml.sample' ):
# Handle the special case where a tool_data_table_conf.xml.sample
# file is being uploaded by parsing the file and adding new entries
@@ -136,15 +134,14 @@ class UploadController( BaseUIController ):
if error:
message = '%s<br/>%s' % ( message, error_message )
if full_path.endswith( '.loc.sample' ):
# Handle the special case where a xxx.loc.sample file is
# being uploaded by copying it to ~/tool-data/xxx.loc.
copy_sample_loc_file( trans.app, full_path )
# Handle the special case where a xxx.loc.sample file is being uploaded by copying it to ~/tool-data/xxx.loc.
copy_sample_file( trans.app, full_path )
# See if the content of the change set was valid.
admin_only = len( repository.downloadable_revisions ) != 1
handle_email_alerts( trans, repository, content_alert_str=content_alert_str, new_repo_alert=new_repo_alert, admin_only=admin_only )
if ok:
# Update the repository files for browsing.
update_for_browsing( trans, repository, current_working_dir, commit_message=commit_message )
update_repository( repo )
# Get the new repository tip.
if tip != repository.tip:
if ( isgzip or isbz2 ) and uncompress_file:
@@ -201,18 +198,18 @@ class UploadController( BaseUIController ):
repo = hg.repository( get_configured_ui(), repo_dir )
files_to_remove = []
content_alert_str = ''
undesirable_dirs_removed = 0
undesirable_files_removed = 0
ok, message = self.__check_archive( tar )
if not ok:
tar.close()
uploaded_file.close()
return ok, message, files_to_remove, content_alert_str
return ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed
else:
if upload_point is not None:
full_path = os.path.abspath( os.path.join( repo_dir, upload_point ) )
else:
full_path = os.path.abspath( repo_dir )
undesirable_dirs_removed = 0
undesirable_files_removed = 0
filenames_in_archive = []
for tarinfo_obj in tar.getmembers():
ok = os.path.basename( tarinfo_obj.name ) not in undesirable_files
@@ -249,14 +246,11 @@ class UploadController( BaseUIController ):
if full_name not in filenames_in_archive:
files_to_remove.append( full_name )
for repo_file in files_to_remove:
# Remove files in the repository (relative to the upload point)
# that are not in the uploaded archive.
# Remove files in the repository (relative to the upload point) that are not in the uploaded archive.
try:
commands.remove( repo.ui, repo, repo_file, force=True )
except Exception, e:
# I never have a problem with commands.remove on a Mac, but in the test/production
# tool shed environment, it throws an exception whenever I delete all files from a
# repository. If this happens, we'll try the following.
log.debug( "Error removing files using the mercurial API, so trying a different approach, the error was: %s" % str( e ))
relative_selected_file = selected_file.split( 'repo_%d' % repository.id )[1].lstrip( '/' )
repo.dirstate.remove( relative_selected_file )
repo.dirstate.write()
@@ -290,19 +284,11 @@ class UploadController( BaseUIController ):
# appending them to the shed's tool_data_table_conf.xml file on disk.
error, message = handle_sample_tool_data_table_conf_file( trans.app, filename_in_archive )
if error:
return False, message, files_to_remove, content_alert_str
return False, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed
if filename_in_archive.endswith( '.loc.sample' ):
# Handle the special case where a xxx.loc.sample file is
# being uploaded by copying it to ~/tool-data/xxx.loc.
copy_sample_loc_file( trans.app, filename_in_archive )
try:
commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message )
except Exception, e:
# I never have a problem with commands.commit on a Mac, but in the test/production
# tool shed environment, it occasionally throws a "TypeError: array item must be char"
# exception. If this happens, we'll try the following.
repo.dirstate.write()
repo.commit( user=trans.user.username, text=commit_message )
# Handle the special case where a xxx.loc.sample file is being uploaded by copying it to ~/tool-data/xxx.loc.
copy_sample_file( trans.app, filename_in_archive )
commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message )
# See if the content of the change set was valid.
admin_only = len( repository.downloadable_revisions ) != 1
handle_email_alerts( trans, repository, content_alert_str=content_alert_str, new_repo_alert=new_repo_alert, admin_only=admin_only )
lib/galaxy/webapps/community/framework/middleware/hg.py
-1
View File
@@ -3,7 +3,6 @@ Middleware for handling hg authentication for users pushing change sets to local
"""
import os, logging
from sqlalchemy import *
from mercurial import ui, hg
from paste.auth.basic import AuthBasicAuthenticator
from paste.httpheaders import REMOTE_USER, AUTH_TYPE
lib/galaxy/webapps/community/model/__init__.py
+4
View File
@@ -9,7 +9,11 @@ from galaxy import util
from galaxy.util.bunch import Bunch
from galaxy.util.hash_util import *
from galaxy.web.form_builder import *
from galaxy import eggs
eggs.require('mercurial')
from mercurial import hg, ui
log = logging.getLogger( __name__ )
class User( object ):
lib/galaxy/webapps/community/model/mapping.py
+2
View File
@@ -13,6 +13,7 @@ from galaxy.model.orm import *
from galaxy.model.orm.ext.assignmapper import *
from galaxy.model.custom_types import *
from galaxy.util.bunch import Bunch
from galaxy.util.shed_util import ShedCounter
from galaxy.webapps.community.security import CommunityRBACAgent
metadata = MetaData()
@@ -247,4 +248,5 @@ def init( file_path, url, engine_options={}, create_tables=False ):
result.create_tables = create_tables
#load local galaxy security policy
result.security_agent = CommunityRBACAgent( result )
result.shed_counter = ShedCounter( result )
return result
lib/galaxy/webapps/community/security/__init__.py
-1
View File
@@ -6,7 +6,6 @@ from datetime import datetime, timedelta
from galaxy.util.bunch import Bunch
from galaxy.util import listify
from galaxy.model.orm import *
from mercurial import hg, ui
log = logging.getLogger(__name__)
lib/galaxy/workflow/modules.py
+3 -2
View File
@@ -199,7 +199,7 @@ class ToolModule( WorkflowModule ):
# TODO: If workflows are ever enhanced to use tool version
# in addition to tool id, enhance the selection process here
# to retrieve the correct version of the tool.
tool_version = self.__get_tool_version( trans, tool_id )
tool_version = Class.__get_tool_version( trans, tool_id )
if tool_version:
tool_version_ids = tool_version.get_version_ids( trans.app )
for tool_version_id in tool_version_ids:
@@ -219,7 +219,8 @@ class ToolModule( WorkflowModule ):
module.post_job_actions = pjadict
return module
return None
def __get_tool_version( self, trans, tool_id ):
@classmethod
def __get_tool_version( cls, trans, tool_id ):
# Return a ToolVersion if one exists for tool_id.
return trans.sa_session.query( trans.app.model.ToolVersion ) \
.filter( trans.app.model.ToolVersion.table.c.tool_id == tool_id ) \
run.sh
Executable → Regular
+14 -1
View File
@@ -52,4 +52,17 @@ if [ $FETCH_EGGS -eq 1 ]; then
fi
fi
python ./scripts/paster.py serve universe_wsgi.ini $@
if [ -n "$GALAXY_RUN_ALL" ]; then
servers=`sed -n 's/^\[server:\(.*\)\]/\1/ p' universe_wsgi.ini | xargs echo`
daemon=`echo "$@" | grep -q daemon`
if [ $? -ne 0 ]; then
echo 'ERROR: $GALAXY_RUN_ALL cannot be used without the `--daemon` or `--stop-daemon` arguments to run.sh'
exit 1
fi
for server in $servers; do
echo "Handling $server with log file $server.log..."
python ./scripts/paster.py serve universe_wsgi.ini --server-name=$server --pid-file=$server.pid --log-file=$server.log $@
done
else
python ./scripts/paster.py serve universe_wsgi.ini $@
fi
run_functional_tests.sh
+18
View File
@@ -20,6 +20,24 @@ elif [ $1 = '-list' ]; then
echo "==========================================================================================================================================="
echo "'run_functional_tests.sh -id bbb' for testing one tool with id 'bbb' ('bbb' is the tool id)"
echo "'run_functional_tests.sh -sid ccc' for testing one section with sid 'ccc' ('ccc' is the string after 'section::')"
elif [ $1 = '-migrated' ]; then
if [ ! $2 ]; then
python ./scripts/functional_tests.py -v functional.test_toolbox --with-nosehtml --html-report-file run_functional_tests.html -migrated
elif [ $2 = '-id' ]; then
# TODO: This option is not tested...
python ./scripts/functional_tests.py -v functional.test_toolbox:TestForTool_$3 --with-nosehtml --html-report-file run_functional_tests.html -migrated
else
python ./scripts/functional_tests.py -v functional.test_toolbox --with-nosehtml --html-report-file run_functional_tests.html -migrated
fi
elif [ $1 = '-installed' ]; then
if [ ! $2 ]; then
python ./scripts/functional_tests.py -v functional.test_toolbox --with-nosehtml --html-report-file run_functional_tests.html -installed
elif [ $2 = '-id' ]; then
# TODO: This option is not tested...
python ./scripts/functional_tests.py -v functional.test_toolbox:TestForTool_$3 --with-nosehtml --html-report-file run_functional_tests.html -installed
else
python ./scripts/functional_tests.py -v functional.test_toolbox --with-nosehtml --html-report-file run_functional_tests.html -installed
fi
else
python ./scripts/functional_tests.py -v --with-nosehtml --html-report-file run_functional_tests.html $1
fi
scripts/api/README
-1
View File
@@ -3,7 +3,6 @@ until the documentation is written.
Set these options in universe_wsgi.ini and start the server:
enable_api = True
admin_users = you@example.org
library_import_dir = /path/to/some/directory
scripts/check_galaxy.py
+1 -1
View File
@@ -249,7 +249,7 @@ class Browser:
#return((p.id, p.status))
def diff(self):
self.get("/datasets/%s/display/index" % self.id)
self.get("/datasets/%s/display/display?to_ext=bed" % self.id)
data = tc.browser.get_html()
tmp = tempfile.mkstemp()
dprint("tmp file: %s" % tmp[1])
scripts/check_python.py
+2 -2
View File
@@ -1,14 +1,14 @@
import os, sys
msg = """ERROR: Your Python version is: %s
Galaxy is currently supported on Python 2.4, 2.5 and 2.6. To run Galaxy,
Galaxy is currently supported on Python 2.5, 2.6 and 2.7. To run Galaxy,
please download and install a supported version from python.org. If a
supported version is installed but is not your default, getgalaxy.org
contains instructions on how to force Galaxy to use a different version.""" % sys.version[:3]
def check_python():
try:
assert sys.version_info[:2] >= ( 2, 4 ) and sys.version_info[:2] <= ( 2, 7 )
assert sys.version_info[:2] >= ( 2, 5 ) and sys.version_info[:2] <= ( 2, 7 )
except AssertionError:
print >>sys.stderr, msg
raise
scripts/cleanup_datasets/cleanup_datasets.py
+1 -1
View File
@@ -505,7 +505,7 @@ def _purge_folder( folder, app, remove_from_disk, info_only = False ):
class CleanupDatasetsApplication( object ):
"""Encapsulates the state of a Universe application"""
def __init__( self, config ):
if config.database_connection is None:
if config.database_connection is False:
config.database_connection = "sqlite:///%s?isolation_level=IMMEDIATE" % config.database
self.object_store = build_object_store_from_config( config )
# Setup the database engine and ORM
scripts/functional_tests.py
+190 -105
View File
@@ -1,6 +1,6 @@
#!/usr/bin/env python
import os, sys, shutil
import os, sys, shutil, tempfile, re
# Assume we are run from the galaxy root directory, add lib to the python path
cwd = os.getcwd()
@@ -32,6 +32,13 @@ from galaxy.app import UniverseApplication
from galaxy.web import buildapp
from galaxy import tools
from galaxy.util import bunch
from galaxy import util
from galaxy.util.json import to_json_string
import nose.core
import nose.config
import nose.loader
import nose.plugins.manager
log = logging.getLogger( "functional_tests.py" )
@@ -40,28 +47,131 @@ default_galaxy_test_port_min = 8000
default_galaxy_test_port_max = 9999
default_galaxy_locales = 'en'
default_galaxy_test_file_dir = "test-data"
migrated_tool_panel_config = 'migrated_tools_conf.xml'
installed_tool_panel_configs = [ 'shed_tool_conf.xml' ]
def main():
def parse_tool_panel_config( config, shed_tools_dict ):
"""
Parse a shed-related tool panel config to generate the shed_tools_dict. This only happens when testing tools installed from the tool shed.
"""
last_galaxy_test_file_dir = None
last_tested_repository_name = None
last_tested_changeset_revision = None
tree = util.parse_xml( config )
root = tree.getroot()
for elem in root:
if elem.tag == 'tool':
galaxy_test_file_dir, \
last_tested_repository_name, \
last_tested_changeset_revision = get_installed_repository_info( elem,
last_galaxy_test_file_dir,
last_tested_repository_name,
last_tested_changeset_revision )
if galaxy_test_file_dir:
if galaxy_test_file_dir != last_galaxy_test_file_dir:
if not os.path.isabs( galaxy_test_file_dir ):
galaxy_test_file_dir = os.path.join( os.getcwd(), galaxy_test_file_dir )
guid = elem.get( 'guid' )
shed_tools_dict[ guid ] = galaxy_test_file_dir
last_galaxy_test_file_dir = galaxy_test_file_dir
elif elem.tag == 'section':
for section_elem in elem:
if section_elem.tag == 'tool':
galaxy_test_file_dir, \
last_tested_repository_name, \
last_tested_changeset_revision = get_installed_repository_info( section_elem,
last_galaxy_test_file_dir,
last_tested_repository_name,
last_tested_changeset_revision )
if galaxy_test_file_dir:
if galaxy_test_file_dir != last_galaxy_test_file_dir:
if not os.path.isabs( galaxy_test_file_dir ):
galaxy_test_file_dir = os.path.join( os.getcwd(), galaxy_test_file_dir )
guid = section_elem.get( 'guid' )
shed_tools_dict[ guid ] = galaxy_test_file_dir
last_galaxy_test_file_dir = galaxy_test_file_dir
return shed_tools_dict
def get_installed_repository_info( elem, last_galaxy_test_file_dir, last_tested_repository_name, last_tested_changeset_revision ):
"""
Return the GALAXY_TEST_FILE_DIR, the containing repository name and the change set revision for the tool elem.
This only happens when testing tools installed from the tool shed.
"""
tool_config_path = elem.get( 'file' )
installed_tool_path_items = tool_config_path.split( '/repos/' )
sans_shed = installed_tool_path_items[ 1 ]
path_items = sans_shed.split( '/' )
repository_owner = path_items[ 0 ]
repository_name = path_items[ 1 ]
changeset_revision = path_items[ 2 ]
if repository_name != last_tested_repository_name or changeset_revision != last_tested_changeset_revision:
# Locate the test-data directory.
installed_tool_path = os.path.join( installed_tool_path_items[ 0 ], 'repos', repository_owner, repository_name, changeset_revision )
for root, dirs, files in os.walk( installed_tool_path ):
if 'test-data' in dirs:
return os.path.join( root, 'test-data' ), repository_name, changeset_revision
return None, repository_name, changeset_revision
return last_galaxy_test_file_dir, last_tested_repository_name, last_tested_changeset_revision
def run_tests( test_config ):
loader = nose.loader.TestLoader( config=test_config )
plug_loader = test_config.plugins.prepareTestLoader( loader )
if plug_loader is not None:
loader = plug_loader
tests = loader.loadTestsFromNames( test_config.testNames )
test_runner = nose.core.TextTestRunner( stream=test_config.stream,
verbosity=test_config.verbosity,
config=test_config )
plug_runner = test_config.plugins.prepareTestRunner( test_runner )
if plug_runner is not None:
test_runner = plug_runner
return test_runner.run( tests )
def main():
# ---- Configuration ------------------------------------------------------
galaxy_test_host = os.environ.get( 'GALAXY_TEST_HOST', default_galaxy_test_host )
galaxy_test_port = os.environ.get( 'GALAXY_TEST_PORT', None )
galaxy_test_save = os.environ.get( 'GALAXY_TEST_SAVE', None)
if 'HTTP_ACCEPT_LANGUAGE' not in os.environ:
os.environ['HTTP_ACCEPT_LANGUAGE'] = default_galaxy_locales
galaxy_test_file_dir = os.environ.get( 'GALAXY_TEST_FILE_DIR', default_galaxy_test_file_dir )
if not os.path.isabs( galaxy_test_file_dir ):
galaxy_test_file_dir = os.path.join( os.getcwd(), galaxy_test_file_dir )
start_server = 'GALAXY_TEST_EXTERNAL' not in os.environ
tool_path = os.environ.get( 'GALAXY_TEST_TOOL_PATH', 'tools' )
tool_config_file = os.environ.get( 'GALAXY_TEST_TOOL_CONF', 'tool_conf.xml.sample' )
tool_data_table_config_path = 'tool_data_table_conf.xml'
tool_dependency_dir = os.environ.get( 'GALAXY_TOOL_DEPENDENCY_DIR', None )
use_distributed_object_store = os.environ.get( 'GALAXY_USE_DISTRIBUTED_OBJECT_STORE', False )
if 'HTTP_ACCEPT_LANGUAGE' not in os.environ:
os.environ[ 'HTTP_ACCEPT_LANGUAGE' ] = default_galaxy_locales
testing_migrated_tools = '-migrated' in sys.argv
testing_installed_tools = '-installed' in sys.argv
if testing_migrated_tools or testing_installed_tools:
sys.argv.pop()
# Store a jsonified dictionary of tool_id : GALAXY_TEST_FILE_DIR pairs.
galaxy_tool_shed_test_file = 'shed_tools_dict'
# We need the upload tool for functional tests, so we'll create a temporary tool panel config that defines it.
fd, tmp_tool_panel_conf = tempfile.mkstemp()
os.write( fd, '<?xml version="1.0"?>\n' )
os.write( fd, '<toolbox>\n' )
os.write( fd, '<tool file="data_source/upload.xml"/>\n' )
os.write( fd, '</toolbox>\n' )
os.close( fd )
tool_config_file = tmp_tool_panel_conf
galaxy_test_file_dir = None
library_import_dir = None
user_library_import_dir = None
# Exclude all files except test_toolbox.py.
ignore_files = ( re.compile( r'^test_[adghlmsu]*' ), re.compile( r'^test_ta*' ) )
else:
tool_config_file = os.environ.get( 'GALAXY_TEST_TOOL_CONF', 'tool_conf.xml.sample' )
galaxy_test_file_dir = os.environ.get( 'GALAXY_TEST_FILE_DIR', default_galaxy_test_file_dir )
if not os.path.isabs( galaxy_test_file_dir ):
galaxy_test_file_dir = os.path.join( os.getcwd(), galaxy_test_file_dir )
library_import_dir = galaxy_test_file_dir
user_library_import_dir = os.path.join( galaxy_test_file_dir, 'users' )
ignore_files = ()
start_server = 'GALAXY_TEST_EXTERNAL' not in os.environ
if os.path.exists( 'tool_data_table_conf.test.xml' ):
tool_data_table_config_path = 'tool_data_table_conf.test.xml'
else:
tool_data_table_config_path = 'tool_data_table_conf.xml'
tool_dependency_dir = os.environ.get( 'GALAXY_TOOL_DEPENDENCY_DIR', None )
use_distributed_object_store = os.environ.get( 'GALAXY_USE_DISTRIBUTED_OBJECT_STORE', False )
if start_server:
psu_production = False
galaxy_test_proxy_port = None
@@ -126,38 +236,21 @@ def main():
try:
os.makedirs( dir )
except OSError:
pass
pass
print "Database connection:", database_connection
# What requires these?
# handy for (eg) functional tests to save outputs?
if galaxy_test_save:
os.environ['GALAXY_TEST_SAVE'] = galaxy_test_save
# pass in through script setenv
# will leave a copy of ALL test validate files
os.environ['GALAXY_TEST_HOST'] = galaxy_test_host
os.environ['GALAXY_TEST_FILE_DIR'] = galaxy_test_file_dir
# ---- Build Application --------------------------------------------------
app = None
# ---- Build Application --------------------------------------------------
app = None
if start_server:
global_conf = { '__file__' : 'universe_wsgi.ini.sample' }
if psu_production:
global_conf = None
if not database_connection.startswith( 'sqlite://' ):
kwargs['database_engine_option_max_overflow'] = '20'
kwargs[ 'database_engine_option_max_overflow' ] = '20'
if tool_dependency_dir is not None:
kwargs['tool_dependency_dir'] = tool_dependency_dir
kwargs[ 'tool_dependency_dir' ] = tool_dependency_dir
if use_distributed_object_store:
kwargs['object_store'] = 'distributed'
kwargs['distributed_object_store_config_file'] = 'distributed_object_store_conf.xml.sample'
kwargs[ 'object_store' ] = 'distributed'
kwargs[ 'distributed_object_store_config_file' ] = 'distributed_object_store_conf.xml.sample'
# Build the Universe Application
app = UniverseApplication( job_queue_workers = 5,
id_secret = 'changethisinproductiontoo',
@@ -179,21 +272,17 @@ def main():
allow_user_deletion = True,
admin_users = 'test@bx.psu.edu',
allow_library_path_paste = True,
library_import_dir = galaxy_test_file_dir,
user_library_import_dir = os.path.join( galaxy_test_file_dir, 'users' ),
library_import_dir = library_import_dir,
user_library_import_dir = user_library_import_dir,
global_conf = global_conf,
running_functional_tests=True,
**kwargs )
log.info( "Embedded Universe application started" );
log.info( "Embedded Universe application started" )
# ---- Run webserver ------------------------------------------------------
server = None
if start_server:
webapp = buildapp.app_factory( dict(), use_translogger = False, static_enabled = False, app=app )
webapp = buildapp.app_factory( dict(), use_translogger=False, static_enabled=False, app=app )
if galaxy_test_port is not None:
server = httpserver.serve( webapp, host=galaxy_test_host, port=galaxy_test_port, start_loop=False )
else:
@@ -217,7 +306,6 @@ def main():
t = threading.Thread( target=server.serve_forever )
t.start()
# Test if the server is up
for i in range( 10 ):
conn = httplib.HTTPConnection( galaxy_test_host, galaxy_test_port ) # directly test the app, not the proxy
@@ -227,85 +315,83 @@ def main():
time.sleep( 0.1 )
else:
raise Exception( "Test HTTP server did not return '200 OK' after 10 tries" )
# Test if the proxy server is up
if psu_production:
conn = httplib.HTTPConnection( galaxy_test_host, galaxy_test_proxy_port ) # directly test the app, not the proxy
conn.request( "GET", "/" )
if not conn.getresponse().status == 200:
raise Exception( "Test HTTP proxy server did not return '200 OK'" )
log.info( "Embedded web server started" )
# ---- Load toolbox for generated tests -----------------------------------
# We don't add the tests to the path until everything is up and running
new_path = [ os.path.join( cwd, "test" ) ]
new_path.extend( sys.path[1:] )
sys.path = new_path
import functional.test_toolbox
if app:
# TODO: provisions for loading toolbox from file when using external server
functional.test_toolbox.toolbox = app.toolbox
functional.test_toolbox.build_tests()
else:
# FIXME: This doesn't work at all now that toolbox requires an 'app' instance
# (to get at datatypes, might just pass a datatype registry directly)
datatypes_registry = galaxy.datatypes.registry.Registry()
datatypes_registry.load_datatypes()
my_app = bunch.Bunch( datatypes_registry )
test_toolbox.toolbox = tools.ToolBox( 'tool_conf.xml.test', 'tools', my_app )
# ---- Find tests ---------------------------------------------------------
if galaxy_test_proxy_port:
log.info( "Functional tests will be run against %s:%s" % ( galaxy_test_host, galaxy_test_proxy_port ) )
else:
log.info( "Functional tests will be run against %s:%s" % ( galaxy_test_host, galaxy_test_port ) )
success = False
try:
import nose.core
import nose.config
import nose.loader
import nose.plugins.manager
test_config = nose.config.Config( env = os.environ, plugins=nose.plugins.manager.DefaultPluginManager() )
test_config.configure( sys.argv )
loader = nose.loader.TestLoader( config = test_config )
plug_loader = test_config.plugins.prepareTestLoader( loader )
if plug_loader is not None:
loader = plug_loader
tests = loader.loadTestsFromNames( test_config.testNames )
test_runner = nose.core.TextTestRunner(
stream = test_config.stream,
verbosity = test_config.verbosity,
config = test_config)
plug_runner = test_config.plugins.prepareTestRunner( test_runner )
if plug_runner is not None:
test_runner = plug_runner
result = test_runner.run( tests )
success = result.wasSuccessful()
# What requires these? Handy for (eg) functional tests to save outputs?
if galaxy_test_save:
os.environ[ 'GALAXY_TEST_SAVE' ] = galaxy_test_save
# Pass in through script setenv, will leave a copy of ALL test validate files
os.environ[ 'GALAXY_TEST_HOST' ] = galaxy_test_host
if testing_migrated_tools or testing_installed_tools:
shed_tools_dict = {}
if testing_migrated_tools:
shed_tools_dict = parse_tool_panel_config( migrated_tool_panel_config, shed_tools_dict )
elif testing_installed_tools:
for shed_tool_config in installed_tool_panel_configs:
shed_tools_dict = parse_tool_panel_config( shed_tool_config, shed_tools_dict )
# Persist the shed_tools_dict to the galaxy_tool_shed_test_file.
shed_tools_file = open( galaxy_tool_shed_test_file, 'w' )
shed_tools_file.write( to_json_string( shed_tools_dict ) )
shed_tools_file.close()
if not os.path.isabs( galaxy_tool_shed_test_file ):
galaxy_tool_shed_test_file = os.path.join( os.getcwd(), galaxy_tool_shed_test_file )
os.environ[ 'GALAXY_TOOL_SHED_TEST_FILE' ] = galaxy_tool_shed_test_file
if testing_installed_tools:
# Eliminate the migrated_tool_panel_config from the app's tool_configs, append the list of installed_tool_panel_configs,
# and reload the app's toolbox.
relative_migrated_tool_panel_config = os.path.join( app.config.root, migrated_tool_panel_config )
tool_configs = app.config.tool_configs
if relative_migrated_tool_panel_config in tool_configs:
tool_configs.remove( relative_migrated_tool_panel_config )
for installed_tool_panel_config in installed_tool_panel_configs:
tool_configs.append( installed_tool_panel_config )
app.toolbox = tools.ToolBox( tool_configs, app.config.tool_path, app )
functional.test_toolbox.toolbox = app.toolbox
functional.test_toolbox.build_tests( testing_shed_tools=True )
test_config = nose.config.Config( env=os.environ, ignoreFiles=ignore_files, plugins=nose.plugins.manager.DefaultPluginManager() )
test_config.configure( sys.argv )
result = run_tests( test_config )
success = result.wasSuccessful()
try:
os.unlink( tmp_tool_panel_conf )
except:
log.info( "Unable to remove temporary file: %s" % tmp_tool_panel_conf )
try:
os.unlink( galaxy_tool_shed_test_file )
except:
log.info( "Unable to remove file: %s" % galaxy_tool_shed_test_file )
else:
functional.test_toolbox.toolbox = app.toolbox
functional.test_toolbox.build_tests()
if galaxy_test_file_dir:
os.environ[ 'GALAXY_TEST_FILE_DIR' ] = galaxy_test_file_dir
test_config = nose.config.Config( env=os.environ, ignoreFiles=ignore_files, plugins=nose.plugins.manager.DefaultPluginManager() )
test_config.configure( sys.argv )
result = run_tests( test_config )
success = result.wasSuccessful()
except:
log.exception( "Failure running tests" )
log.info( "Shutting down" )
# ---- Teardown -----------------------------------------------------------
# ---- Tear down -----------------------------------------------------------
if server:
log.info( "Shutting down embedded web server" )
server.server_close()
@@ -330,7 +416,6 @@ def main():
shutil.rmtree( dir )
except:
pass
if success:
return 0
else:
scripts/galaxy_messaging/server/data_transfer.py
+1 -1
View File
@@ -177,7 +177,7 @@ class DataTransfer( object ):
self.update_status( SampleDataset.transfer_status.ADD_TO_LIBRARY )
try:
data = {}
data[ 'folder_id' ] = api.encode_id( self.config_id_secret, '%s.%s' % ( 'folder', self.folder_id ) )
data[ 'folder_id' ] = 'F%s' % api.encode_id( self.config_id_secret, self.folder_id )
data[ 'file_type' ] = 'auto'
data[ 'server_dir' ] = self.server_dir
data[ 'dbkey' ] = ''
static/june_2007_style/base.less
+96 -41
View File
@@ -314,6 +314,11 @@ div.unified-panel-body {
-webkit-border-radius: 1em;
}
#dialog-box.dialog-box{
min-width: 660px;
margin: -250px 0 0 -330px;
}
.dialog-box {
border: solid #999 1px;
background: white;
@@ -407,14 +412,26 @@ div.unified-panel-body {
}
.title {
.navbar.brand();
position: absolute;
left: 0;
top: 0;
font-family: verdana;
font-weight: bold;
font-size: 20px;
line-height: 1;
color: white;
// Override margin and padding due to shorter navbar height
padding: 5px 20px 12px;
margin-left: -15px;
z-index: 2000;
img {
display: inline;
width: 26px;
vertical-align: top;
}
a {
color: white;
text-decoration: none;
@@ -1210,6 +1227,10 @@ a.action-button {
.menubutton {
.btn();
&:hover { .btn.hover(); }
&:active { .btn.active(); }
&:focus { .tab-focus(); }
display: inline-block;
cursor: pointer;
position: relative;
@@ -1223,11 +1244,21 @@ a.action-button {
// padding: 1px 0.25em;
// margin: -1px -0.25em;
a {
text-decoration: none;
}
.label {
position: relative;
// display: block;
display: inline-block;
border-right: none;
text-decoration: none;
text-align: left;
// The following properties truncate the text and force the button to have one line
max-height: 2*@baseLineHeight;
line-height: @baseLineHeight;
overflow: hidden;
text-overflow: ellipsis;
}
&.popup .label {
@@ -1235,21 +1266,16 @@ a.action-button {
padding-right: 6px;
}
&.popup {
&.popup, &.popup.split {
padding-right: 18px;
&:after {
margin-top: 6px;
position: absolute;
top: 2px;
right: 6px;
.caret();
}
}
&.popup.split {
&:after {
margin-top: 6px;
margin-left: 0px;
.caret();
}
}
}
// A split menu button, the main button has an action, the arrow causes the
@@ -1401,8 +1427,6 @@ div.permissionContainer {
padding: 0;
}
@import "base_sprites";
.tipsy {
padding: 5px;
font-size: 10px;
@@ -1521,19 +1545,19 @@ div.historyItem-failed_metadata {
div.historyItem-error {
// border-color: @history_error_border;
background: @history_error_bg;
.state-icon {
-sprite-group: history-states;
-sprite-image: data_error.png;
}
// .state-icon {
// -sprite-group: history-states;
// -sprite-image: data_error.png;
// }
}
div.historyItem-empty {
// border-color: @history_error_border;
background: @history_error_bg;
.state-icon {
-sprite-group: history-states;
-sprite-image: data_empty.png;
}
// .state-icon {
// -sprite-group: history-states;
// -sprite-image: data_empty.png;
// }
}
div.historyItem-running {
@@ -1561,11 +1585,11 @@ div.historyItem-upload {
}
div.historyItem-queued {
// background: @history_queued_bg;
.state-icon {
-sprite-group: history-states;
-sprite-image: data_queued.png;
}
background: @history_queued_bg;
// .state-icon {
// -sprite-group: history-states;
// -sprite-image: data_queued.png;
// }
}
div.historyItem-noPermission {
@@ -1606,21 +1630,15 @@ pre.peek {
// ==== Tool menu styles
body.toolMenuPage
.toolMenuContainer
{
background: white;
color: @base_text;
background: @menu_bg_over;
margin: 5px;
margin-right: 10px;
margin-left: 10px;
margin-top: 5px;
}
hr
{
border: none;
height: 0px;
margin-top: 0px;
.unified-panel-body .toolMenu {
padding: 10px;
}
div.toolSectionPad
@@ -1632,7 +1650,7 @@ div.toolSectionPad
}
div.toolSectionWrapper {
// margin-bottom: 5px;
margin-bottom: 5px;
}
div.toolSectionDetailsInner
@@ -1648,8 +1666,8 @@ div.toolSectionTitle
div.toolPanelLabel
{
//padding-top: 10px;
//padding-bottom: 5px;
padding-top: 10px;
padding-bottom: 5px;
font-weight: bold;
color: gray;
text-transform: uppercase;
@@ -1677,7 +1695,44 @@ div.toolSectionBody div.toolPanelLabel
div.toolTitleNoSection
{
padding-bottom: 0px;
padding-bottom: 5px;
font-weight: bold;
}
#tool-search {
padding-top: 5px;
padding-bottom: 10px;
position: relative;
}
// Dataset Display Styles
#loading_indicator{
position:fixed;
right:10px;
top:10px;
height:32px;
width:32px;
background:url(largespinner.gif);
}
#content_table td{
text-align:right;
white-space:nowrap;
padding:2px 10px;
}
#content_table td.stringalign{
text-align:left;
}
// ==== Integrated tool form styles
.toolMenuAndView .toolForm
{
float: left;
background-color: white;
margin: 10px;
}
@import "base_sprites";
static/june_2007_style/base_sprites.less.tmpl
+24
View File
@@ -87,6 +87,10 @@
-sprite-image: fugue/plus-circle.png;
}
.icon-button.plus-button {
-sprite-group: fugue;
-sprite-image: fugue/plus-button-bw.png;
}
.icon-button.plus-button:hover {
-sprite-group: fugue;
-sprite-image: fugue/plus-button.png;
}
@@ -94,9 +98,29 @@
-sprite-group: fugue;
-sprite-image: fugue/gear.png;
}
.icon-button.chart_curve {
-sprite-group: fugue;
-sprite-image: silk/chart_curve.png;
}
.text-and-autocomplete-select {
-sprite-group: fugue;
-sprite-image: fugue/control-270.png;
-sprite-horiz-position: right;
}
div.historyItem-error .state-icon {
-sprite-group: history-states;
-sprite-image: data_error.png;
}
div.historyItem-empty .state-icon {
-sprite-group: history-states;
-sprite-image: data_empty.png;
}
div.historyItem-queued .state-icon {
-sprite-group: history-states;
-sprite-image: data_queued.png;
}

Some files were not shown because too many files have changed in this diff Show More