diff --git a/.hgignore b/.hgignore index 59c2b3ce68b..72ceb2ad345 100644 --- a/.hgignore +++ b/.hgignore @@ -36,6 +36,7 @@ shed_tool_conf.xml tool_data_table_conf.xml tool_sheds_conf.xml integrated_tool_panel.xml +openid_conf.xml static/welcome.html.* static/welcome.html @@ -63,8 +64,12 @@ run_functional_tests.html # Jars tool-data/shared/jars/ +# CSS build artifacts. +*/variables.less +static/june_2007_style/blue/base_sprites.less + # Misc *.orig .DS_Store *.rej -*~ +*~ \ No newline at end of file diff --git a/community_wsgi.ini.sample b/community_wsgi.ini.sample index 7dff5b2da3d..4e2dbd712e0 100644 --- a/community_wsgi.ini.sample +++ b/community_wsgi.ini.sample @@ -4,7 +4,12 @@ use = egg:Paste#http port = 9009 + +# The address on which to listen. By default, only listen to localhost (the tool shed will not +# be accessible over the network). Use '0.0.0.0' to listen on all available network interfaces. +#host = 0.0.0.0 host = 127.0.0.1 + use_threadpool = true threadpool_workers = 10 diff --git a/contrib/universe_merger.py b/contrib/universe_merger.py new file mode 100644 index 00000000000..7790f970fbe --- /dev/null +++ b/contrib/universe_merger.py @@ -0,0 +1,87 @@ +#! /usr/bin/env python +''' +universe_merger.py + +Created by Anne Pajon on 31 Jan 2012 + +Copyright (c) 2012 Cancer Research UK - Cambridge Research Institute. + +This source file is licensed under the Academic Free License version +3.0 available at http://www.opensource.org/licenses/AFL-3.0. + +Permission is hereby granted to reproduce, translate, adapt, alter, +transform, modify, or arrange this source file (the "Original Work"); +to distribute or communicate copies of it under any license of your +choice that does not contradict the terms and conditions; to perform +or display the Original Work publicly. + +THE ORIGINAL WORK IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS +AND WITHOUT WARRANTY, EITHER EXPRESS OR IMPLIED, INCLUDING, WITHOUT +LIMITATION, THE WARRANTIES OF NON-INFRINGEMENT, MERCHANTABILITY OR +FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY OF +THE ORIGINAL WORK IS WITH YOU. + +Script for merging specific local Galaxy config universe_wsgi.ini.cri with default Galaxy universe_wsgi.ini.sample +''' +import ConfigParser +import sys +import optparse +import logging + +def main(): + # logging configuration + logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) + + # get the options + parser = optparse.OptionParser() + parser.add_option("-s", "--sample", dest="sample", action="store", help="path to Galaxy universe_wsgi.ini.sample file") + parser.add_option("-c", "--config", dest="config", action="store", help="path to your own universe_wsgi.ini file") + parser.add_option("-o", "--output", dest="output", action="store", help="path to the new merged universe_wsgi.ini.new file") + (options, args) = parser.parse_args() + + for option in ['sample', 'config']: + if getattr(options, option) == None: + print "Please supply a --%s parameter.\n" % (option) + parser.print_help() + sys.exit() + + config_sample = ConfigParser.RawConfigParser() + config_sample.read(options.sample) + config_sample_content = open(options.sample, 'r').read() + + config = ConfigParser.RawConfigParser() + config.read(options.config) + + logging.info("Merging your own config file %s into the sample one %s." % (options.config, options.sample)) + logging.info("---------- DIFFERENCE ANALYSIS BEGIN ----------") + for section in config.sections(): + if not config_sample.has_section(section): + logging.warning("-MISSING- section [%s] not found in sample file. It will be ignored." % section) + else: + for (name, value) in config.items(section): + if not config_sample.has_option(section, name): + if not "#%s" % name in config_sample_content: + logging.warning("-MISSING- section [%s] option '%s' not found in sample file. It will be ignored." % (section, name)) + else: + logging.info("-notset- section [%s] option '%s' not set in sample file. It will be added." % (section, name)) + config_sample.set(section, name, value) + else: + if not config_sample.get(section, name) == value: + logging.info("- diff - section [%s] option '%s' has different value ('%s':'%s'). It will be modified." % (section, name, config_sample.get(section, name), value)) + config_sample.set(section, name, value) + logging.info("---------- DIFFERENCE ANALYSIS END ----------") + + if options.output: + outputfile = open(options.output, 'w') + config_sample.write(outputfile) + outputfile.close() + else: + #print "----------" + #config_sample.write(sys.stdout) + #print "----------" + logging.info("use -o OUTPUT to write the merged configuration into a file.") + + logging.info("read Galaxy universe_wsgi.ini.sample for detailed information.") + +if __name__ == '__main__': + main() diff --git a/datatypes_conf.xml.sample b/datatypes_conf.xml.sample index b261594ef0c..74d3b3ba1c5 100644 --- a/datatypes_conf.xml.sample +++ b/datatypes_conf.xml.sample @@ -10,6 +10,7 @@ + @@ -19,6 +20,7 @@ + @@ -31,9 +33,11 @@ + + @@ -81,7 +85,11 @@ - + + + + + @@ -170,13 +178,13 @@ - - + + @@ -268,7 +276,6 @@ - diff --git a/display_applications/igb/bam.xml b/display_applications/igb/bam.xml new file mode 100644 index 00000000000..2401a8d1de3 --- /dev/null +++ b/display_applications/igb/bam.xml @@ -0,0 +1,58 @@ + + + http://localhost:7085/UnibrowControl?version=${bam_file.dbkey}&feature_url_0=${bam_file.url}&sym_name_0=${thenicename}&sym_method_0=${bam_file.url}&query_url=${bam_file.url}&server_url=galaxy + + + #import re + #set nm=$bam_file_for_name.name + ${re.sub('\W',"_",nm)} + + + + + + + $jnlp.url + + + #import re + #set nm=$bam_file_for_name.name + ${re.sub('\W',"_",nm)} + + + + +<?xml version="1.0" encoding="utf-8"?> +<jnlp spec="6.0+" version="" codebase="http://www.bioviz.org/igb/releases/current/" > + <information> + <title>IGB</title> + <vendor>Genoviz Project</vendor> + <homepage href="http://genoviz.sourceforge.net/" /> + <description>Integrated Genome Browser</description> + <icon href="igb.gif"/> + <offline-allowed/> + <shortcut online="true"> + <desktop/> + <menu submenu="IGB" /> + </shortcut> + </information> + <security> + <all-permissions/> + </security> + <resources> + <java version="1.6+" initial-heap-size="32m" max-heap-size="1024m"/> + <jar href="igb_exe.jar" main="true" /> + <jar href="igb-i18n.jar" /> + <property name="apple.laf.useScreenMenuBar" value="true" /> + <property name="http.agent" value="IGB (Webstart)" /> + </resources> + <application-desc main-class="com.affymetrix.main.Main"> + <argument>-prefs</argument> + <argument>http://www.bioviz.org/igb/releases/current/igb_prefs.xml</argument> + <argument>-href</argument> + <argument>http://localhost:7085/UnibrowControl?version=${bam_file.dbkey}&feature_url_0=${bam_file.url}&sym_name_0=${thenicename}&sym_method_0=${bam_file.url}&query_url=${bam_file.url}&server_url=galaxy </argument> + </application-desc> +</jnlp> + + + diff --git a/display_applications/igb/bb.xml b/display_applications/igb/bb.xml new file mode 100644 index 00000000000..8a58054bc88 --- /dev/null +++ b/display_applications/igb/bb.xml @@ -0,0 +1,58 @@ + + + + http://localhost:7085/UnibrowControl?version=${bigbed_file.dbkey}&loadresidues=false&feature_url_0=${bigbed_file.url}&sym_name_0=${thenicename}&sym_method_0=${bigbed_file.url}&query_url=${bigbed_file.url}&server_url=galaxy + + + #import re + #set nm=$bigbed_file_for_name.name + ${re.sub('\W',"_",nm)} + + + + + + $jnlp.url + + + #import re + #set nm=$bigbed_file_for_name.name + ${re.sub('\W',"_",nm)} + + + +<?xml version="1.0" encoding="utf-8"?> +<jnlp spec="6.0+" version="" codebase="http://www.bioviz.org/igb/releases/current/" > + <information> + <title>IGB</title> + <vendor>Genoviz Project</vendor> + <homepage href="http://genoviz.sourceforge.net/" /> + <description>Integrated Genome Browser</description> + <icon href="igb.gif"/> + <offline-allowed/> + <shortcut online="true"> + <desktop/> + <menu submenu="IGB" /> + </shortcut> + </information> + <security> + <all-permissions/> + </security> + <resources> + <java version="1.6+" initial-heap-size="32m" max-heap-size="1024m"/> + <jar href="igb_exe.jar" main="true" /> + <jar href="igb-i18n.jar" /> + <property name="apple.laf.useScreenMenuBar" value="true" /> + <property name="http.agent" value="IGB (Webstart)" /> + </resources> + <application-desc main-class="com.affymetrix.main.Main"> + <argument>-prefs</argument> + <argument>http://www.bioviz.org/igb/releases/current/igb_prefs.xml</argument> + <argument>-href</argument> + <argument>http://localhost:7085/UnibrowControl?version=${bigbed_file.dbkey}&loadresidues=false&feature_url_0=${bigbed_file.url}&sym_name_0=${thenicename}&sym_method_0=${bigbed_file.url}&query_url=${bigbed_file.url}&server_url=galaxy </argument> + </application-desc> +</jnlp> + + + + diff --git a/display_applications/igb/bed.xml b/display_applications/igb/bed.xml new file mode 100644 index 00000000000..ebb20a6b6f4 --- /dev/null +++ b/display_applications/igb/bed.xml @@ -0,0 +1,73 @@ + + + + http://localhost:7085/UnibrowControl?version=${bed_file.dbkey}&${position}&loadresidues=false&feature_url_0=${bed_file.url}&sym_name_0=${thenicename}&sym_method_0=${bed_file.url}&query_url=${bed_file.url}&server_url=galaxy + + + #import re + #set nm=$bed_file_for_name.name + ${re.sub('\W',"_",nm)} + + + + #set chrom, start, end = $bed_file.datatype.get_estimated_display_viewport( $bed_file ) + #if $chrom is not None: + seqid=${chrom}&start=${start}&end=${int(end) + 1} + #else: + seqid=&start=&end= + #end if + + + + + $jnlp.url + + + #import re + #set nm=$bed_file_for_name.name + ${re.sub('\W',"_",nm)} + + + + #set chrom, start, end = $bed_file.datatype.get_estimated_display_viewport( $bed_file ) + #if $chrom is not None: + seqid=${chrom}&start=${start}&end=${int(end) + 1} + #else: + seqid=&start=&end= + #end if + + +<?xml version="1.0" encoding="utf-8"?> +<jnlp spec="6.0+" version="" codebase="http://www.bioviz.org/igb/releases/current/" > + <information> + <title>IGB</title> + <vendor>Genoviz Project</vendor> + <homepage href="http://genoviz.sourceforge.net/" /> + <description>Integrated Genome Browser</description> + <icon href="igb.gif"/> + <offline-allowed/> + <shortcut online="true"> + <desktop/> + <menu submenu="IGB" /> + </shortcut> + </information> + <security> + <all-permissions/> + </security> + <resources> + <java version="1.6+" initial-heap-size="32m" max-heap-size="1024m"/> + <jar href="igb_exe.jar" main="true" /> + <jar href="igb-i18n.jar" /> + <property name="apple.laf.useScreenMenuBar" value="true" /> + <property name="http.agent" value="IGB (Webstart)" /> + </resources> + <application-desc main-class="com.affymetrix.main.Main"> + <argument>-prefs</argument> + <argument>http://www.bioviz.org/igb/releases/current/igb_prefs.xml</argument> + <argument>-href</argument> + <argument>http://localhost:7085/UnibrowControl?version=${bed_file.dbkey}&${position}&loadresidues=false&feature_url_0=${bed_file.url}&sym_name_0=${thenicename}&sym_method_0=${bed_file.url}&query_url=${bed_file.url}&server_url=galaxy </argument> + </application-desc> +</jnlp> + + + diff --git a/display_applications/igb/bigwig.xml b/display_applications/igb/bigwig.xml new file mode 100644 index 00000000000..ebd451fac86 --- /dev/null +++ b/display_applications/igb/bigwig.xml @@ -0,0 +1,65 @@ + + + + http://localhost:7085/UnibrowControl?version=${bigwig_file.dbkey}&loadresidues=false&feature_url_0=${bigwig_file.url}&sym_name_0=${thenicename}&sym_method_0=${bigwig_file.url}&query_url=${bigwig_file.url}&server_url=galaxy + + + #import re + #set nm=$bigwig_file_for_name.name + ${re.sub('\W',"_",nm)} + + + + + + + $jnlp.url + + + #import re + #set nm=$bigwig_file_for_name.name + ${re.sub('\W',"_",nm)} + + + +<?xml version="1.0" encoding="utf-8"?> +<jnlp spec="6.0+" version="" codebase="http://www.bioviz.org/igb/releases/current/" > + <information> + <title>IGB</title> + <vendor>Genoviz Project</vendor> + <homepage href="http://genoviz.sourceforge.net/" /> + <description>Integrated Genome Browser</description> + <icon href="igb.gif"/> + <offline-allowed/> + <shortcut online="true"> + <desktop/> + <menu submenu="IGB" /> + </shortcut> + </information> + <security> + <all-permissions/> + </security> + <resources> + <java version="1.6+" initial-heap-size="32m" max-heap-size="1024m"/> + <jar href="igb_exe.jar" main="true" /> + <jar href="igb-i18n.jar" /> + <property name="apple.laf.useScreenMenuBar" value="true" /> + <property name="http.agent" value="IGB (Webstart)" /> + </resources> + <application-desc main-class="com.affymetrix.main.Main"> + <argument>-prefs</argument> + <argument>http://www.bioviz.org/igb/releases/current/igb_prefs.xml</argument> + <argument>-href</argument> + <argument>http://localhost:7085/UnibrowControl?version=${bigwig_file.dbkey}&loadresidues=false&feature_url_0=${bigwig_file.url}&sym_name_0=${thenicename}&sym_method_0=${bigwig_file.url}&query_url=${bigwig_file.url}&server_url=galaxy </argument> + </application-desc> +</jnlp> + + + diff --git a/display_applications/igb/wig.xml b/display_applications/igb/wig.xml new file mode 100644 index 00000000000..ef6f569b4bd --- /dev/null +++ b/display_applications/igb/wig.xml @@ -0,0 +1,74 @@ + + + + http://localhost:7085/UnibrowControl?version=${wig_file.dbkey}&${position}&loadresidues=false&feature_url_0=${wig_file.url}&sym_name_0=${thenicename}&sym_method_0=${wig_file.url}&query_url=${wig_file.url}&server_url=galaxy + + + #import re + #set nm=$wig_file_for_name.name + ${re.sub('\W',"_",nm)} + + + + #set chrom, start, end = $wig_file.datatype.get_estimated_display_viewport( $wig_file ) + #if $chrom is not None: + seqid=${chrom}&start=${start}&end=${int(end) + 1} + #else: + seqid=&start=&end= + #end if + + + + + $jnlp.url + + + #import re + #set nm=$wig_file_for_name.name + ${re.sub('\W',"_",nm)} + + + + #set chrom, start, end = $wig_file.datatype.get_estimated_display_viewport( $wig_file ) + #if $chrom is not None: + seqid=${chrom}&start=${start}&end=${int(end) + 1} + #else: + seqid=&start=&end= + #end if + + +<?xml version="1.0" encoding="utf-8"?> +<jnlp spec="6.0+" version="" codebase="http://www.bioviz.org/igb/releases/current/" > + <information> + <title>IGB</title> + <vendor>Genoviz Project</vendor> + <homepage href="http://genoviz.sourceforge.net/" /> + <description>Integrated Genome Browser</description> + <icon href="igb.gif"/> + <offline-allowed/> + <shortcut online="true"> + <desktop/> + <menu submenu="IGB" /> + </shortcut> + </information> + <security> + <all-permissions/> + </security> + <resources> + <java version="1.6+" initial-heap-size="32m" max-heap-size="1024m"/> + <jar href="igb_exe.jar" main="true" /> + <jar href="igb-i18n.jar" /> + <property name="apple.laf.useScreenMenuBar" value="true" /> + <property name="http.agent" value="IGB (Webstart)" /> + </resources> + <application-desc main-class="com.affymetrix.main.Main"> + <argument>-prefs</argument> + <argument>http://www.bioviz.org/igb/releases/current/igb_prefs.xml</argument> + <argument>-href</argument> + <argument>http://localhost:7085/UnibrowControl?version=${wig_file.dbkey}&${position}&loadresidues=false&feature_url_0=${wig_file.url}&sym_name_0=${thenicename}&sym_method_0=${wig_file.url}&query_url=${wig_file.url}&server_url=galaxy </argument> + </application-desc> +</jnlp> + + + + diff --git a/dist-eggs.ini b/dist-eggs.ini index 969050fa676..08b43ff6bef 100644 --- a/dist-eggs.ini +++ b/dist-eggs.ini @@ -7,105 +7,82 @@ ; [hosts] -py2.4-linux-i686-ucs2 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs2/bin/python2.4 -py2.4-linux-i686-ucs4 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs4/bin/python2.4 py2.5-linux-i686-ucs2 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs2/bin/python2.5 py2.5-linux-i686-ucs4 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs4/bin/python2.5 py2.6-linux-i686-ucs2 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs2/bin/python2.6 py2.6-linux-i686-ucs4 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs4/bin/python2.6 py2.7-linux-i686-ucs2 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs2/bin/python2.7 py2.7-linux-i686-ucs4 = stegmaier.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-i686-ucs4/bin/python2.7 -py2.4-linux-x86_64-ucs2 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs2/bin/python2.4 -py2.4-linux-x86_64-ucs4 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.4 py2.5-linux-x86_64-ucs2 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs2/bin/python2.5 py2.5-linux-x86_64-ucs4 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.5 py2.6-linux-x86_64-ucs2 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs2/bin/python2.6 py2.6-linux-x86_64-ucs4 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.6 py2.7-linux-x86_64-ucs2 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs2/bin/python2.7 py2.7-linux-x86_64-ucs4 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.7 -py2.4-macosx-10.3-fat-ucs2 = weyerbacher.bx.psu.edu /usr/local/bin/python2.4 -py2.5-macosx-10.3-fat-ucs2 = weyerbacher.bx.psu.edu /usr/local/bin/python2.5 -py2.6-macosx-10.3-fat-ucs2 = weyerbacher.bx.psu.edu /usr/local/bin/python2.6 -py2.7-macosx-10.3-fat-ucs2 = weyerbacher.bx.psu.edu /usr/local/bin/python2.7 +py2.5-macosx-10.3-fat-ucs2 = weyerbacher.bx.psu.edu /Library/Frameworks/Python.framework/Versions/2.5/bin/python2.5 +py2.6-macosx-10.3-fat-ucs2 = weyerbacher.bx.psu.edu /Library/Frameworks/Python.framework/Versions/2.6/bin/python2.6 +py2.7-macosx-10.3-fat-ucs2 = weyerbacher.bx.psu.edu /Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7 py2.6-macosx-10.6-universal-ucs2 = lion.bx.psu.edu /usr/bin/python2.6 py2.7-macosx-10.6-intel-ucs2 = lion.bx.psu.edu /usr/local/bin/python2.7 -py2.4-solaris-2.10-i86pc_32-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_32-ucs2/bin/python2.4 py2.5-solaris-2.10-i86pc_32-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_32-ucs2/bin/python2.5 py2.6-solaris-2.10-i86pc_32-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_32-ucs2/bin/python2.6 py2.7-solaris-2.10-i86pc_32-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_32-ucs2/bin/python2.7 -py2.4-solaris-2.10-i86pc_64-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_64-ucs2/bin/python2.4 py2.5-solaris-2.10-i86pc_64-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_64-ucs2/bin/python2.5 py2.6-solaris-2.10-i86pc_64-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_64-ucs2/bin/python2.6 py2.7-solaris-2.10-i86pc_64-ucs2 = thumper.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-i86pc_64-ucs2/bin/python2.7 -py2.4-solaris-2.10-sun4u_32-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_32-ucs2/bin/python2.4 py2.5-solaris-2.10-sun4u_32-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_32-ucs2/bin/python2.5 py2.6-solaris-2.10-sun4u_32-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_32-ucs2/bin/python2.6 py2.7-solaris-2.10-sun4u_32-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_32-ucs2/bin/python2.7 -py2.4-solaris-2.10-sun4u_64-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_64-ucs2/bin/python2.4 -py2.5-solaris-2.10-sun4u_64-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_64-ucs2/bin/python2.5 -py2.6-solaris-2.10-sun4u_64-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_64-ucs2/bin/python2.6 -py2.7-solaris-2.10-sun4u_64-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.8-sun4u_64-ucs2/bin/python2.7 +py2.5-solaris-2.10-sun4u_64-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-sun4u_64-ucs2/bin/python2.5 +py2.6-solaris-2.10-sun4u_64-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-sun4u_64-ucs2/bin/python2.6 +py2.7-solaris-2.10-sun4u_64-ucs2 = early.bx.psu.edu /afs/bx.psu.edu/project/pythons/solaris-2.10-sun4u_64-ucs2/bin/python2.7 ; these hosts are used to build eggs with no C extensions -py2.4 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.4 py2.5 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.5 py2.6 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.6 py2.7 = straub.bx.psu.edu /afs/bx.psu.edu/project/pythons/linux-x86_64-ucs4/bin/python2.7 [groups] -py2.4-linux-i686 = py2.4-linux-i686-ucs2 py2.4-linux-i686-ucs4 -py2.4-linux-x86_64 = py2.4-linux-x86_64-ucs2 py2.4-linux-x86_64-ucs4 py2.5-linux-i686 = py2.5-linux-i686-ucs2 py2.5-linux-i686-ucs4 py2.5-linux-x86_64 = py2.5-linux-x86_64-ucs2 py2.5-linux-x86_64-ucs4 py2.6-linux-i686 = py2.6-linux-i686-ucs2 py2.6-linux-i686-ucs4 py2.6-linux-x86_64 = py2.6-linux-x86_64-ucs2 py2.6-linux-x86_64-ucs4 py2.7-linux-i686 = py2.7-linux-i686-ucs2 py2.7-linux-i686-ucs4 py2.7-linux-x86_64 = py2.7-linux-x86_64-ucs2 py2.7-linux-x86_64-ucs4 -py2.4-linux = py2.4-linux-i686 py2.4-linux-x86_64 py2.5-linux = py2.5-linux-i686 py2.5-linux-x86_64 py2.6-linux = py2.6-linux-i686 py2.6-linux-x86_64 py2.7-linux = py2.7-linux-i686 py2.7-linux-x86_64 -linux-i686 = py2.4-linux-i686 py2.5-linux-i686 py2.6-linux-i686 py2.7-linux-i686 -linux-x86_64 = py2.4-linux-x86_64 py2.5-linux-x86_64 py2.6-linux-x86_64 py2.7-linux-x86_64 +linux-i686 = py2.5-linux-i686 py2.6-linux-i686 py2.7-linux-i686 +linux-x86_64 = py2.5-linux-x86_64 py2.6-linux-x86_64 py2.7-linux-x86_64 linux = linux-i686 linux-x86_64 -py2.4-macosx = py2.4-macosx-10.3-fat-ucs2 py2.5-macosx = py2.5-macosx-10.3-fat-ucs2 py2.6-macosx = py2.6-macosx-10.3-fat-ucs2 py2.6-macosx-10.6-universal-ucs2 py2.7-macosx = py2.7-macosx-10.3-fat-ucs2 py2.7-macosx-10.6-intel-ucs2 -macosx = py2.4-macosx py2.5-macosx py2.6-macosx py2.7-macosx -py2.4-solaris-i86pc = py2.4-solaris-2.10-i86pc_32-ucs2 py2.4-solaris-2.10-i86pc_64-ucs2 +macosx = py2.5-macosx py2.6-macosx py2.7-macosx py2.5-solaris-i86pc = py2.5-solaris-2.10-i86pc_32-ucs2 py2.5-solaris-2.10-i86pc_64-ucs2 py2.6-solaris-i86pc = py2.6-solaris-2.10-i86pc_32-ucs2 py2.6-solaris-2.10-i86pc_64-ucs2 py2.7-solaris-i86pc = py2.7-solaris-2.10-i86pc_32-ucs2 py2.7-solaris-2.10-i86pc_64-ucs2 -py2.4-solaris-sun4u = py2.4-solaris-2.10-sun4u_32-ucs2 py2.4-solaris-2.10-sun4u_64-ucs2 py2.5-solaris-sun4u = py2.5-solaris-2.10-sun4u_32-ucs2 py2.5-solaris-2.10-sun4u_64-ucs2 py2.6-solaris-sun4u = py2.6-solaris-2.10-sun4u_32-ucs2 py2.6-solaris-2.10-sun4u_64-ucs2 py2.7-solaris-sun4u = py2.7-solaris-2.10-sun4u_32-ucs2 py2.7-solaris-2.10-sun4u_64-ucs2 -py2.4-solaris = py2.4-solaris-i86pc py2.4-solaris-sun4u py2.5-solaris = py2.5-solaris-i86pc py2.5-solaris-sun4u py2.6-solaris = py2.6-solaris-i86pc py2.6-solaris-sun4u py2.7-solaris = py2.7-solaris-i86pc py2.7-solaris-sun4u -solaris-i86pc = py2.4-solaris-i86pc py2.5-solaris-i86pc py2.6-solaris-i86pc py2.7-solaris-i86pc -solaris-sun4u = py2.4-solaris-sun4u py2.5-solaris-sun4u py2.6-solaris-sun4u py2.7-solaris-sun4u +solaris-i86pc = py2.5-solaris-i86pc py2.6-solaris-i86pc py2.7-solaris-i86pc +solaris-sun4u = py2.5-solaris-sun4u py2.6-solaris-sun4u py2.7-solaris-sun4u solaris = solaris-i86pc solaris-sun4u -py2.4-all = py2.4-linux py2.4-macosx py2.4-solaris py2.5-all = py2.5-linux py2.5-macosx py2.5-solaris py2.6-all = py2.6-linux py2.6-macosx py2.6-solaris py2.7-all = py2.7-linux py2.7-macosx py2.7-solaris -; group for building pysam on solaris 10 sparc -;solaris-2.10-sun4u = py2.4-solaris-2.10-sun4u_32-ucs2 py2.5-solaris-2.10-sun4u_32-ucs2 py2.6-solaris-2.10-sun4u_32-ucs2 py2.4-solaris-2.10-sun4u_64-ucs2 py2.5-solaris-2.10-sun4u_64-ucs2 py2.6-solaris-2.10-sun4u_64-ucs2 - ; the 'all' key is used internally by the build system to specify which hosts ; to build on when no hosts are specified on the dist-eggs.py command line. all = linux macosx solaris ; the 'noplatform' key, likewise, is for which build hosts should be used when ; building pure python (noplatform) eggs. -noplatform = py2.4 py2.5 py2.6 py2.7 +noplatform = py2.5 py2.6 py2.7 ; don't build these eggs on these platforms: [ignore] -GeneTrack = py2.4 -python-daemon = py2.4 ctypes = py2.5-linux-i686-ucs2 py2.5-linux-i686-ucs4 py2.6-linux-i686-ucs2 py2.6-linux-i686-ucs4 py2.7-linux-i686-ucs2 py2.7-linux-i686-ucs4 py2.5-linux-x86_64-ucs2 py2.5-linux-x86_64-ucs4 py2.6-linux-x86_64-ucs2 py2.6-linux-x86_64-ucs4 py2.7-linux-x86_64-ucs2 py2.7-linux-x86_64-ucs4 py2.5-macosx-10.3-fat-ucs2 py2.6-macosx-10.3-fat-ucs2 py2.6-macosx-10.6-universal-ucs2 py2.7-macosx-10.3-fat-ucs2 py2.5-solaris-2.10-i86pc_32-ucs2 py2.6-solaris-2.10-i86pc_32-ucs2 py2.7-solaris-2.10-i86pc_32-ucs2 py2.5-solaris-2.10-i86pc_64-ucs2 py2.6-solaris-2.10-i86pc_64-ucs2 py2.7-solaris-2.10-i86pc_64-ucs2 py2.5-solaris-2.10-sun4u_32-ucs2 py2.6-solaris-2.10-sun4u_32-ucs2 py2.7-solaris-2.10-sun4u_32-ucs2 py2.5-solaris-2.10-sun4u_64-ucs2 py2.6-solaris-2.10-sun4u_64-ucs2 py2.7-solaris-2.10-sun4u_64-ucs2 diff --git a/eggs.ini b/eggs.ini index d5547b28a71..30c2ed5c742 100644 --- a/eggs.ini +++ b/eggs.ini @@ -17,6 +17,7 @@ Cheetah = 2.2.2 ctypes = 1.0.2 DRMAA_python = 0.2 MarkupSafe = 0.12 +mercurial = 2.1.2 MySQL_python = 1.2.3c1 numpy = 1.6.0 pbs_python = 4.1.0 diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index be195d90693..2c1c49a4a3b 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -12,6 +12,7 @@ from galaxy.objectstore import build_object_store_from_config import galaxy.quota from galaxy.tags.tag_handler import GalaxyTagHandler from galaxy.tools.imp_exp import load_history_imp_exp_tools +from galaxy.tools.genome_index import load_genome_index_tools from galaxy.sample_tracking import external_service_types from galaxy.openid.providers import OpenIDProviders @@ -90,6 +91,8 @@ class UniverseApplication( object ): self.datatypes_registry.load_external_metadata_tool( self.toolbox ) # Load history import/export tools. load_history_imp_exp_tools( self.toolbox ) + # Load genome indexer tool. + load_genome_index_tools( self.toolbox ) # Load security policy. self.security_agent = self.model.security_agent self.host_security_agent = galaxy.security.HostAgent( model=self.security_agent.model, permitted_actions=self.security_agent.permitted_actions ) @@ -124,8 +127,9 @@ class UniverseApplication( object ): if self.config.get_bool( 'enable_beta_job_managers', False ): from jobs import transfer_manager self.transfer_manager = transfer_manager.TransferManager( self ) - # Start the job queue - self.job_manager = jobs.JobManager( self ) + # Start the job manager + from jobs import manager + self.job_manager = manager.JobManager( self ) # FIXME: These are exposed directly for backward compatibility self.job_queue = self.job_manager.job_queue self.job_stop_queue = self.job_manager.job_stop_queue diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 6fc42236775..a122a1063ed 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -42,8 +42,7 @@ class Configuration( object ): tempfile.tempdir = self.new_file_path self.openid_consumer_cache_path = resolve_path( kwargs.get( "openid_consumer_cache_path", "database/openid_consumer_cache" ), self.root ) self.cookie_path = kwargs.get( "cookie_path", "/" ) - # web API - self.enable_api = string_as_bool( kwargs.get( 'enable_api', False ) ) + self.genome_data_path = kwargs.get( "genome_data_path", "tool-data/genome" ) # Galaxy OpenID settings self.enable_openid = string_as_bool( kwargs.get( 'enable_openid', False ) ) self.openid_config = kwargs.get( 'openid_config_file', 'openid_conf.xml' ) @@ -86,6 +85,7 @@ class Configuration( object ): self.allow_user_dataset_purge = string_as_bool( kwargs.get( "allow_user_dataset_purge", "False" ) ) self.allow_user_impersonation = string_as_bool( kwargs.get( "allow_user_impersonation", "False" ) ) self.new_user_dataset_access_role_default_private = string_as_bool( kwargs.get( "new_user_dataset_access_role_default_private", "False" ) ) + self.collect_outputs_from = [ x.strip() for x in kwargs.get( 'collect_outputs_from', 'new_file_path,job_working_directory' ).lower().split(',') ] self.template_path = resolve_path( kwargs.get( "template_path", "templates" ), self.root ) self.template_cache = resolve_path( kwargs.get( "template_cache_path", "database/compiled_templates" ), self.root ) self.local_job_queue_workers = int( kwargs.get( "local_job_queue_workers", "5" ) ) @@ -105,6 +105,7 @@ class Configuration( object ): self.smtp_username = kwargs.get( 'smtp_username', None ) self.smtp_password = kwargs.get( 'smtp_password', None ) self.start_job_runners = kwargs.get( 'start_job_runners', None ) + self.expose_dataset_path = string_as_bool( kwargs.get( 'expose_dataset_path', 'False' ) ) # External Service types used in sample tracking self.external_service_type_config_file = resolve_path( kwargs.get( 'external_service_type_config_file', 'external_service_types_conf.xml' ), self.root ) self.external_service_type_path = resolve_path( kwargs.get( 'external_service_type_path', 'external_service_types' ), self.root ) @@ -184,14 +185,43 @@ class Configuration( object ): # Heartbeat log file name override if global_conf is not None: self.heartbeat_log = global_conf.get( 'heartbeat_log', 'heartbeat.log' ) - #Store per-tool runner configs. + # Determine which 'server:' this is + self.server_name = 'main' + for arg in sys.argv: + # Crummy, but PasteScript does not give you a way to determine this + if arg.lower().startswith('--server-name='): + self.server_name = arg.split('=', 1)[-1] + # Store advanced job management config + self.job_manager = kwargs.get('job_manager', self.server_name).strip() + self.job_handlers = [ x.strip() for x in kwargs.get('job_handlers', self.server_name).split(',') ] + self.default_job_handlers = [ x.strip() for x in kwargs.get('default_job_handlers', ','.join( self.job_handlers ) ).split(',') ] + # Use database for IPC unless this is a standalone server (or multiple servers doing self dispatching in memory) + self.track_jobs_in_database = True + if ( len( self.job_handlers ) == 1 ) and ( self.job_handlers[0] == self.server_name ) and ( self.job_manager == self.server_name ): + self.track_jobs_in_database = False + # Store per-tool runner configs + self.tool_handlers = self.__read_tool_job_config( global_conf_parser, 'galaxy:tool_handlers', 'name' ) + self.tool_runners = self.__read_tool_job_config( global_conf_parser, 'galaxy:tool_runners', 'url' ) + self.datatypes_config = kwargs.get( 'datatypes_config_file', 'datatypes_conf.xml' ) + # Cloud configuration options + self.enable_cloud_launch = string_as_bool( kwargs.get( 'enable_cloud_launch', False ) ) + # Galaxy messaging (AMQP) configuration options + self.amqp = {} try: - tool_runners_config = global_conf_parser.items("galaxy:tool_runners") + amqp_config = global_conf_parser.items("galaxy_amqp") + except ConfigParser.NoSectionError: + amqp_config = {} + for k, v in amqp_config: + self.amqp[k] = v + self.running_functional_tests = string_as_bool( kwargs.get( 'running_functional_tests', False ) ) + def __read_tool_job_config( self, global_conf_parser, section, key ): + try: + tool_runners_config = global_conf_parser.items( section ) # Process config to group multiple configs for the same tool. - tool_runners = {} + rval = {} for entry in tool_runners_config: - tool_config, url = entry + tool_config, val = entry tool = None runner_dict = {} if tool_config.find("[") != -1: @@ -206,29 +236,18 @@ class Configuration( object ): tool = tool_config # Add runner URL. - runner_dict[ 'url' ] = url + runner_dict[ key ] = val # Create tool entry if necessary. - if tool not in tool_runners: - tool_runners[ tool ] = [] + if tool not in rval: + rval[ tool ] = [] # Add entry to runners. - tool_runners[ tool ].append( runner_dict ) + rval[ tool ].append( runner_dict ) - self.tool_runners = tool_runners + return rval except ConfigParser.NoSectionError: - self.tool_runners = [] - self.datatypes_config = kwargs.get( 'datatypes_config_file', 'datatypes_conf.xml' ) - # Cloud configuration options - self.enable_cloud_launch = string_as_bool( kwargs.get( 'enable_cloud_launch', False ) ) - # Galaxy messaging (AMQP) configuration options - self.amqp = {} - try: - amqp_config = global_conf_parser.items("galaxy_amqp") - except ConfigParser.NoSectionError: - amqp_config = {} - for k, v in amqp_config: - self.amqp[k] = v + return [] def get( self, key, default ): return self.config_dict.get( key, default ) def get_bool( self, key, default ): @@ -238,16 +257,6 @@ class Configuration( object ): return default def check( self ): paths_to_check = [ self.root, self.tool_path, self.tool_data_path, self.template_path ] - # Look for any tool shed configs and retrieve the tool_path attribute from the tag. - tool_configs = self.tool_configs - if self.migrated_tools_config not in tool_configs: - tool_configs.append( self.migrated_tools_config ) - for config_filename in tool_configs: - tree = parse_xml( config_filename ) - root = tree.getroot() - tool_path = root.get( 'tool_path' ) - if tool_path not in [ None, False ]: - paths_to_check.append( resolve_path( tool_path, self.root ) ) # Check that required directories exist for path in paths_to_check: if path not in [ None, False ] and not os.path.isdir( path ): diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 938d1e37f22..2bc451454f0 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -34,6 +34,17 @@ class Binary( data.Data ): """Returns the mime type of the datatype""" return 'application/octet-stream' + def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, size=None, offset=None, **kwd): + trans.response.set_content_type(dataset.get_mime()) + trans.log_event( "Display dataset id: %s" % str( dataset.id ) ) + trans.response.headers['Content-Length'] = int( os.stat( dataset.file_name ).st_size ) + to_ext = dataset.extension + valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' + fname = ''.join(c in valid_chars and c or '_' for c in dataset.name)[0:150] + trans.response.set_content_type( "application/octet-stream" ) #force octet-stream so Safari doesn't append mime extensions to filename + trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy%s-[%s].%s"' % (dataset.hid, fname, to_ext) + return open( dataset.file_name ) + class Ab1( Binary ): """Class describing an ab1 binary sequence file""" file_ext = "ab1" diff --git a/lib/galaxy/datatypes/converters/sam_to_bam.py b/lib/galaxy/datatypes/converters/sam_to_bam.py new file mode 100644 index 00000000000..d26e28f70df --- /dev/null +++ b/lib/galaxy/datatypes/converters/sam_to_bam.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +#Dan Blankenberg + +""" +A wrapper script for converting SAM to BAM, with sorting. +%prog input_filename.sam output_filename.bam +""" + +import sys, optparse, os, tempfile, subprocess, shutil + +CHUNK_SIZE = 2**20 #1mb + + +def cleanup_before_exit( tmp_dir ): + if tmp_dir and os.path.exists( tmp_dir ): + shutil.rmtree( tmp_dir ) + +def __main__(): + #Parse Command Line + parser = optparse.OptionParser() + (options, args) = parser.parse_args() + + assert len( args ) == 2, 'You must specify the input and output filenames' + input_filename, output_filename = args + + tmp_dir = tempfile.mkdtemp( prefix='tmp-sam_to_bam_converter-' ) + + #convert to SAM + unsorted_bam_filename = os.path.join( tmp_dir, 'unsorted.bam' ) + unsorted_stderr_filename = os.path.join( tmp_dir, 'unsorted.stderr' ) + cmd = 'samtools view -bS "%s" > "%s"' % ( input_filename, unsorted_bam_filename ) + proc = subprocess.Popen( args=cmd, stderr=open( unsorted_stderr_filename, 'wb' ), shell=True, cwd=tmp_dir ) + return_code = proc.wait() + if return_code: + stderr_target = sys.stderr + else: + stderr_target = sys.stdout + stderr = open( unsorted_stderr_filename ) + while True: + chunk = stderr.read( CHUNK_SIZE ) + if chunk: + stderr_target.write( chunk ) + else: + break + stderr.close() + + #sort sam, so indexing will not fail + sorted_stderr_filename = os.path.join( tmp_dir, 'sorted.stderr' ) + sorting_prefix = os.path.join( tmp_dir, 'sorted_bam' ) + cmd = 'samtools sort -o "%s" "%s" > "%s"' % ( unsorted_bam_filename, sorting_prefix, output_filename ) + proc = subprocess.Popen( args=cmd, stderr=open( sorted_stderr_filename, 'wb' ), shell=True, cwd=tmp_dir ) + return_code = proc.wait() + + if return_code: + stderr_target = sys.stderr + else: + stderr_target = sys.stdout + stderr = open( sorted_stderr_filename ) + while True: + chunk = stderr.read( CHUNK_SIZE ) + if chunk: + stderr_target.write( chunk ) + else: + break + stderr.close() + + cleanup_before_exit( tmp_dir ) + +if __name__=="__main__": __main__() diff --git a/lib/galaxy/datatypes/converters/sam_to_bam.xml b/lib/galaxy/datatypes/converters/sam_to_bam.xml index 3b9c37c8355..1a5ce77b794 100644 --- a/lib/galaxy/datatypes/converters/sam_to_bam.xml +++ b/lib/galaxy/datatypes/converters/sam_to_bam.xml @@ -1,11 +1,11 @@ - + - samtools view -bS $input1 > $output 2> /dev/null + sam_to_bam.py $input1 $output diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index f68c9f8a07f..f1d7cc3949c 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -3,11 +3,39 @@ from galaxy import util from galaxy.util.odict import odict from galaxy.util.bunch import Bunch from galaxy.util import inflector +from galaxy.util.sanitize_html import sanitize_html from cgi import escape +import mimetypes import metadata import zipfile from metadata import MetadataElement #import directly to maintain ease of use in Datatype class definitions + +if sys.version_info[:2] < ( 2, 6 ): + zipfile.BadZipFile = zipfile.error +if sys.version_info[:2] < ( 2, 5 ): + zipfile.LargeZipFile = zipfile.error + +tmpd = tempfile.mkdtemp() +comptypes=[] +ziptype = '32' +tmpf = os.path.join( tmpd, 'compression_test.zip' ) +try: + archive = zipfile.ZipFile( tmpf, 'w', zipfile.ZIP_DEFLATED, True ) + archive.close() + comptypes.append( 'zip' ) + ziptype = '64' +except RuntimeError: + log.exception( "Compression error when testing zip compression. This option will be disabled for library downloads." ) +except (TypeError, zipfile.LargeZipFile): # ZIP64 is only in Python2.5+. Remove TypeError when 2.4 support is dropped + log.warning( 'Max zip file size is 2GB, ZIP64 not supported' ) + comptypes.append( 'zip' ) +try: + os.unlink( tmpf ) +except OSError: + pass +os.rmdir( tmpd ) + log = logging.getLogger(__name__) # Valid first column and strand column values vor bed, other formats @@ -39,7 +67,7 @@ class Data( object ): 'test' >>> type( DataTest.metadata_spec.test.param ) - + """ __metaclass__ = DataMeta # Add metadata elements @@ -60,7 +88,7 @@ class Data( object ): primary_file_name = 'index' #A per datatype setting (inherited): max file size (in bytes) for setting optional metadata _max_optional_metadata_filesize = None - + def __init__(self, **kwd): """Initialize the datatype""" object.__init__(self, **kwd) @@ -118,7 +146,7 @@ class Data( object ): to_check = dataset.metadata.items() for key, value in to_check: if key in skip or ( not check and dataset.metadata.spec[key].get( "optional" ) ): - continue #we skip check for optional and nonrequested values here + continue #we skip check for optional and nonrequested values here if not value: return True return False @@ -142,6 +170,7 @@ class Data( object ): else: dataset.peek = 'file does not exist' dataset.blurb = 'file purged from disk' + def display_peek(self, dataset ): """Create HTML table, used for displaying peek""" out = [''] @@ -163,6 +192,151 @@ class Data( object ): except Exception, exc: out = "Can't create peek %s" % str( exc ) return out + + def _archive_composite_dataset( self, trans, data=None, **kwd ): + # save a composite object into a compressed archive for downloading + params = util.Params( kwd ) + valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' + outfname = data.name[0:150] + outfname = ''.join(c in valid_chars and c or '_' for c in outfname) + if (params.do_action == None): + params.do_action = 'zip' # default + msg = util.restore_text( params.get( 'msg', '' ) ) + messagetype = params.get( 'messagetype', 'done' ) + if not data: + msg = "You must select at least one dataset" + messagetype = 'error' + else: + error = False + try: + if (params.do_action == 'zip'): + # Can't use mkstemp - the file must not exist first + tmpd = tempfile.mkdtemp() + tmpf = os.path.join( tmpd, 'library_download.' + params.do_action ) + if ziptype == '64': + archive = zipfile.ZipFile( tmpf, 'w', zipfile.ZIP_DEFLATED, True ) + else: + archive = zipfile.ZipFile( tmpf, 'w', zipfile.ZIP_DEFLATED ) + archive.add = lambda x, y: archive.write( x, y.encode('CP437') ) + elif params.do_action == 'tgz': + archive = util.streamball.StreamBall( 'w|gz' ) + elif params.do_action == 'tbz': + archive = util.streamball.StreamBall( 'w|bz2' ) + except (OSError, zipfile.BadZipFile): + error = True + log.exception( "Unable to create archive for download" ) + msg = "Unable to create archive for %s for download, please report this error" % outfname + messagetype = 'error' + if not error: + current_user_roles = trans.get_current_user_roles() + ext = data.extension + path = data.file_name + fname = os.path.split(path)[-1] + efp = data.extra_files_path + htmlname = os.path.splitext(outfname)[0] + if not htmlname.endswith(ext): + htmlname = '%s_%s' % (htmlname,ext) + archname = '%s.html' % htmlname # fake the real nature of the html file + try: + archive.add(data.file_name,archname) + except IOError: + error = True + log.exception( "Unable to add composite parent %s to temporary library download archive" % data.file_name) + msg = "Unable to create archive for download, please report this error" + messagetype = 'error' + for root, dirs, files in os.walk(efp): + for fname in files: + fpath = os.path.join(root,fname) + rpath = os.path.relpath(fpath,efp) + try: + archive.add( fpath,rpath ) + except IOError: + error = True + log.exception( "Unable to add %s to temporary library download archive" % rpath) + msg = "Unable to create archive for download, please report this error" + messagetype = 'error' + continue + if not error: + if params.do_action == 'zip': + archive.close() + tmpfh = open( tmpf ) + # CANNOT clean up - unlink/rmdir was always failing because file handle retained to return - must rely on a cron job to clean up tmp + trans.response.set_content_type( "application/x-zip-compressed" ) + trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s.zip"' % outfname + return tmpfh + else: + trans.response.set_content_type( "application/x-tar" ) + outext = 'tgz' + if params.do_action == 'tbz': + outext = 'tbz' + trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s.%s"' % (outfname,outext) + archive.wsgi_status = trans.response.wsgi_status() + archive.wsgi_headeritems = trans.response.wsgi_headeritems() + return archive.stream + return trans.show_error_message( msg ) + + def _serve_raw(self, trans, dataset, to_ext): + trans.response.headers['Content-Length'] = int( os.stat( dataset.file_name ).st_size ) + valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' + fname = ''.join(c in valid_chars and c or '_' for c in dataset.name)[0:150] + trans.response.set_content_type( "application/octet-stream" ) #force octet-stream so Safari doesn't append mime extensions to filename + trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy%s-[%s].%s"' % (dataset.hid, fname, to_ext) + return open( dataset.file_name ) + + def display_data(self, trans, data, preview=False, filename=None, to_ext=None, size=None, offset=None, **kwd): + """ Old display method, for transition """ + #Relocate all composite datatype display to a common location. + composite_extensions = trans.app.datatypes_registry.get_composite_extensions( ) + composite_extensions.append('html') # for archiving composite datatypes + if isinstance( data, basestring ): + return data + if filename and filename != "index": + # For files in extra_files_path + file_path = trans.app.object_store.get_filename(data.dataset, extra_dir='dataset_%s_files' % data.dataset.id, alt_name=filename) + if os.path.exists( file_path ): + if os.path.isdir( file_path ): + return trans.show_error_message( "Directory listing is not allowed." ) #TODO: Reconsider allowing listing of directories? + mime, encoding = mimetypes.guess_type( file_path ) + if not mime: + try: + mime = trans.app.datatypes_registry.get_mimetype_by_extension( ".".split( file_path )[-1] ) + except: + mime = "text/plain" + trans.response.set_content_type( mime ) + return open( file_path ) + else: + return trans.show_error_message( "Could not find '%s' on the extra files path %s." % ( filename, file_path ) ) + trans.response.set_content_type(data.get_mime()) + trans.log_event( "Display dataset id: %s" % str( data.id ) ) + from galaxy import datatypes #DBTODO REMOVE THIS AT REFACTOR + if to_ext or isinstance(data.datatype, datatypes.binary.Binary): # Saving the file, or binary file + if data.extension in composite_extensions: + return self._archive_composite_dataset( trans, data, **kwd ) + else: + trans.response.headers['Content-Length'] = int( os.stat( data.file_name ).st_size ) + if not to_ext: + to_ext = data.extension + valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' + fname = ''.join(c in valid_chars and c or '_' for c in data.name)[0:150] + trans.response.set_content_type( "application/octet-stream" ) #force octet-stream so Safari doesn't append mime extensions to filename + trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy%s-[%s].%s"' % (data.hid, fname, to_ext) + return open( data.file_name ) + if not os.path.exists( data.file_name ): + raise paste.httpexceptions.HTTPNotFound( "File Not Found (%s)." % data.file_name ) + max_peek_size = 1000000 # 1 MB + if isinstance(data.datatype, datatypes.images.Html): + max_peek_size = 10000000 # 10 MB for html + if not preview or isinstance(data.datatype, datatypes.images.Image) or os.stat( data.file_name ).st_size < max_peek_size: + if trans.app.config.sanitize_all_html and trans.response.get_content_type() == "text/html": + # Sanitize anytime we respond with plain text/html content. + return sanitize_html(open( data.file_name ).read()) + return open( data.file_name ) + else: + trans.response.set_content_type( "text/html" ) + return trans.stream_template_mako( "/dataset/large_file.mako", + truncated_data = open( data.file_name ).read(max_peek_size), + data = data) + def display_name(self, dataset): """Returns formatted html of dataset name""" try: @@ -183,11 +357,11 @@ class Data( object ): info = info.replace( '\r', '
' ) if info.find( '\n' ) >= 0: info = info.replace( '\n', '
' ) - + # Convert to unicode to display non-ascii characters. if type( info ) is not unicode: info = unicode( info, 'utf-8') - + return info except: return "info unavailable" @@ -272,7 +446,7 @@ class Data( object ): def convert_dataset(self, trans, original_dataset, target_type, return_output=False, visible=True, deps=None, set_output_history=True): """This function adds a job to the queue to convert a dataset to another type. Returns a message about success/failure.""" converter = trans.app.datatypes_registry.get_converter_by_target_type( original_dataset.ext, target_type ) - + if converter is None: raise Exception( "A converter does not exist for %s to %s." % ( original_dataset.ext, target_type ) ) #Generate parameter dictionary @@ -284,7 +458,7 @@ class Data( object ): params[value.name] = deps[value.name] elif value.type == 'data': input_name = key - + params[input_name] = original_dataset #Run converter, job is dispatched through Queue converted_dataset = converter.execute( trans, incoming=params, set_output_hid=visible, set_output_history=set_output_history)[1] @@ -351,18 +525,18 @@ class Data( object ): @property def has_resolution(self): return False - - - def merge( split_files, output_file): + + + def merge( split_files, output_file): """ TODO: Do we need to merge gzip files using gzjoin? cat seems to work, but might be brittle. Need to revisit this. """ if len(split_files) == 1: - cmd = 'mv -f %s %s' % ( split_files[0], output_file ) + cmd = 'mv -f %s %s' % ( split_files[0], output_file ) else: - cmd = 'cat %s > %s' % ( ' '.join(split_files), output_file ) + cmd = 'cat %s > %s' % ( ' '.join(split_files), output_file ) result = os.system(cmd) if result != 0: raise Exception('Result %s from %s' % (result, cmd)) @@ -377,7 +551,7 @@ class Text( Data ): def write_from_stream(self, dataset, stream): """Writes data from a stream""" - # write it twice for now + # write it twice for now fd, temp_name = tempfile.mkstemp() while 1: chunk = stream.read(1048576) @@ -468,11 +642,11 @@ class Text( Data ): """ if split_params is None: return - + if len(input_datasets) > 1: raise Exception("Text file splitting does not support multiple files") input_files = [ds.file_name for ds in input_datasets] - + lines_per_file = None chunk_size = None if split_params['split_mode'] == 'number_of_parts': @@ -501,7 +675,7 @@ class Text( Data ): chunk_size = int(split_params['split_size']) else: raise Exception('Unsupported split mode %s' % split_params['split_mode']) - + f = open(input_files[0], 'rt') try: chunk_idx = 0 @@ -562,7 +736,7 @@ def get_test_fname( fname ): def get_file_peek( file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5, skipchars=[] ): """ Returns the first LINE_COUNT lines wrapped to WIDTH - + ## >>> fname = get_test_fname('4.bed') ## >>> get_file_peek(fname) ## 'chr22 30128507 31828507 uc003bnx.1_cds_2_0_chr22_29227_f 0 +\n' @@ -601,11 +775,12 @@ def get_file_peek( file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5, skip lines.append( line ) count += 1 temp.close() - if file_type in [ 'gzipped', 'binary' ]: - text = "%s file" % file_type + if file_type in [ 'gzipped', 'binary' ]: + text = "%s file" % file_type else: try: text = unicode( '\n'.join( lines ), 'utf-8' ) except UnicodeDecodeError: text = "binary/unknown file" return text + diff --git a/lib/galaxy/datatypes/interval.py b/lib/galaxy/datatypes/interval.py index 351ef0ddda4..c022183b90a 100644 --- a/lib/galaxy/datatypes/interval.py +++ b/lib/galaxy/datatypes/interval.py @@ -328,6 +328,9 @@ class Interval( Tabular ): def get_track_resolution( self, dataset, start, end): return None + + def get_track_type( self ): + return "FeatureTrack", {"data": "tabix", "index": "summary_tree"} class BedGraph( Interval ): """Tab delimited chrom/start/end/datavalue dataset""" @@ -950,6 +953,9 @@ class Gtf( Gff ): dataset.metadata.attribute_types = attribute_types dataset.metadata.attributes = len( attribute_types ) Gff.set_meta( self, dataset, overwrite = overwrite, skip = i ) + + def get_track_type( self ): + return "FeatureTrack", {"data": "tabix", "index": "summary_tree"} class Wiggle( Tabular, _RemoteCallMixin ): diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 6686927bd78..50315904e00 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -2,7 +2,7 @@ Provides mapping between extensions and datatypes, mime-types, etc. """ import os, sys, tempfile, threading, logging -import data, tabular, interval, images, sequence, qualityscore, genetics, xml, coverage, tracks, chrominfo, binary, assembly, ngsindex, wsf +import data, tabular, interval, images, sequence, qualityscore, genetics, xml, coverage, tracks, chrominfo, binary, assembly, ngsindex import galaxy.util from galaxy.util.odict import odict from display_applications.application import DisplayApplication @@ -141,7 +141,7 @@ class Registry( object ): if hasattr( imported_module, datatype_class_name ): datatype_class = getattr( imported_module, datatype_class_name ) except Exception, e: - full_path = os.path.join( full_path, proprietary_datatype_module ) + full_path = os.path.join( proprietary_path, proprietary_datatype_module ) self.log.debug( "Exception importing proprietary code file %s: %s" % ( str( full_path ), str( e ) ) ) finally: lock.release() diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 73584ea2f27..bf9fda1f88e 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -13,34 +13,37 @@ from galaxy.datatypes import metadata from galaxy.datatypes.metadata import MetadataElement import galaxy_utils.sequence.vcf from sniff import * +from galaxy.util.json import to_json_string log = logging.getLogger(__name__) class Tabular( data.Text ): """Tab delimited data""" + CHUNK_SIZE = 20000 """Add metadata elements""" MetadataElement( name="comment_lines", default=0, desc="Number of comment lines", readonly=False, optional=True, no_value=0 ) MetadataElement( name="columns", default=0, desc="Number of columns", readonly=True, visible=False, no_value=0 ) MetadataElement( name="column_types", default=[], desc="Column types", param=metadata.ColumnTypesParameter, readonly=True, visible=False, no_value=[] ) + MetadataElement( name="column_names", default=[], desc="Column names", readonly=True, visible=False, optional=True, no_value=[] ) def init_meta( self, dataset, copy_from=None ): data.Text.init_meta( self, dataset, copy_from=copy_from ) - def set_meta( self, dataset, overwrite = True, skip = None, max_data_lines = 100000, **kwd ): + def set_meta( self, dataset, overwrite = True, skip = None, max_data_lines = 100000, max_guess_type_data_lines = None, **kwd ): """ Tries to determine the number of columns as well as those columns that contain numerical values in the dataset. A skip parameter is used because various tabular data types reuse this function, and their data type classes are responsible to determine how many invalid - comment lines should be skipped. Using None for skip will cause skip - to be zero, but the first line will be processed as a header. A - max_data_lines parameter is used because various tabular data types - reuse this function, and their data type classes are responsible to + comment lines should be skipped. Using None for skip will cause skip + to be zero, but the first line will be processed as a header. A + max_data_lines parameter is used because various tabular data types + reuse this function, and their data type classes are responsible to determine how many data lines should be processed to ensure that the - non-optional metadata parameters are properly set; if used, optional - metadata parameters will be set to None, unless the entire file has - already been read. Using None (default) for max_data_lines will - process all data lines. + non-optional metadata parameters are properly set; if used, optional + metadata parameters will be set to None, unless the entire file has + already been read. Using None (default) for max_data_lines will + process all data lines. Items of interest: 1. We treat 'overwrite' as always True (we always want to set tabular metadata when called). @@ -57,7 +60,7 @@ class Tabular( data.Text ): column_type_set_order = [ 'int', 'float', 'list', 'str' ] #Order to set column types in default_column_type = column_type_set_order[-1] # Default column type is lowest in list column_type_compare_order = list( column_type_set_order ) #Order to compare column types - column_type_compare_order.reverse() + column_type_compare_order.reverse() def type_overrules_type( column_type1, column_type2 ): if column_type1 is None or column_type1 == column_type2: return False @@ -74,13 +77,13 @@ class Tabular( data.Text ): try: int( column_text ) return True - except: + except: return False def is_float( column_text ): try: float( column_text ) return True - except: + except: if column_text.strip().lower() == 'na': return True #na is special cased to be a float return False @@ -116,15 +119,16 @@ class Tabular( data.Text ): comment_lines += 1 else: data_lines += 1 - fields = line.split( '\t' ) - for field_count, field in enumerate( fields ): - if field_count >= len( column_types ): #found a previously unknown column, we append None - column_types.append( None ) - column_type = guess_column_type( field ) - if type_overrules_type( column_type, column_types[field_count] ): - column_types[field_count] = column_type + if max_guess_type_data_lines is None or data_lines <= max_guess_type_data_lines: + fields = line.split( '\t' ) + for field_count, field in enumerate( fields ): + if field_count >= len( column_types ): #found a previously unknown column, we append None + column_types.append( None ) + column_type = guess_column_type( field ) + if type_overrules_type( column_type, column_types[field_count] ): + column_types[field_count] = column_type if i == 0 and requested_skip is None: - # This is our first line, people seem to like to upload files that have a header line, but do not + # This is our first line, people seem to like to upload files that have a header line, but do not # start with '#' (i.e. all column types would then most likely be detected as str). We will assume # that the first line is always a header (this was previous behavior - it was always skipped). When # the requested skip is None, we only use the data from the first line if we have no other data for @@ -146,7 +150,7 @@ class Tabular( data.Text ): break i += 1 dataset_fh.close() - + #we error on the larger number of columns #first we pad our column_types by using data from first line if len( first_line_column_types ) > len( column_types ): @@ -175,9 +179,19 @@ class Tabular( data.Text ): except Exception, exc: out = "Can't create peek %s" % str( exc ) return out - def make_html_peek_header( self, dataset, skipchars=[], column_names=[], column_number_format='%s', column_parameter_alias={}, **kwargs ): + + def make_html_peek_header( self, dataset, skipchars=None, column_names=None, column_number_format='%s', column_parameter_alias=None, **kwargs ): + if skipchars is None: + skipchars = [] + if column_names is None: + column_names = [] + if column_parameter_alias is None: + column_parameter_alias = {} out = [] try: + if not column_names and dataset.metadata.column_names: + column_names = dataset.metadata.column_names + column_headers = [None] * dataset.metadata.columns # fill in empty headers with data from column_names @@ -207,7 +221,10 @@ class Tabular( data.Text ): except Exception, exc: raise Exception, "Can't create peek header %s" % str( exc ) return "".join( out ) - def make_html_peek_rows( self, dataset, skipchars=[], **kwargs ): + + def make_html_peek_rows( self, dataset, skipchars=None, **kwargs ): + if skipchars is None: + skipchars = [] out = [] try: if not dataset.peek: @@ -228,6 +245,28 @@ class Tabular( data.Text ): except Exception, exc: raise Exception, "Can't create peek rows %s" % str( exc ) return "".join( out ) + + def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, chunk=None): + #TODO Prevent failure when displaying extremely long > 50kb lines. + if to_ext: + return self._serve_raw(trans, dataset, to_ext) + if chunk: + ck_index = int(chunk) + f = open(dataset.file_name) + f.seek(ck_index * self.CHUNK_SIZE) + # If we aren't at the start of the file, seek to next newline. Do this better eventually. + if f.tell() != 0: + cursor = f.read(1) + while cursor and cursor != '\n': + cursor = f.read(1) + ck_data = f.read(self.CHUNK_SIZE) + cursor = f.read(1) + while cursor and ck_data[-1] != '\n': + ck_data += cursor + cursor = f.read(1) + return to_json_string({'ck_data': ck_data, 'ck_index': ck_index+1}) + return trans.fill_template( "/dataset/tabular_chunked.mako",dataset = dataset) + def set_peek( self, dataset, line_count=None, is_multi_byte=False): super(Tabular, self).set_peek( dataset, line_count=line_count, is_multi_byte=is_multi_byte) if dataset.metadata.comment_lines: @@ -276,7 +315,7 @@ class Sam( Tabular ): def sniff( self, filename ): """ Determines whether the file is in SAM format - + A file in SAM format consists of lines of tab-separated data. The following header line may be the first line: @QNAME FLAG RNAME POS MAPQ CIGAR MRNM MPOS ISIZE SEQ QUAL @@ -285,12 +324,12 @@ class Sam( Tabular ): Data in the OPT column is optional and can consist of tab-separated data For complete details see http://samtools.sourceforge.net/SAM1.pdf - + Rules for sniffing as True: There must be 11 or more columns of data on each line Columns 2 (FLAG), 4(POS), 5 (MAPQ), 8 (MPOS), and 9 (ISIZE) must be numbers (9 can be negative) We will only check that up to the first 5 alignments are correctly formatted. - + >>> fname = get_test_fname( 'sequence.maf' ) >>> Sam().sniff( fname ) False @@ -306,7 +345,7 @@ class Sam( Tabular ): line = line.strip() if not line: break #EOF - if line: + if line: if line[0] != '@': linePieces = line.split('\t') if len(linePieces) < 11: @@ -368,10 +407,10 @@ class Sam( Tabular ): if result != 0: raise Exception('Result %s from %s' % (result, cmd)) merge = staticmethod(merge) - + def get_track_type( self ): return "ReadTrack", {"data": "bam", "index": "summary_tree"} - + class Pileup( Tabular ): """Tab delimited data in pileup (6- or 10-column) format""" file_ext = "pileup" @@ -397,7 +436,7 @@ class Pileup( Tabular ): """ Checks for 'pileup-ness' - There are two main types of pileup: 6-column and 10-column. For both, + There are two main types of pileup: 6-column and 10-column. For both, the first three and last two columns are the same. We only check the first three to allow for some personalization of the format. @@ -431,27 +470,27 @@ class Pileup( Tabular ): class ElandMulti( Tabular ): file_ext = 'elandmulti' - + def sniff( self, filename ): return False - + class Vcf( Tabular ): """ Variant Call Format for describing SNPs and other simple genome variations. """ - + file_ext = 'vcf' column_names = [ 'Chrom', 'Pos', 'ID', 'Ref', 'Alt', 'Qual', 'Filter', 'Info', 'Format', 'data' ] - + MetadataElement( name="columns", default=10, desc="Number of columns", readonly=True, visible=False ) MetadataElement( name="column_types", default=['str','int','str','str','str','int','str','list','str','str'], param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False ) MetadataElement( name="viz_filter_cols", desc="Score column for visualization", default=[5], param=metadata.ColumnParameter, multiple=True ) - + def sniff( self, filename ): headers = get_headers( filename, '\n', count=1 ) return headers[0][0].startswith("##fileformat=VCF") def display_peek( self, dataset ): """Returns formated html of peek""" return Tabular.make_html_table( self, dataset, column_names=self.column_names ) - + def get_track_type( self ): return "VcfTrack", {"data": "tabix", "index": "summary_tree"} @@ -473,8 +512,10 @@ class Eland( Tabular ): 'POSITION', 'STRAND', 'DESC', 'SRAS', 'PRAS', 'PART_CHROM' 'PART_CONTIG', 'PART_OFFSET', 'PART_STRAND', 'FILT' ] - def make_html_table( self, dataset, skipchars=[] ): + def make_html_table( self, dataset, skipchars=None ): """Create HTML table, used for displaying peek""" + if skipchars is None: + skipchars = [] out = ['
'] try: # Generate column header @@ -495,10 +536,10 @@ class Eland( Tabular ): def sniff( self, filename ): """ Determines whether the file is in ELAND export format - + A file in ELAND export format consists of lines of tab-separated data. There is no header. - + Rules for sniffing as True: There must be 22 columns on each line LANE, TILEm X, Y, INDEX, READ_NO, SEQ, QUAL, POSITION, *STRAND, FILT must be correct @@ -517,7 +558,7 @@ class Eland( Tabular ): line = line.strip() if not line: break #EOF - if line: + if line: linePieces = line.split('\t') if len(linePieces) != 22: return False @@ -563,7 +604,7 @@ class Eland( Tabular ): #else: # # Otherwise, read the whole thing and set num data lines. for i, line in enumerate(dataset_fh): - if line: + if line: linePieces = line.split('\t') if len(linePieces) != 22: raise Exception('%s:%d:Corrupt line!' % (dataset.file_name,i)) @@ -581,5 +622,5 @@ class Eland( Tabular ): dataset.metadata.tiles = ["%04d" % int(t) for t in tiles.keys()] dataset.metadata.barcodes = filter(lambda x: x != '0', barcodes.keys()) + ['NoIndex' for x in barcodes.keys() if x == '0'] dataset.metadata.reads = reads.keys() - - + + diff --git a/lib/galaxy/datatypes/util/gff_util.py b/lib/galaxy/datatypes/util/gff_util.py index 0a960e9e47a..680bde245f6 100644 --- a/lib/galaxy/datatypes/util/gff_util.py +++ b/lib/galaxy/datatypes/util/gff_util.py @@ -10,8 +10,8 @@ class GFFInterval( GenomicInterval ): A GFF interval, including attributes. If file is strictly a GFF file, only attribute is 'group.' """ - def __init__( self, reader, fields, chrom_col, feature_col, start_col, end_col, \ - strand_col, score_col, default_strand, fix_strand=False ): + def __init__( self, reader, fields, chrom_col=0, feature_col=2, start_col=3, end_col=4, \ + strand_col=6, score_col=5, default_strand='.', fix_strand=False ): # HACK: GFF format allows '.' for strand but GenomicInterval does not. To get around this, # temporarily set strand and then unset after initing GenomicInterval. unknown_strand = False @@ -45,8 +45,8 @@ class GFFFeature( GFFInterval ): """ A GFF feature, which can include multiple intervals. """ - def __init__( self, reader, chrom_col, feature_col, start_col, end_col, \ - strand_col, score_col, default_strand, fix_strand=False, intervals=[], \ + def __init__( self, reader, chrom_col=0, feature_col=2, start_col=3, end_col=4, \ + strand_col=6, score_col=5, default_strand='.', fix_strand=False, intervals=[], \ raw_size=0 ): GFFInterval.__init__( self, reader, intervals[0].fields, chrom_col, feature_col, \ start_col, end_col, strand_col, score_col, default_strand, \ diff --git a/lib/galaxy/datatypes/wsf.py b/lib/galaxy/datatypes/wsf.py deleted file mode 100644 index 551c10515d2..00000000000 --- a/lib/galaxy/datatypes/wsf.py +++ /dev/null @@ -1,156 +0,0 @@ -""" -SnpFile datatype -""" - -import re -import data -from galaxy import util -from galaxy.datatypes.sniff import * -from galaxy.datatypes.tabular import Tabular -from galaxy.datatypes import metadata -from galaxy.datatypes.metadata import MetadataElement - -class SnpFile( Tabular ): - """ Webb's SNP file format """ - file_ext = 'wsf' - species_regex = re.compile('species=(\S+)') - MetadataElement( name="species", desc="species", default='', no_value='', visible=False, readonly=True ) - MetadataElement( name="scaffold", desc="scaffold column", param=metadata.ColumnParameter, default=0 ) - MetadataElement( name="pos", desc="pos column", param=metadata.ColumnParameter, default=0 ) - MetadataElement( name="ref", desc="ref column", param=metadata.ColumnParameter, default=0 ) - MetadataElement( name="rPos", desc="rPos column", param=metadata.ColumnParameter, default=0 ) - MetadataElement( name="labels", desc="Number of labels", default=0, no_value=0, visible=False, readonly=True ) - MetadataElement( name="label_for_column", desc="Mapping from column to label", default=[], no_value=[], visible=False, readonly=True ) - MetadataElement( name="columns_with_label", desc="Mapping from label to columns", param=metadata.DictParameter, default={}, no_value={}, visible=False, readonly=True ) - MetadataElement( name="column_headers", desc="Column headers", default=[], no_value=[], visible=False, readonly=True ) - - - def set_meta( self, dataset, overwrite = True, **kwd ): - Tabular.set_meta( self, dataset, overwrite=overwrite, max_data_lines=None, **kwd ) - # these two if statements work around a potential bug in metadata.py - if dataset.metadata.labels is None or dataset.metadata.labels == dataset.metadata.spec['labels'].no_value: - self._set_column_labels_metadata( dataset ) - if dataset.metadata.column_headers is None or dataset.metadata.column_headers == dataset.metadata.spec['column_headers'].no_value: - self._set_column_headers_metadata( dataset ) - self._set_columnParameter_metadata( dataset ) - - - def _set_column_labels_metadata( self, dataset ): - def build_map_from_label_to_comma_separated_column_list( labels ): - map = {} - for index, label in enumerate( labels ): - map.setdefault( label, [] ).append( index ) - - for label in map: - map[label] = ','.join( [ str( index + 1 ) for index in map[label] ] ) - return map - - def strip_list_elements( list ): - return [ element.strip() for element in list ] - - def initial_comment_lines_of_dataset( dataset ): - comment_lines = [] - if dataset.has_data(): - try: - fh = open( dataset.file_name, 'r' ) - for line in fh: - if not line.startswith('#'): - break - line = line[1:] - line = line.rstrip( '\r\n' ) - if line: - comment_lines.append( line ) - fh.close() - except: - pass - return comment_lines - - def set_metadata_from_comment_lines( dataset ): - labels = [] - comment_lines = initial_comment_lines_of_dataset( dataset ) - - for line in comment_lines: - match = SnpFile.species_regex.match( line ) - if match: - dataset.metadata.species = match.group(1) - continue - elems = line.split( '\t' ) - if len(elems) > 1: - labels = strip_list_elements( elems ) - - dataset.metadata.labels = len( labels ) - dataset.metadata.label_for_column = labels[:] - if labels: - dataset.metadata.label_for_column.insert(0, '') - dataset.metadata.columns_with_label = build_map_from_label_to_comma_separated_column_list( labels ) - - set_metadata_from_comment_lines( dataset ) - - - def _set_column_headers_metadata( self, dataset ): - if dataset.metadata.labels < dataset.metadata.columns: - column_headers = dataset.metadata.label_for_column[1:] + [ '' ] * ( dataset.metadata.columns - dataset.metadata.labels ) - else: - column_headers = dataset.metadata.label_for_column[1:dataset.metadata.columns+1] - - dataset.metadata.column_headers = column_headers - - - def _set_columnParameter_metadata( self, dataset ): - def unique_column_number_or_zero( string ): - try: - val = int( string ) - except: - val = 0 - return val - - for name in self._metadata_columnParameter_names( dataset ): - if name in dataset.metadata.columns_with_label: - if dataset.metadata.columns_with_label[name]: - column = unique_column_number_or_zero( dataset.metadata.columns_with_label[name] ) - if column: - setattr( dataset.metadata, name, column ) - - - def _metadata_columnParameter_names( self, dataset ): - for name, spec in dataset.metadata.spec.items(): - if isinstance( spec.param, metadata.ColumnParameter ): - yield name - - - def set_peek( self, dataset, line_count=None, is_multi_byte=False ): - super(Tabular, self).set_peek( dataset, line_count=line_count, is_multi_byte=is_multi_byte, skipchars=[ '#' ]) - - - def make_html_table( self, dataset, skipchars=[ '#' ] ): - """Create HTML table, used for displaying peek""" - def table_header_values( dataset ): - headers = dataset.metadata.column_headers[:] - for name in self._metadata_columnParameter_names( dataset ): - col = getattr( dataset.metadata, name ) - assert col <= dataset.metadata.columns, Exception( 'ColumnParameter %s %d > %d columns for dataset %s.' % ( name, col, dataset.metadata.columns, dataset.id ) ) - if col > 0: - headers[ col - 1 ] = name - return headers - - def table_headers( dataset ): - out = [ '' ] - headers = table_header_values( dataset ) - for index, header in enumerate( headers ): - column = index + 1 - if header: - out.append( "" % ( column, header ) ) - else: - out.append( "" % column ) - out.append( '' ) - return out - - try: - out = ['
%d.%s%d.
'] - out.extend( table_headers( dataset ) ) - out.append( self.make_html_peek_rows( dataset, skipchars=skipchars ) ) - out.append( '
' ) - out = "".join( out ) - except Exception, exc: - out = "Can't create peek %s" % exc - return out diff --git a/lib/galaxy/datatypes/xml.py b/lib/galaxy/datatypes/xml.py index d9e659bd70c..ef1f1623e28 100644 --- a/lib/galaxy/datatypes/xml.py +++ b/lib/galaxy/datatypes/xml.py @@ -104,13 +104,41 @@ class BlastXml( GenericXml ): for f in split_files: h = open(f) body = False - header = [] + header = h.readline() + if not header: + out.close() + h.close() + raise ValueError("BLAST XML file %s was empty" % f) + if header.strip() != '': + out.write(header) #for diagnosis + out.close() + h.close() + raise ValueError("%s is not an XML file!" % f) + line = h.readline() + header += line + if line.strip() not in ['', + '']: + out.write(header) #for diagnosis + out.close() + h.close() + raise ValueError("%s is not a BLAST XML file!" % f) while True: line = h.readline() - header.append(line) + if not line: + out.write(header) #for diagnosis + out.close() + h.close() + raise ValueError("BLAST XML file %s ended prematurely" % f) + header += line if "" in line: break - header = "".join(header) + if len(header) > 10000: + #Something has gone wrong, don't load too much into memory! + #Write what we have to the merged file for diagnostics + out.write(header) + out.close() + h.close() + raise ValueError("BLAST XML file %s has too long a header!" % f) if "" not in header: out.close() h.close() diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index cbe636d04b0..ce98e6ddf30 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -1,4 +1,15 @@ -import logging, threading, sys, os, time, traceback, shutil +""" +Support for running a tool in Galaxy via an internal job management system +""" + +import os +import sys +import pwd +import time +import logging +import threading +import traceback +import subprocess import galaxy from galaxy import util, model @@ -9,51 +20,16 @@ from galaxy.datatypes import metadata from galaxy.util.json import from_json_string from galaxy.util.expressions import ExpressionContext from galaxy.jobs.actions.post import ActionBox -import subprocess, pwd from galaxy.exceptions import ObjectInvalid -from sqlalchemy.sql.expression import and_, or_ - -import pkg_resources -pkg_resources.require( "PasteDeploy" ) - -from Queue import Queue, Empty - log = logging.getLogger( __name__ ) -# States for running a job. These are NOT the same as data states -JOB_WAIT, JOB_ERROR, JOB_INPUT_ERROR, JOB_INPUT_DELETED, JOB_READY, JOB_DELETED, JOB_ADMIN_DELETED = 'wait', 'error', 'input_error', 'input_deleted', 'ready', 'deleted', 'admin_deleted' - # This file, if created in the job's working directory, will be used for # setting advanced metadata properties on the job and its associated outputs. # This interface is currently experimental, is only used by the upload tool, # and should eventually become API'd TOOL_PROVIDED_JOB_METADATA_FILE = 'galaxy.json' -class JobManager( object ): - """ - Highest level interface to job management. - - TODO: Currently the app accesses "job_queue" and "job_stop_queue" directly. - This should be decoupled. - """ - def __init__( self, app ): - self.app = app - if self.app.config.get_bool( "enable_job_running", True ): - # The dispatcher launches the underlying job runners - self.dispatcher = DefaultJobDispatcher( app ) - # Queues for starting and stopping jobs - self.job_queue = JobQueue( app, self.dispatcher ) - self.job_stop_queue = JobStopQueue( app, self.dispatcher ) - if self.app.config.enable_beta_job_managers: - from galaxy.jobs.deferred import DeferredJobQueue - self.deferred_job_queue = DeferredJobQueue( app ) - else: - self.job_queue = self.job_stop_queue = NoopQueue() - def shutdown( self ): - self.job_queue.shutdown() - self.job_stop_queue.shutdown() - class Sleeper( object ): """ Provides a 'sleep' method that sleeps for a number of seconds *unless* @@ -70,238 +46,6 @@ class Sleeper( object ): self.condition.notify() self.condition.release() -class JobQueue( object ): - """ - Job manager, waits for jobs to be runnable and then dispatches to - a JobRunner. - """ - STOP_SIGNAL = object() - def __init__( self, app, dispatcher ): - """Start the job manager""" - self.app = app - self.sa_session = app.model.context - self.job_lock = False - # Should we read jobs form the database, or use an in memory queue - self.track_jobs_in_database = app.config.get_bool( 'track_jobs_in_database', False ) - # Keep track of the pid that started the job manager, only it - # has valid threads - self.parent_pid = os.getpid() - # Contains new jobs. Note this is not used if track_jobs_in_database is True - self.queue = Queue() - # Contains jobs that are waiting (only use from monitor thread) - ## This and jobs_to_check[] are closest to a "Job Queue" - self.waiting_jobs = [] - # Helper for interruptable sleep - self.sleeper = Sleeper() - self.running = True - self.dispatcher = dispatcher - self.monitor_thread = threading.Thread( target=self.__monitor ) - # Recover jobs at startup - if app.config.get_bool( 'enable_job_recovery', True ): - self.__check_jobs_at_startup() - # Start the queue - self.monitor_thread.start() - log.info( "job manager started" ) - - def __check_jobs_at_startup( self ): - """ - Checks all jobs that are in the 'new', 'queued' or 'running' state in - the database and requeues or cleans up as necessary. Only run as the - job manager starts. - """ - model = self.app.model # DBTODO Why? - for job in self.sa_session.query( model.Job ).filter( model.Job.state == model.Job.states.NEW ): - if job.tool_id not in self.app.toolbox.tools_by_id: - log.warning( "Tool '%s' removed from tool config, unable to recover job: %s" % ( job.tool_id, job.id ) ) - JobWrapper( job, self ).fail( 'This tool was disabled before the job completed. Please contact your Galaxy administrator, or' ) - else: - log.debug( "no runner: %s is still in new state, adding to the jobs queue" %job.id ) - self.queue.put( ( job.id, job.tool_id ) ) - for job in self.sa_session.query( model.Job ).enable_eagerloads( False ).filter( ( model.Job.state == model.Job.states.RUNNING ) | ( model.Job.state == model.Job.states.QUEUED ) ): - if job.tool_id not in self.app.toolbox.tools_by_id: - log.warning( "Tool '%s' removed from tool config, unable to recover job: %s" % ( job.tool_id, job.id ) ) - JobWrapper( job, self ).fail( 'This tool was disabled before the job completed. Please contact your Galaxy administrator, or' ) - elif job.job_runner_name is None: - log.debug( "no runner: %s is still in queued state, adding to the jobs queue" %job.id ) - if self.track_jobs_in_database: - job.state = model.Job.states.NEW - else: - self.queue.put( ( job.id, job.tool_id ) ) - else: - job_wrapper = JobWrapper( job, self ) - self.dispatcher.recover( job, job_wrapper ) - if self.sa_session.dirty: - self.sa_session.flush() - - def __monitor( self ): - """ - Continually iterate the waiting jobs, checking is each is ready to - run and dispatching if so. - """ - # HACK: Delay until after forking, we need a way to do post fork notification!!! - time.sleep( 10 ) - while self.running: - try: - self.__monitor_step() - except: - log.exception( "Exception in monitor_step" ) - # Sleep - self.sleeper.sleep( 1 ) - - def __monitor_step( self ): - """ - Called repeatedly by `monitor` to process waiting jobs. Gets any new - jobs (either from the database or from its own queue), then iterates - over all new and waiting jobs to check the state of the jobs each - depends on. If the job has dependencies that have not finished, it - it goes to the waiting queue. If the job has dependencies with errors, - it is marked as having errors and removed from the queue. Otherwise, - the job is dispatched. - """ - # Pull all new jobs from the queue at once - jobs_to_check = [] - if self.track_jobs_in_database: - # Clear the session so we get fresh states for job and all datasets - self.sa_session.expunge_all() - # Fetch all new jobs - jobs_to_check = self.sa_session.query( model.Job ).enable_eagerloads( False ) \ - .filter( model.Job.state == model.Job.states.NEW ).all() - else: - # Get job objects and append to watch queue for any which were - # previously waiting - for job_id in self.waiting_jobs: - jobs_to_check.append( self.sa_session.query( model.Job ).get( job_id ) ) - try: - while 1: - message = self.queue.get_nowait() - if message is self.STOP_SIGNAL: - return - # Unpack the message - job_id, tool_id = message - # Get the job object and append to watch queue - jobs_to_check.append( self.sa_session.query( model.Job ).get( job_id ) ) - except Empty: - pass - # Iterate over new and waiting jobs and look for any that are - # ready to run - new_waiting_jobs = [] - for job in jobs_to_check: - try: - # Check the job's dependencies, requeue if they're not done - job_state = self.__check_if_ready_to_run( job ) - if job_state == JOB_WAIT: - if not self.track_jobs_in_database: - new_waiting_jobs.append( job.id ) - elif job_state == JOB_INPUT_ERROR: - log.info( "job %d unable to run: one or more inputs in error state" % job.id ) - elif job_state == JOB_INPUT_DELETED: - log.info( "job %d unable to run: one or more inputs deleted" % job.id ) - elif job_state == JOB_READY: - if self.job_lock: - log.info( "Job dispatch attempted for %s, but prevented by administrative lock." % job.id ) - if not self.track_jobs_in_database: - new_waiting_jobs.append( job.id ) - else: - self.dispatcher.put( JobWrapper( job, self ) ) - log.info( "job %d dispatched" % job.id ) - elif job_state == JOB_DELETED: - log.info( "job %d deleted by user while still queued" % job.id ) - elif job_state == JOB_ADMIN_DELETED: - log.info( "job %d deleted by admin while still queued" % job.id ) - else: - log.error( "unknown job state '%s' for job %d" % ( job_state, job.id ) ) - if not self.track_jobs_in_database: - new_waiting_jobs.append( job.id ) - except Exception: - log.exception( "failure running job %d" % job.id ) - # Update the waiting list - self.waiting_jobs = new_waiting_jobs - # Done with the session - self.sa_session.remove() - - def __check_if_ready_to_run( self, job ): - """ - Check if a job is ready to run by verifying that each of its input - datasets is ready (specifically in the OK state). If any input dataset - has an error, fail the job and return JOB_INPUT_ERROR. If any input - dataset is deleted, fail the job and return JOB_INPUT_DELETED. If all - input datasets are in OK state, return JOB_READY indicating that the - job can be dispatched. Otherwise, return JOB_WAIT indicating that input - datasets are still being prepared. - """ - if job.state == model.Job.states.DELETED: - return JOB_DELETED - elif job.state == model.Job.states.ERROR: - return JOB_ADMIN_DELETED - elif self.app.config.enable_quotas: - quota = self.app.quota_agent.get_quota( job.user ) - if quota is not None: - try: - usage = self.app.quota_agent.get_usage( user=job.user, history=job.history ) - if usage > quota: - return JOB_WAIT - except AssertionError, e: - pass # No history, should not happen with an anon user - for dataset_assoc in job.input_datasets + job.input_library_datasets: - idata = dataset_assoc.dataset - if not idata: - continue - # don't run jobs for which the input dataset was deleted - if idata.deleted: - JobWrapper( job, self ).fail( "input data %d (file: %s) was deleted before the job started" % ( idata.hid, idata.file_name ) ) - return JOB_INPUT_DELETED - # an error in the input data causes us to bail immediately - elif idata.state == idata.states.ERROR: - JobWrapper( job, self ).fail( "input data %d is in error state" % ( idata.hid ) ) - return JOB_INPUT_ERROR - elif idata.state == idata.states.FAILED_METADATA: - JobWrapper( job, self ).fail( "input data %d failed to properly set metadata" % ( idata.hid ) ) - return JOB_INPUT_ERROR - elif idata.state != idata.states.OK and not ( idata.state == idata.states.SETTING_METADATA and job.tool_id is not None and job.tool_id == self.app.datatypes_registry.set_external_metadata_tool.id ): - # need to requeue - return JOB_WAIT - return self.__check_user_jobs( job ) - - def __check_user_jobs( self, job ): - if not self.app.config.user_job_limit: - return JOB_READY - if job.user: - count = self.sa_session.query( model.Job ).enable_eagerloads( False ) \ - .filter( and_( model.Job.user_id == job.user.id, - or_( model.Job.state == model.Job.states.RUNNING, - model.Job.state == model.Job.states.QUEUED ) ) ).count() - elif job.galaxy_session: - count = self.sa_session.query( model.Job ).enable_eagerloads( False ) \ - .filter( and_( model.Job.session_id == job.galaxy_session.id, - or_( model.Job.state == model.Job.states.RUNNING, - model.Job.state == model.Job.states.QUEUED ) ) ).count() - else: - log.warning( 'Job %s is not associated with a user or session so job concurrency limit cannot be checked.' % job.id ) - return JOB_READY - if count >= self.app.config.user_job_limit: - return JOB_WAIT - return JOB_READY - - def put( self, job_id, tool ): - """Add a job to the queue (by job identifier)""" - if not self.track_jobs_in_database: - self.queue.put( ( job_id, tool.id ) ) - self.sleeper.wake() - - def shutdown( self ): - """Attempts to gracefully shut down the worker thread""" - if self.parent_pid != os.getpid(): - # We're not the real job queue, do nothing - return - else: - log.info( "sending stop signal to worker thread" ) - self.running = False - if not self.track_jobs_in_database: - self.queue.put( self.STOP_SIGNAL ) - self.sleeper.wake() - log.info( "job queue stopped" ) - self.dispatcher.shutdown() - class JobWrapper( object ): """ Wraps a 'model.Job' with convenience methods for running processes and @@ -400,9 +144,12 @@ class JobWrapper( object ): self.file_name = dataset.file_name self.metadata = dict() self.children = [] - jeha = self.sa_session.query( model.JobExportHistoryArchive ).filter_by( job=job ).first() - if jeha: - out_data[ "output_file" ] = FakeDatasetAssociation( dataset=jeha.dataset ) + special = self.sa_session.query( model.JobExportHistoryArchive ).filter_by( job=job ).first() + if not special: + special = self.sa_session.query( model.GenomeIndexToolData ).filter_by( job=job ).first() + if special: + out_data[ "output_file" ] = FakeDatasetAssociation( dataset=special.dataset ) + # These can be passed on the command line if wanted as $userId $userEmail if job.history and job.history.user: # check for anonymous user! userId = '%d' % job.history.user.id @@ -698,8 +445,11 @@ class JobWrapper( object ): param_dict = self.tool.params_from_strings( param_dict, self.app ) # Check for and move associated_files self.tool.collect_associated_files(out_data, self.working_directory) + gitd = self.sa_session.query( model.GenomeIndexToolData ).filter_by( job=job ).first() + if gitd: + self.tool.collect_associated_files({'' : gitd}, self.working_directory) # Create generated output children and primary datasets and add to param_dict - collected_datasets = {'children':self.tool.collect_child_datasets(out_data),'primary':self.tool.collect_primary_datasets(out_data)} + collected_datasets = {'children':self.tool.collect_child_datasets(out_data, self.working_directory),'primary':self.tool.collect_primary_datasets(out_data, self.working_directory)} param_dict.update({'__collected_datasets__':collected_datasets}) # Certain tools require tasks to be completed after job execution # ( this used to be performed in the "exec_after_process" hook, but hooks are deprecated ). @@ -736,6 +486,7 @@ class JobWrapper( object ): self.external_output_metadata.cleanup_external_metadata( self.sa_session ) galaxy.tools.imp_exp.JobExportHistoryArchiveWrapper( self.job_id ).cleanup_after_job( self.sa_session ) galaxy.tools.imp_exp.JobImportHistoryArchiveWrapper( self.job_id ).cleanup_after_job( self.sa_session ) + galaxy.tools.genome_index.GenomeIndexToolWrapper( self.job_id ).postprocessing( self.sa_session, self.app ) self.app.object_store.delete(self.get_job(), base_dir='job_work', entire_dir=True, dir_only=True, extra_dir=str(self.job_id)) except: log.exception( "Unable to cleanup job %d" % self.job_id ) @@ -791,9 +542,11 @@ class JobWrapper( object ): else: return self.false_path job = self.get_job() - # Job output datasets are combination of output datasets, library datasets, and jeha datasets. - jeha = self.sa_session.query( model.JobExportHistoryArchive ).filter_by( job=job ).first() - jeha_false_path = None + # Job output datasets are combination of history, library, jeha and gitd datasets. + special = self.sa_session.query( model.JobExportHistoryArchive ).filter_by( job=job ).first() + if not special: + special = self.sa_session.query( model.GenomeIndexToolData ).filter_by( job=job ).first() + false_path = None if self.app.config.outputs_to_working_directory: self.output_paths = [] self.output_hdas_and_paths = {} @@ -802,14 +555,14 @@ class JobWrapper( object ): dsp = DatasetPath( hda.dataset.id, hda.dataset.file_name, false_path ) self.output_paths.append( dsp ) self.output_hdas_and_paths[name] = hda, dsp - if jeha: - jeha_false_path = os.path.abspath( os.path.join( self.working_directory, "galaxy_dataset_%d.dat" % jeha.dataset.id ) ) + if special: + false_path = os.path.abspath( os.path.join( self.working_directory, "galaxy_dataset_%d.dat" % special.dataset.id ) ) else: results = [ ( da.name, da.dataset, DatasetPath( da.dataset.dataset.id, da.dataset.file_name ) ) for da in job.output_datasets + job.output_library_datasets ] self.output_paths = [t[2] for t in results] self.output_hdas_and_paths = dict([(t[0], t[1:]) for t in results]) - if jeha: - dsp = DatasetPath( jeha.dataset.id, jeha.dataset.file_name, jeha_false_path ) + if special: + dsp = DatasetPath( special.dataset.id, special.dataset.file_name, false_path ) self.output_paths.append( dsp ) return self.output_paths @@ -1177,179 +930,13 @@ class TaskWrapper(JobWrapper): # There is no metadata setting for tasks. This is handled after the merge, at the job level. return "" -class DefaultJobDispatcher( object ): - def __init__( self, app ): - self.app = app - self.job_runners = {} - start_job_runners = ["local"] - if app.config.start_job_runners is not None: - start_job_runners.extend( app.config.start_job_runners.split(",") ) - if app.config.use_tasked_jobs: - start_job_runners.append("tasks") - for name in start_job_runners: - self._load_plugin( name ) - - def _load_plugin( self, name ): - module_name = 'galaxy.jobs.runners.' + name - try: - module = __import__( module_name ) - except: - log.exception( 'Job runner is not loadable: %s' % module_name ) - return - for comp in module_name.split( "." )[1:]: - module = getattr( module, comp ) - if '__all__' not in dir( module ): - log.error( 'Runner "%s" does not contain a list of exported classes in __all__' % module_name ) - return - for obj in module.__all__: - display_name = ':'.join( ( module_name, obj ) ) - runner = getattr( module, obj ) - self.job_runners[name] = runner( self.app ) - log.debug( 'Loaded job runner: %s' % display_name ) - - def __get_runner_name( self, job_wrapper ): - if self.app.config.use_tasked_jobs and job_wrapper.tool.parallelism is not None and not isinstance(job_wrapper, TaskWrapper): - runner_name = "tasks" - else: - runner_name = ( job_wrapper.get_job_runner().split(":", 1) )[0] - return runner_name - - def put( self, job_wrapper ): - try: - runner_name = self.__get_runner_name( job_wrapper ) - if self.app.config.use_tasked_jobs and job_wrapper.tool.parallelism is not None and isinstance(job_wrapper, TaskWrapper): - #DBTODO Refactor - log.debug( "dispatching task %s, of job %d, to %s runner" %( job_wrapper.task_id, job_wrapper.job_id, runner_name ) ) - else: - log.debug( "dispatching job %d to %s runner" %( job_wrapper.job_id, runner_name ) ) - self.job_runners[runner_name].put( job_wrapper ) - except KeyError: - log.error( 'put(): (%s) Invalid job runner: %s' % ( job_wrapper.job_id, runner_name ) ) - job_wrapper.fail( 'Unable to run job due to a misconfiguration of the Galaxy job running system. Please contact a site administrator.' ) - - def stop( self, job ): - runner_name = ( job.job_runner_name.split(":", 1) )[0] - log.debug( "stopping job %d in %s runner" %( job.id, runner_name ) ) - try: - self.job_runners[runner_name].stop_job( job ) - except KeyError: - log.error( 'stop(): (%s) Invalid job runner: %s' % ( job_wrapper.job_id, runner_name ) ) - # Job and output dataset states have already been updated, so nothing is done here. - - def recover( self, job, job_wrapper ): - runner_name = ( job.job_runner_name.split(":", 1) )[0] - log.debug( "recovering job %d in %s runner" %( job.id, runner_name ) ) - try: - self.job_runners[runner_name].recover( job, job_wrapper ) - except KeyError: - log.error( 'recover(): (%s) Invalid job runner: %s' % ( job_wrapper.job_id, runner_name ) ) - job_wrapper.fail( 'Unable to run job due to a misconfiguration of the Galaxy job running system. Please contact a site administrator.' ) - - def shutdown( self ): - for runner in self.job_runners.itervalues(): - runner.shutdown() - -class JobStopQueue( object ): - """ - A queue for jobs which need to be terminated prematurely. - """ - STOP_SIGNAL = object() - def __init__( self, app, dispatcher ): - self.app = app - self.sa_session = app.model.context - self.dispatcher = dispatcher - - self.track_jobs_in_database = app.config.get_bool( 'track_jobs_in_database', False ) - - # Keep track of the pid that started the job manager, only it - # has valid threads - self.parent_pid = os.getpid() - # Contains new jobs. Note this is not used if track_jobs_in_database is True - self.queue = Queue() - - # Contains jobs that are waiting (only use from monitor thread) - self.waiting = [] - - # Helper for interruptable sleep - self.sleeper = Sleeper() - self.running = True - self.monitor_thread = threading.Thread( target=self.monitor ) - self.monitor_thread.start() - log.info( "job stopper started" ) - - def monitor( self ): - """ - Continually iterate the waiting jobs, stop any that are found. - """ - # HACK: Delay until after forking, we need a way to do post fork notification!!! - time.sleep( 10 ) - while self.running: - try: - self.monitor_step() - except: - log.exception( "Exception in monitor_step" ) - # Sleep - self.sleeper.sleep( 1 ) - - def monitor_step( self ): - """ - Called repeatedly by `monitor` to stop jobs. - """ - # Pull all new jobs from the queue at once - jobs_to_check = [] - if self.track_jobs_in_database: - # Clear the session so we get fresh states for job and all datasets - self.sa_session.expunge_all() - # Fetch all new jobs - newly_deleted_jobs = self.sa_session.query( model.Job ).enable_eagerloads( False ) \ - .filter( model.Job.state == model.Job.states.DELETED_NEW ).all() - for job in newly_deleted_jobs: - jobs_to_check.append( ( job, None ) ) - # Also pull from the queue (in the case of Administrative stopped jobs) - try: - while 1: - message = self.queue.get_nowait() - if message is self.STOP_SIGNAL: - return - # Unpack the message - job_id, error_msg = message - # Get the job object and append to watch queue - jobs_to_check.append( ( self.sa_session.query( model.Job ).get( job_id ), error_msg ) ) - except Empty: - pass - for job, error_msg in jobs_to_check: - if error_msg is not None: - job.state = job.states.ERROR - job.info = error_msg - else: - job.state = job.states.DELETED - self.sa_session.add( job ) - self.sa_session.flush() - if job.job_runner_name is not None: - # tell the dispatcher to stop the job - self.dispatcher.stop( job ) - - def put( self, job_id, error_msg=None ): - self.queue.put( ( job_id, error_msg ) ) - - def shutdown( self ): - """Attempts to gracefully shut down the worker thread""" - if self.parent_pid != os.getpid(): - # We're not the real job queue, do nothing - return - else: - log.info( "sending stop signal to worker thread" ) - self.running = False - if not self.track_jobs_in_database: - self.queue.put( self.STOP_SIGNAL ) - self.sleeper.wake() - log.info( "job stopper stopped" ) - class NoopQueue( object ): """ Implements the JobQueue / JobStopQueue interface but does nothing """ def put( self, *args ): return + def put_stop( self, *args ): + return def shutdown( self ): return diff --git a/lib/galaxy/jobs/actions/post.py b/lib/galaxy/jobs/actions/post.py index 368af7db4ef..670420d8a52 100644 --- a/lib/galaxy/jobs/actions/post.py +++ b/lib/galaxy/jobs/actions/post.py @@ -203,11 +203,10 @@ class RenameDatasetAction(DefaultJobAction): @classmethod def get_config_form(cls, trans): form = """ - if ((pja.action_arguments !== undefined) && (pja.action_arguments.newname !== undefined)){ + if (pja.action_arguments && pja.action_arguments.newname){ p_str += "\ "; } - else{ p_str += "\ "; diff --git a/lib/galaxy/jobs/deferred/deferred_indexing.py b/lib/galaxy/jobs/deferred/deferred_indexing.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lib/galaxy/jobs/deferred/genome_transfer.py b/lib/galaxy/jobs/deferred/genome_transfer.py new file mode 100644 index 00000000000..bc0e536aa56 --- /dev/null +++ b/lib/galaxy/jobs/deferred/genome_transfer.py @@ -0,0 +1,250 @@ +""" +Module for managing genome transfer jobs. +""" +import logging, shutil, gzip, bz2, zipfile, tempfile, tarfile, sys + +from galaxy import eggs +from sqlalchemy import and_ + +from galaxy.util.odict import odict +from galaxy.workflow.modules import module_factory +from galaxy.jobs.actions.post import ActionBox + +from galaxy.tools.parameters import visit_input_values +from galaxy.tools.parameters.basic import DataToolParameter +from galaxy.tools.data import ToolDataTableManager + +from galaxy.datatypes.checkers import * +from galaxy.datatypes.sequence import Fasta +from data_transfer import * + +log = logging.getLogger( __name__ ) + +__all__ = [ 'GenomeTransferPlugin' ] + +class GenomeTransferPlugin( DataTransfer ): + + locations = {} + + def __init__( self, app ): + super( GenomeTransferPlugin, self ).__init__( app ) + self.app = app + self.tool = app.toolbox.tools_by_id['__GENOME_INDEX__'] + self.sa_session = app.model.context.current + tdtman = ToolDataTableManager() + xmltree = tdtman.load_from_config_file(app.config.tool_data_table_config_path) + for node in xmltree: + table = node.get('name') + location = node.findall('file')[0].get('path') + self.locations[table] = location + + def create_job( self, trans, url, dbkey, intname, indexes ): + job = trans.app.transfer_manager.new( protocol='http', url=url ) + params = dict( user=trans.user.id, transfer_job_id=job.id, protocol='http', type='init_transfer', url=url, dbkey=dbkey, indexes=indexes, intname=intname, liftover=None ) + deferred = trans.app.model.DeferredJob( state = self.app.model.DeferredJob.states.NEW, plugin = 'GenomeTransferPlugin', params = params ) + self.sa_session.add( deferred ) + self.sa_session.flush() + return deferred.id + + def check_job( self, job ): + if job.params['type'] == 'init_transfer': + if not hasattr(job, 'transfer_job'): + job.transfer_job = self.sa_session.query( self.app.model.TransferJob ).get( int( job.params[ 'transfer_job_id' ] ) ) + else: + self.sa_session.refresh( job.transfer_job ) + if job.transfer_job.state == 'done': + transfer = job.transfer_job + transfer.state = 'downloaded' + job.params['type'] = 'extract_transfer' + self.sa_session.add( job ) + self.sa_session.add( transfer ) + self.sa_session.flush() + return self.job_states.READY + elif job.transfer_job.state == 'running': + return self.job_states.WAIT + elif job.transfer_job.state == 'new': + assert job.params[ 'protocol' ] in [ 'http', 'ftp', 'https' ], 'Unknown protocol %s' % job.params[ 'protocol' ] + self.app.transfer_manager.run( job.transfer_job ) + self.sa_session.add( job.transfer_job ) + self.sa_session.flush() + return self.job_states.WAIT + else: + log.error( "An error occurred while downloading from %s" % job.params[ 'url' ] ) + return self.job_states.INVALID + elif job.params[ 'type' ] == 'extract_transfer': + return self.job_states.READY + + def get_job_status( self, jobid ): + job = self.sa_session.query( self.app.model.DeferredJob ).get( int( jobid ) ) + if not hasattr( job, 'transfer_job' ): + job.transfer_job = self.sa_session.query( self.app.model.TransferJob ).get( int( job.params[ 'transfer_job_id' ] ) ) + else: + self.sa_session.refresh( job.transfer_job ) + return job + + def run_job( self, job ): + params = job.params + dbkey = params[ 'dbkey' ] + if not hasattr( job, 'transfer_job' ): + job.transfer_job = self.sa_session.query( self.app.model.TransferJob ).get( int( job.params[ 'transfer_job_id' ] ) ) + else: + self.sa_session.refresh( job.transfer_job ) + transfer = job.transfer_job + if params[ 'type' ] == 'extract_transfer': + CHUNK_SIZE = 2**20 + destpath = os.path.join( self.app.config.get( 'genome_data_path', 'tool-data/genome' ), job.params[ 'dbkey' ], 'seq' ) + destfile = '%s.fa' % job.params[ 'dbkey' ] + destfilepath = os.path.join( destpath, destfile ) + tmpprefix = '%s_%s_download_unzip_' % ( job.params['dbkey'], job.params[ 'transfer_job_id' ] ) + tmppath = os.path.dirname( os.path.abspath( transfer.path ) ) + if not os.path.exists( destpath ): + os.makedirs( destpath ) + protocol = job.params[ 'protocol' ] + data_type = self._check_compress( transfer.path ) + if data_type is None: + sniffer = Fasta() + if sniffer.sniff( transfer.path ): + data_type = 'fasta' + fd, uncompressed = tempfile.mkstemp( prefix=tmpprefix, dir=tmppath, text=False ) + if data_type in [ 'tar.gzip', 'tar.bzip' ]: + fp = open( transfer.path, 'r' ) + tar = tarfile.open( mode = 'r:*', bufsize = CHUNK_SIZE, fileobj = fp ) + files = tar.getmembers() + for filename in files: + z = tar.extractfile(filename) + try: + chunk = z.read( CHUNK_SIZE ) + except IOError: + os.close( fd ) + log.error( 'Problem decompressing compressed data' ) + exit() + if not chunk: + break + os.write( fd, chunk ) + os.write( fd, '\n' ) + os.close( fd ) + tar.close() + fp.close() + elif data_type == 'gzip': + compressed = gzip.open( transfer.path, mode = 'rb' ) + while 1: + try: + chunk = compressed.read( CHUNK_SIZE ) + except IOError: + compressed.close() + log.error( 'Problem decompressing compressed data' ) + exit() + if not chunk: + break + os.write( fd, chunk ) + os.write( fd, '\n' ) + os.close( fd ) + compressed.close() + elif data_type == 'bzip': + compressed = bz2.BZ2File( transfer.path, mode = 'r' ) + while 1: + try: + chunk = compressed.read( CHUNK_SIZE ) + except IOError: + compressed.close() + log.error( 'Problem decompressing compressed data' ) + exit() + if not chunk: + break + os.write( fd, chunk ) + os.write( fd, '\n' ) + os.close( fd ) + compressed.close() + elif data_type == 'zip': + uncompressed_name = None + unzipped = False + z = zipfile.ZipFile( transfer.path ) + z.debug = 3 + for name in z.namelist(): + if name.endswith('/'): + continue + if sys.version_info[:2] >= ( 2, 6 ): + zipped_file = z.open( name ) + while 1: + try: + chunk = zipped_file.read( CHUNK_SIZE ) + except IOError: + os.close( fd ) + log.error( 'Problem decompressing zipped data' ) + return self.app.model.DeferredJob.states.INVALID + if not chunk: + break + os.write( fd, chunk ) + os.write( fd, '\n' ) + zipped_file.close() + else: + try: + outfile = open( fd, 'wb' ) + outfile.write( z.read( name ) ) + outfile.close() + except IOError: + os.close( fd ) + log.error( 'Problem decompressing zipped data' ) + return + os.close( fd ) + z.close() + elif data_type == 'fasta': + uncompressed = transfer.path + else: + job.state = self.app.model.DeferredJob.states.INVALID + log.error( "Unrecognized compression format for file %s." % transfer.path ) + self.sa_session.add( job ) + self.sa_session.flush() + return + shutil.move( uncompressed, destfilepath ) + if os.path.exists( transfer.path ): + os.remove( transfer.path ) + os.chmod( destfilepath, 0644 ) + fastaline = '\t'.join( [ dbkey, dbkey, params[ 'intname' ], os.path.abspath( destfilepath ) ] ) + self._add_line( 'all_fasta', fastaline ) + job.state = self.app.model.DeferredJob.states.OK + job.params[ 'type' ] = 'finish_transfer' + transfer.path = os.path.abspath(destfilepath) + transfer.state = 'done' + self.sa_session.add( job ) + self.sa_session.add( transfer ) + self.sa_session.flush() + if transfer.state == 'done' and params[ 'indexes' ] is not None: + for indexer in params[ 'indexes' ]: + incoming = dict(indexer=indexer, dbkey=params[ 'dbkey' ], intname=params[ 'intname' ], path=transfer.path, user=params['user'] ) + deferred = self.tool.execute( self, set_output_hid=False, history=None, incoming=incoming, transfer=transfer, deferred=job ) + return self.app.model.DeferredJob.states.OK + + def _check_compress( self, filepath ): + retval = '' + if tarfile.is_tarfile( filepath ): + retval = 'tar.' + if check_zip( filepath ): + return 'zip' + is_bzipped, is_valid = check_bz2( filepath ) + if is_bzipped and is_valid: + return retval + 'bzip' + is_gzipped, is_valid = check_gzip( filepath ) + if is_gzipped and is_valid: + return retval + 'gzip' + return None + + def _add_line( self, locfile, newline ): + filepath = self.locations[ locfile ] + origlines = [] + output = [] + comments = [] + with open( filepath, 'r' ) as destfile: + for line in destfile: + if line.startswith( '#' ): + comments.append( line.strip() ) + else: + origlines.append( line.strip() ) + if newline not in origlines: + origlines.append( newline ) + output.extend( comments ) + origlines.sort() + output.extend( origlines ) + with open( filepath, 'w+' ) as destfile: + destfile.write( '\n'.join( output ) ) + diff --git a/lib/galaxy/jobs/deferred/liftover_transfer.py b/lib/galaxy/jobs/deferred/liftover_transfer.py new file mode 100644 index 00000000000..4423c19e96f --- /dev/null +++ b/lib/galaxy/jobs/deferred/liftover_transfer.py @@ -0,0 +1,134 @@ +""" +Module for managing genome transfer jobs. +""" +import logging, shutil, gzip, tempfile, sys + +from galaxy import eggs +from sqlalchemy import and_ + +from galaxy.util.odict import odict +from galaxy.workflow.modules import module_factory +from galaxy.jobs.actions.post import ActionBox + +from galaxy.tools.parameters import visit_input_values +from galaxy.tools.parameters.basic import DataToolParameter +from galaxy.tools.data import ToolDataTableManager + +from galaxy.datatypes.checkers import * + +from data_transfer import * + +log = logging.getLogger( __name__ ) + +__all__ = [ 'LiftOverTransferPlugin' ] + +class LiftOverTransferPlugin( DataTransfer ): + + locations = {} + + def __init__( self, app ): + super( LiftOverTransferPlugin, self ).__init__( app ) + self.app = app + self.sa_session = app.model.context.current + + def create_job( self, trans, url, dbkey, from_genome, to_genome, destfile ): + job = trans.app.transfer_manager.new( protocol='http', url=url ) + params = dict( user=trans.user.id, transfer_job_id=job.id, protocol='http', + type='init_transfer', dbkey=dbkey, from_genome=from_genome, + to_genome=to_genome, destfile=destfile ) + deferred = trans.app.model.DeferredJob( state = self.app.model.DeferredJob.states.NEW, plugin = 'LiftOverTransferPlugin', params = params ) + self.sa_session.add( deferred ) + self.sa_session.flush() + return deferred.id + + def check_job( self, job ): + if job.params['type'] == 'init_transfer': + if not hasattr(job, 'transfer_job'): + job.transfer_job = self.sa_session.query( self.app.model.TransferJob ).get( int( job.params[ 'transfer_job_id' ] ) ) + else: + self.sa_session.refresh( job.transfer_job ) + if job.transfer_job.state == 'done': + transfer = job.transfer_job + transfer.state = 'downloaded' + job.params['type'] = 'extract_transfer' + self.sa_session.add( job ) + self.sa_session.add( transfer ) + self.sa_session.flush() + return self.job_states.READY + elif job.transfer_job.state == 'running': + return self.job_states.WAIT + elif job.transfer_job.state == 'new': + assert job.params[ 'protocol' ] in [ 'http', 'ftp', 'https' ], 'Unknown protocol %s' % job.params[ 'protocol' ] + self.app.transfer_manager.run( job.transfer_job ) + self.sa_session.add( job.transfer_job ) + self.sa_session.flush() + return self.job_states.WAIT + else: + log.error( "An error occurred while downloading from %s" % job.params[ 'url' ] ) + return self.job_states.INVALID + elif job.params[ 'type' ] == 'extract_transfer': + return self.job_states.READY + + def get_job_status( self, jobid ): + job = self.sa_session.query( self.app.model.DeferredJob ).get( int( jobid ) ) + return job + + def run_job( self, job ): + params = job.params + dbkey = params[ 'dbkey' ] + source = params[ 'from_genome' ] + target = params[ 'to_genome' ] + if not hasattr( job, 'transfer_job' ): + job.transfer_job = self.sa_session.query( self.app.model.TransferJob ).get( int( job.params[ 'transfer_job_id' ] ) ) + else: + self.sa_session.refresh( job.transfer_job ) + transfer = job.transfer_job + if params[ 'type' ] == 'extract_transfer': + CHUNK_SIZE = 2**20 + destpath = os.path.join( self.app.config.get( 'genome_data_path', 'tool-data/genome' ), job.params[ 'dbkey' ], 'liftOver' ) + destfile = job.params[ 'destfile' ] + destfilepath = os.path.join( destpath, destfile ) + tmpprefix = '%s_%s_download_unzip_' % ( job.params['dbkey'], job.params[ 'transfer_job_id' ] ) + tmppath = os.path.dirname( os.path.abspath( transfer.path ) ) + if not os.path.exists( destpath ): + os.makedirs( destpath ) + fd, uncompressed = tempfile.mkstemp( prefix=tmpprefix, dir=tmppath, text=False ) + chain = gzip.open( transfer.path, 'rb' ) + while 1: + try: + chunk = chain.read( CHUNK_SIZE ) + except IOError: + os.close( fd ) + log.error( 'Problem decompressing compressed data' ) + exit() + if not chunk: + break + os.write( fd, chunk ) + os.close( fd ) + chain.close() + # Replace the gzipped file with the decompressed file if it's safe to do so + shutil.move( uncompressed, destfilepath ) + os.remove( transfer.path ) + os.chmod( destfilepath, 0644 ) + locline = '\t'.join( [ source, target, os.path.abspath( destfilepath ) ] ) + self._add_line( locline ) + job.state = self.app.model.DeferredJob.states.OK + job.params[ 'type' ] = 'finish_transfer' + transfer.path = os.path.abspath(destfilepath) + transfer.state = 'done' + self.sa_session.add( job ) + self.sa_session.add( transfer ) + self.sa_session.flush() + return self.app.model.DeferredJob.states.OK + + def _add_line( self, newline ): + filepath = 'tool-data/liftOver.loc' + origlines = [] + with open( filepath, 'r' ) as destfile: + for line in destfile: + origlines.append( line.strip() ) + if newline not in origlines: + origlines.append( newline ) + with open( filepath, 'w+' ) as destfile: + destfile.write( '\n'.join( origlines ) ) + diff --git a/lib/galaxy/jobs/handler.py b/lib/galaxy/jobs/handler.py new file mode 100644 index 00000000000..33a334263fc --- /dev/null +++ b/lib/galaxy/jobs/handler.py @@ -0,0 +1,430 @@ +""" +Galaxy job handler, prepares, runs, tracks, and finishes Galaxy jobs +""" + +import os +import time +import logging +import threading +from Queue import Queue, Empty + +from sqlalchemy.sql.expression import and_, or_ + +from galaxy import util, model +from galaxy.jobs import Sleeper, JobWrapper, TaskWrapper + +log = logging.getLogger( __name__ ) + +# States for running a job. These are NOT the same as data states +JOB_WAIT, JOB_ERROR, JOB_INPUT_ERROR, JOB_INPUT_DELETED, JOB_READY, JOB_DELETED, JOB_ADMIN_DELETED = 'wait', 'error', 'input_error', 'input_deleted', 'ready', 'deleted', 'admin_deleted' + +class JobHandler( object ): + """ + Handle the preparation, running, tracking, and finishing of jobs + """ + def __init__( self, app ): + self.app = app + # The dispatcher launches the underlying job runners + self.dispatcher = DefaultJobDispatcher( app ) + # Queues for starting and stopping jobs + self.job_queue = JobHandlerQueue( app, self.dispatcher ) + self.job_stop_queue = JobHandlerStopQueue( app, self.dispatcher ) + def start( self ): + self.job_queue.start() + def shutdown( self ): + self.job_queue.shutdown() + self.job_stop_queue.shutdown() + +class JobHandlerQueue( object ): + """ + Job manager, waits for jobs to be runnable and then dispatches to + a JobRunner. + """ + STOP_SIGNAL = object() + def __init__( self, app, dispatcher ): + """Start the job manager""" + self.app = app + self.dispatcher = dispatcher + + self.sa_session = app.model.context + self.track_jobs_in_database = self.app.config.track_jobs_in_database + + # Keep track of the pid that started the job manager, only it + # has valid threads + self.parent_pid = os.getpid() + # Contains new jobs. Note this is not used if track_jobs_in_database is True + self.queue = Queue() + # Contains jobs that are waiting (only use from monitor thread) + ## This and jobs_to_check[] are closest to a "Job Queue" + self.waiting_jobs = [] + # Helper for interruptable sleep + self.sleeper = Sleeper() + self.running = True + self.monitor_thread = threading.Thread( target=self.__monitor ) + + def start( self ): + """ + The JobManager should start, and then start its Handler, if it has one. + """ + # Recover jobs at startup + self.__check_jobs_at_startup() + # Start the queue + self.monitor_thread.start() + log.info( "job handler queue started" ) + + def __check_jobs_at_startup( self ): + """ + Checks all jobs that are in the 'new', 'queued' or 'running' state in + the database and requeues or cleans up as necessary. Only run as the + job manager starts. + """ + for job in self.sa_session.query( model.Job ).enable_eagerloads( False ) \ + .filter( ( ( model.Job.state == model.Job.states.NEW ) \ + | ( model.Job.state == model.Job.states.RUNNING ) \ + | ( model.Job.state == model.Job.states.QUEUED ) ) \ + & ( model.Job.handler == self.app.config.server_name ) ): + if job.tool_id not in self.app.toolbox.tools_by_id: + log.warning( "(%s) Tool '%s' removed from tool config, unable to recover job" % ( job.id, job.tool_id ) ) + JobWrapper( job, self ).fail( 'This tool was disabled before the job completed. Please contact your Galaxy administrator.' ) + elif job.job_runner_name is None: + log.debug( "(%s) No job runner assigned and job still in '%s' state, adding to the job handler queue" % ( job.id, job.state ) ) + if self.track_jobs_in_database: + job.state = model.Job.states.NEW + else: + self.queue.put( ( job.id, job.tool_id ) ) + else: + job_wrapper = JobWrapper( job, self ) + self.dispatcher.recover( job, job_wrapper ) + if self.sa_session.dirty: + self.sa_session.flush() + + def __monitor( self ): + """ + Continually iterate the waiting jobs, checking is each is ready to + run and dispatching if so. + """ + while self.running: + try: + self.__monitor_step() + except: + log.exception( "Exception in monitor_step" ) + # Sleep + self.sleeper.sleep( 1 ) + + def __monitor_step( self ): + """ + Called repeatedly by `monitor` to process waiting jobs. Gets any new + jobs (either from the database or from its own queue), then iterates + over all new and waiting jobs to check the state of the jobs each + depends on. If the job has dependencies that have not finished, it + it goes to the waiting queue. If the job has dependencies with errors, + it is marked as having errors and removed from the queue. Otherwise, + the job is dispatched. + """ + # Pull all new jobs from the queue at once + jobs_to_check = [] + if self.track_jobs_in_database: + # Clear the session so we get fresh states for job and all datasets + self.sa_session.expunge_all() + # Fetch all new jobs + jobs_to_check = self.sa_session.query( model.Job ).enable_eagerloads( False ) \ + .filter( ( model.Job.state == model.Job.states.NEW ) \ + & ( model.Job.handler == self.app.config.server_name ) ).all() + else: + # Get job objects and append to watch queue for any which were + # previously waiting + for job_id in self.waiting_jobs: + jobs_to_check.append( self.sa_session.query( model.Job ).get( job_id ) ) + try: + while 1: + message = self.queue.get_nowait() + if message is self.STOP_SIGNAL: + return + # Unpack the message + job_id, tool_id = message + # Get the job object and append to watch queue + jobs_to_check.append( self.sa_session.query( model.Job ).get( job_id ) ) + except Empty: + pass + # Iterate over new and waiting jobs and look for any that are + # ready to run + new_waiting_jobs = [] + for job in jobs_to_check: + try: + # Check the job's dependencies, requeue if they're not done + job_state = self.__check_if_ready_to_run( job ) + if job_state == JOB_WAIT: + if not self.track_jobs_in_database: + new_waiting_jobs.append( job.id ) + elif job_state == JOB_INPUT_ERROR: + log.info( "(%d) Job unable to run: one or more inputs in error state" % job.id ) + elif job_state == JOB_INPUT_DELETED: + log.info( "(%d) Job unable to run: one or more inputs deleted" % job.id ) + elif job_state == JOB_READY: + self.dispatcher.put( JobWrapper( job, self ) ) + log.info( "(%d) Job dispatched" % job.id ) + elif job_state == JOB_DELETED: + log.info( "(%d) Job deleted by user while still queued" % job.id ) + elif job_state == JOB_ADMIN_DELETED: + log.info( "(%d) Job deleted by admin while still queued" % job.id ) + else: + log.error( "(%d) Job in unknown state '%s'" % ( job.id, job_state ) ) + if not self.track_jobs_in_database: + new_waiting_jobs.append( job.id ) + except Exception: + log.exception( "failure running job %d" % job.id ) + # Update the waiting list + self.waiting_jobs = new_waiting_jobs + # Done with the session + self.sa_session.remove() + + def __check_if_ready_to_run( self, job ): + """ + Check if a job is ready to run by verifying that each of its input + datasets is ready (specifically in the OK state). If any input dataset + has an error, fail the job and return JOB_INPUT_ERROR. If any input + dataset is deleted, fail the job and return JOB_INPUT_DELETED. If all + input datasets are in OK state, return JOB_READY indicating that the + job can be dispatched. Otherwise, return JOB_WAIT indicating that input + datasets are still being prepared. + """ + if job.state == model.Job.states.DELETED: + return JOB_DELETED + elif job.state == model.Job.states.ERROR: + return JOB_ADMIN_DELETED + elif self.app.config.enable_quotas: + quota = self.app.quota_agent.get_quota( job.user ) + if quota is not None: + try: + usage = self.app.quota_agent.get_usage( user=job.user, history=job.history ) + if usage > quota: + return JOB_WAIT + except AssertionError, e: + pass # No history, should not happen with an anon user + for dataset_assoc in job.input_datasets + job.input_library_datasets: + idata = dataset_assoc.dataset + if not idata: + continue + # don't run jobs for which the input dataset was deleted + if idata.deleted: + JobWrapper( job, self ).fail( "input data %s (file: %s) was deleted before the job started" % ( idata.hid, idata.file_name ) ) + return JOB_INPUT_DELETED + # an error in the input data causes us to bail immediately + elif idata.state == idata.states.ERROR: + JobWrapper( job, self ).fail( "input data %s is in error state" % ( idata.hid ) ) + return JOB_INPUT_ERROR + elif idata.state == idata.states.FAILED_METADATA: + JobWrapper( job, self ).fail( "input data %s failed to properly set metadata" % ( idata.hid ) ) + return JOB_INPUT_ERROR + elif idata.state != idata.states.OK and not ( idata.state == idata.states.SETTING_METADATA and job.tool_id is not None and job.tool_id == self.app.datatypes_registry.set_external_metadata_tool.id ): + # need to requeue + return JOB_WAIT + return self.__check_user_jobs( job ) + + def __check_user_jobs( self, job ): + if not self.app.config.user_job_limit: + return JOB_READY + if job.user: + count = self.sa_session.query( model.Job ).enable_eagerloads( False ) \ + .filter( and_( model.Job.user_id == job.user.id, + or_( model.Job.state == model.Job.states.RUNNING, + model.Job.state == model.Job.states.QUEUED ) ) ).count() + elif job.galaxy_session: + count = self.sa_session.query( model.Job ).enable_eagerloads( False ) \ + .filter( and_( model.Job.session_id == job.galaxy_session.id, + or_( model.Job.state == model.Job.states.RUNNING, + model.Job.state == model.Job.states.QUEUED ) ) ).count() + else: + log.warning( 'Job %s is not associated with a user or session so job concurrency limit cannot be checked.' % job.id ) + return JOB_READY + if count >= self.app.config.user_job_limit: + return JOB_WAIT + return JOB_READY + + def put( self, job_id, tool_id ): + """Add a job to the queue (by job identifier)""" + if not self.track_jobs_in_database: + self.queue.put( ( job_id, tool_id ) ) + self.sleeper.wake() + + def shutdown( self ): + """Attempts to gracefully shut down the worker thread""" + if self.parent_pid != os.getpid(): + # We're not the real job queue, do nothing + return + else: + log.info( "sending stop signal to worker thread" ) + self.running = False + if not self.app.config.track_jobs_in_database: + self.queue.put( self.STOP_SIGNAL ) + self.sleeper.wake() + log.info( "job handler queue stopped" ) + self.dispatcher.shutdown() + +class JobHandlerStopQueue( object ): + """ + A queue for jobs which need to be terminated prematurely. + """ + STOP_SIGNAL = object() + def __init__( self, app, dispatcher ): + self.app = app + self.dispatcher = dispatcher + + self.sa_session = app.model.context + + # Keep track of the pid that started the job manager, only it + # has valid threads + self.parent_pid = os.getpid() + # Contains new jobs. Note this is not used if track_jobs_in_database is True + self.queue = Queue() + + # Contains jobs that are waiting (only use from monitor thread) + self.waiting = [] + + # Helper for interruptable sleep + self.sleeper = Sleeper() + self.running = True + self.monitor_thread = threading.Thread( target=self.monitor ) + self.monitor_thread.start() + log.info( "job handler stop queue started" ) + + def monitor( self ): + """ + Continually iterate the waiting jobs, stop any that are found. + """ + # HACK: Delay until after forking, we need a way to do post fork notification!!! + time.sleep( 10 ) + while self.running: + try: + self.monitor_step() + except: + log.exception( "Exception in monitor_step" ) + # Sleep + self.sleeper.sleep( 1 ) + + def monitor_step( self ): + """ + Called repeatedly by `monitor` to stop jobs. + """ + # Pull all new jobs from the queue at once + jobs_to_check = [] + if self.app.config.track_jobs_in_database: + # Clear the session so we get fresh states for job and all datasets + self.sa_session.expunge_all() + # Fetch all new jobs + newly_deleted_jobs = self.sa_session.query( model.Job ).enable_eagerloads( False ) \ + .filter( ( model.Job.state == model.Job.states.DELETED_NEW ) \ + & ( model.Job.handler == self.app.config.server_name ) ).all() + for job in newly_deleted_jobs: + jobs_to_check.append( ( job, None ) ) + # Also pull from the queue (in the case of Administrative stopped jobs) + try: + while 1: + message = self.queue.get_nowait() + if message is self.STOP_SIGNAL: + return + # Unpack the message + job_id, error_msg = message + # Get the job object and append to watch queue + jobs_to_check.append( ( self.sa_session.query( model.Job ).get( job_id ), error_msg ) ) + except Empty: + pass + for job, error_msg in jobs_to_check: + if error_msg is not None: + job.state = job.states.ERROR + job.info = error_msg + else: + job.state = job.states.DELETED + self.sa_session.add( job ) + self.sa_session.flush() + if job.job_runner_name is not None: + # tell the dispatcher to stop the job + self.dispatcher.stop( job ) + + def put( self, job_id, error_msg=None ): + self.queue.put( ( job_id, error_msg ) ) + + def shutdown( self ): + """Attempts to gracefully shut down the worker thread""" + if self.parent_pid != os.getpid(): + # We're not the real job queue, do nothing + return + else: + log.info( "sending stop signal to worker thread" ) + self.running = False + if not self.app.config.track_jobs_in_database: + self.queue.put( self.STOP_SIGNAL ) + self.sleeper.wake() + log.info( "job handler stop queue stopped" ) + +class DefaultJobDispatcher( object ): + def __init__( self, app ): + self.app = app + self.job_runners = {} + start_job_runners = ["local"] + if app.config.start_job_runners is not None: + start_job_runners.extend( [ x.strip() for x in util.listify( app.config.start_job_runners ) ] ) + if app.config.use_tasked_jobs: + start_job_runners.append("tasks") + for name in start_job_runners: + self._load_plugin( name ) + + def _load_plugin( self, name ): + module_name = 'galaxy.jobs.runners.' + name + try: + module = __import__( module_name ) + except: + log.exception( 'Job runner is not loadable: %s' % module_name ) + return + for comp in module_name.split( "." )[1:]: + module = getattr( module, comp ) + if '__all__' not in dir( module ): + log.error( 'Runner "%s" does not contain a list of exported classes in __all__' % module_name ) + return + for obj in module.__all__: + display_name = ':'.join( ( module_name, obj ) ) + runner = getattr( module, obj ) + self.job_runners[name] = runner( self.app ) + log.debug( 'Loaded job runner: %s' % display_name ) + + def __get_runner_name( self, job_wrapper ): + if self.app.config.use_tasked_jobs and job_wrapper.tool.parallelism is not None and not isinstance(job_wrapper, TaskWrapper): + runner_name = "tasks" + else: + runner_name = ( job_wrapper.get_job_runner().split(":", 1) )[0] + return runner_name + + def put( self, job_wrapper ): + try: + runner_name = self.__get_runner_name( job_wrapper ) + if self.app.config.use_tasked_jobs and job_wrapper.tool.parallelism is not None and isinstance(job_wrapper, TaskWrapper): + #DBTODO Refactor + log.debug( "dispatching task %s, of job %d, to %s runner" %( job_wrapper.task_id, job_wrapper.job_id, runner_name ) ) + else: + log.debug( "dispatching job %d to %s runner" %( job_wrapper.job_id, runner_name ) ) + self.job_runners[runner_name].put( job_wrapper ) + except KeyError: + log.error( 'put(): (%s) Invalid job runner: %s' % ( job_wrapper.job_id, runner_name ) ) + job_wrapper.fail( 'Unable to run job due to a misconfiguration of the Galaxy job running system. Please contact a site administrator.' ) + + def stop( self, job ): + runner_name = ( job.job_runner_name.split(":", 1) )[0] + log.debug( "stopping job %d in %s runner" %( job.id, runner_name ) ) + try: + self.job_runners[runner_name].stop_job( job ) + except KeyError: + log.error( 'stop(): (%s) Invalid job runner: %s' % ( job_wrapper.job_id, runner_name ) ) + # Job and output dataset states have already been updated, so nothing is done here. + + def recover( self, job, job_wrapper ): + runner_name = ( job.job_runner_name.split(":", 1) )[0] + log.debug( "recovering job %d in %s runner" %( job.id, runner_name ) ) + try: + self.job_runners[runner_name].recover( job, job_wrapper ) + except KeyError: + log.error( 'recover(): (%s) Invalid job runner: %s' % ( job_wrapper.job_id, runner_name ) ) + job_wrapper.fail( 'Unable to run job due to a misconfiguration of the Galaxy job running system. Please contact a site administrator.' ) + + def shutdown( self ): + for runner in self.job_runners.itervalues(): + runner.shutdown() diff --git a/lib/galaxy/jobs/manager.py b/lib/galaxy/jobs/manager.py new file mode 100644 index 00000000000..b5d0cdc0a9d --- /dev/null +++ b/lib/galaxy/jobs/manager.py @@ -0,0 +1,276 @@ +""" +Top-level Galaxy job manager, moves jobs to handler(s) +""" + +import os +import time +import random +import logging +import threading +from Queue import Queue, Empty + +from sqlalchemy.sql.expression import and_, or_ + +from galaxy import model +from galaxy.jobs import handler, Sleeper, NoopQueue +from galaxy.util.json import from_json_string + +log = logging.getLogger( __name__ ) + +class JobManager( object ): + """ + Highest level interface to job management. + + TODO: Currently the app accesses "job_queue" and "job_stop_queue" directly. + This should be decoupled. + """ + def __init__( self, app ): + self.app = app + self.job_handler = NoopHandler() + if self.app.config.server_name in self.app.config.job_handlers: + self.job_handler = handler.JobHandler( app ) + if self.app.config.server_name == self.app.config.job_manager: + job_handler = NoopHandler() + # In the case that webapp == manager == handler, pass jobs in memory + if not self.app.config.track_jobs_in_database: + job_handler = self.job_handler + # Otherwise, even if the manager == one of the handlers, its handler will pick up jobs from the database + self.job_queue = JobManagerQueue( app, job_handler ) + self.job_stop_queue = JobManagerStopQueue( app, job_handler ) + if self.app.config.enable_beta_job_managers: + from galaxy.jobs.deferred import DeferredJobQueue + self.deferred_job_queue = DeferredJobQueue( app ) + else: + self.job_queue = self.job_stop_queue = NoopQueue() + self.job_handler.start() + def shutdown( self ): + self.job_queue.shutdown() + self.job_stop_queue.shutdown() + self.job_handler.shutdown() + +class JobManagerQueue( object ): + """ + Job manager, waits for jobs to be runnable and then dispatches to a + JobHandler. + """ + STOP_SIGNAL = object() + def __init__( self, app, job_handler ): + self.app = app + self.job_handler = job_handler # the (singular) handler if we are passing jobs in memory + + self.sa_session = app.model.context + self.job_lock = False + # Keep track of the pid that started the job manager, only it + # has valid threads + self.parent_pid = os.getpid() + # Contains new jobs. Note this is not used if track_jobs_in_database is True + self.queue = Queue() + # Helper for interruptable sleep + self.sleeper = Sleeper() + self.running = True + self.monitor_thread = threading.Thread( target=self.__monitor ) + # Recover jobs at startup + self.__check_jobs_at_startup() + # Start the queue + self.monitor_thread.start() + log.info( "job manager queue started" ) + + def __check_jobs_at_startup( self ): + """ + Checks all jobs that are in the 'new', 'queued' or 'running' state in + the database and requeues or cleans up as necessary. Only run as the + job manager starts. + """ + for job in self.sa_session.query( model.Job ).enable_eagerloads( False ) \ + .filter( ( ( model.Job.state == model.Job.states.NEW ) \ + | ( model.Job.state == model.Job.states.RUNNING ) \ + | ( model.Job.state == model.Job.states.QUEUED ) ) \ + & ( model.Job.handler == None ) ): + if job.tool_id not in self.app.toolbox.tools_by_id: + log.warning( "(%s) Tool '%s' removed from tool config, unable to recover job" % ( job.id, job.tool_id ) ) + JobWrapper( job, self ).fail( 'This tool was disabled before the job completed. Please contact your Galaxy administrator.' ) + else: + job.handler = self.__get_handler( job ) # handler's recovery method will take it from here + log.info( "(%d) Job in '%s' state had no handler at job manager startup, assigned '%s' handler" % ( job.id, job.state, job.handler ) ) + if self.sa_session.dirty: + self.sa_session.flush() + + def __monitor( self ): + """ + Continually iterate the waiting jobs and dispatch to a handler + """ + # HACK: Delay until after forking, we need a way to do post fork notification!!! + time.sleep( 10 ) + while self.running: + try: + self.__monitor_step() + except: + log.exception( "Exception in monitor_step" ) + # Sleep + self.sleeper.sleep( 1 ) + + def __monitor_step( self ): + """ + Called repeatedly by `monitor` to process waiting jobs. Gets any new + jobs (either from the database or from its own queue), then assigns a + handler. + """ + # Do nothing if the queue is locked + if self.job_lock: + log.info( 'Job queue is administratively locked, sleeping...' ) + time.sleep( 10 ) + return + # Pull all new jobs from the queue at once + jobs_to_check = [] + if self.app.config.track_jobs_in_database: + # Clear the session so we get fresh states for job and all datasets + self.sa_session.expunge_all() + # Fetch all new jobs + jobs_to_check = self.sa_session.query( model.Job ).enable_eagerloads( False ) \ + .filter( ( model.Job.state == model.Job.states.NEW ) \ + & ( model.Job.handler == None ) ).all() + else: + # Get job objects and append to watch queue for any which were + # previously waiting + try: + while 1: + message = self.queue.get_nowait() + if message is self.STOP_SIGNAL: + return + # Unpack the message + job_id, tool_id = message + # Get the job object and append to watch queue + jobs_to_check.append( self.sa_session.query( model.Job ).get( job_id ) ) + except Empty: + pass + + for job in jobs_to_check: + job.handler = self.__get_handler( job ) + log.debug( "(%s) Job assigned to handler '%s'" % ( job.id, job.handler ) ) + self.sa_session.add( job ) + + # If tracking in the database, handlers will pick up the job now + self.sa_session.flush() + + time.sleep( 5 ) + + # This only does something in the case that there is only one handler and it is this Galaxy process + for job in jobs_to_check: + self.job_handler.job_queue.put( job.id, job.tool_id ) + + def __get_handler( self, job ): + try: + params = None + if job.params: + params = from_json_string( job.params ) + return self.app.toolbox.tools_by_id.get( job.tool_id, None ).get_job_handler( params ) + except: + log.exception( "(%s) Caught exception attempting to get tool-specific job handler for tool '%s', selecting at random from available handlers instead:" % ( job.id, job.tool_id ) ) + return random.choice( self.app.config.job_handlers ) + + def put( self, job_id, tool ): + """Add a job to the queue (by job identifier)""" + if not self.app.config.track_jobs_in_database: + self.queue.put( ( job_id, tool.id ) ) + self.sleeper.wake() + + def shutdown( self ): + """Attempts to gracefully shut down the worker thread""" + if self.parent_pid != os.getpid(): + # We're not the real job queue, do nothing + return + else: + log.info( "sending stop signal to worker thread" ) + self.running = False + if not self.app.config.track_jobs_in_database: + self.queue.put( self.STOP_SIGNAL ) + self.sleeper.wake() + log.info( "job manager queue stopped" ) + +class JobManagerStopQueue( object ): + """ + A queue for jobs which need to be terminated prematurely. + """ + STOP_SIGNAL = object() + def __init__( self, app, job_handler ): + self.app = app + self.job_handler = job_handler + + self.sa_session = app.model.context + + # Keep track of the pid that started the job manager, only it + # has valid threads + self.parent_pid = os.getpid() + # Contains new jobs. Note this is not used if track_jobs_in_database is True + self.queue = Queue() + + # Contains jobs that are waiting (only use from monitor thread) + self.waiting = [] + + # Helper for interruptable sleep + self.sleeper = Sleeper() + self.running = True + self.monitor_thread = threading.Thread( target=self.monitor ) + self.monitor_thread.start() + log.info( "job manager stop queue started" ) + + def monitor( self ): + """ + Continually iterate the waiting jobs, stop any that are found. + """ + # HACK: Delay until after forking, we need a way to do post fork notification!!! + time.sleep( 10 ) + while self.running: + try: + self.monitor_step() + except: + log.exception( "Exception in monitor_step" ) + # Sleep + self.sleeper.sleep( 1 ) + + def monitor_step( self ): + """ + Called repeatedly by `monitor` to stop jobs. + """ + jobs_to_check = [] + # Pull from the queue even if tracking in the database (in the case of Administrative stopped jobs) + try: + while 1: + message = self.queue.get_nowait() + if message is self.STOP_SIGNAL: + return + # Unpack the message + job_id, error_msg = message + # Get the job object and append to watch queue + jobs_to_check.append( ( self.sa_session.query( model.Job ).get( job_id ), error_msg ) ) + except Empty: + pass + + # If tracking in the database, the handler will pick up the stop itself. Otherwise, notify the handler. + for job, error_msg in jobs_to_check: + self.job_handler.job_stop_queue.put( job.id, error_msg ) + + def put( self, job_id, error_msg=None ): + self.queue.put( ( job_id, error_msg ) ) + + def shutdown( self ): + """Attempts to gracefully shut down the worker thread""" + if self.parent_pid != os.getpid(): + # We're not the real job queue, do nothing + return + else: + log.info( "sending stop signal to worker thread" ) + self.running = False + if not self.app.config.track_jobs_in_database: + self.queue.put( self.STOP_SIGNAL ) + self.sleeper.wake() + log.info( "job manager stop queue stopped" ) + +class NoopHandler( object ): + def __init__( self, *args, **kwargs ): + self.job_queue = NoopQueue() + self.job_stop_queue = NoopQueue() + def start( self ): + pass + def shutdown( self, *args ): + pass diff --git a/lib/galaxy/jobs/runners/tasks.py b/lib/galaxy/jobs/runners/tasks.py index 4a27b11f2ee..5acab43fc66 100644 --- a/lib/galaxy/jobs/runners/tasks.py +++ b/lib/galaxy/jobs/runners/tasks.py @@ -83,7 +83,7 @@ class TaskedJobRunner( object ): for task in tasks: tw = TaskWrapper(task, job_wrapper.queue) task_wrappers.append(tw) - self.app.job_manager.dispatcher.put(tw) + self.app.job_manager.job_handler.dispatcher.put(tw) tasks_incomplete = False count_complete = 0 sleep_time = 1 diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index a47778cb4e7..33eca7970be 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -132,6 +132,7 @@ class Job( object ): self.job_runner_external_id = None self.post_job_actions = [] self.imported = False + self.handler = None def add_parameter( self, name, value ): self.parameters.append( JobParameter( name, value ) ) @@ -171,14 +172,11 @@ class Job( object ): if not dataset.deleted: return False return True - def mark_deleted( self, enable_job_running=True, track_jobs_in_database=False ): + def mark_deleted( self, track_jobs_in_database=False ): """ Mark this job as deleted, and mark any output datasets as discarded. """ - # This could be handled with *just* track_jobs_in_database, but I - # didn't want to make setting track_jobs_in_database required in - # non-runner configs. - if not enable_job_running or track_jobs_in_database: + if track_jobs_in_database: self.state = Job.states.DELETED_NEW else: self.state = Job.states.DELETED @@ -303,6 +301,20 @@ class JobImportHistoryArchive( object ): self.history = history self.archive_dir=archive_dir +class GenomeIndexToolData( object ): + def __init__( self, job=None, params=None, dataset=None, deferred_job=None, \ + transfer_job=None, fasta_path=None, created_time=None, modified_time=None, \ + dbkey=None, user=None, indexer=None ): + self.job = job + self.dataset = dataset + self.fasta_path = fasta_path + self.user = user + self.indexer = indexer + self.created_time = created_time + self.modified_time = modified_time + self.deferred = deferred_job + self.transfer = transfer_job + class DeferredJob( object ): states = Bunch( NEW = 'new', WAITING = 'waiting', @@ -1128,8 +1140,10 @@ class HistoryDatasetAssociation( DatasetInstance ): def clear_associated_files( self, metadata_safe = False, purge = False ): # metadata_safe = True means to only clear when assoc.metadata_safe == False for assoc in self.implicitly_converted_datasets: - if not metadata_safe or not assoc.metadata_safe: + if not assoc.deleted and ( not metadata_safe or not assoc.metadata_safe ): assoc.clear( purge = purge ) + for assoc in self.implicitly_converted_parent_datasets: + assoc.clear( purge = purge, delete_dataset = False ) def get_display_name( self ): ## Name can be either a string or a unicode object. If string, convert to unicode object assuming 'utf-8' format. hda_name = self.name @@ -1188,6 +1202,12 @@ class HistoryDatasetAssociationDisplayAtAuthorization( object ): self.history_dataset_association = hda self.user = user self.site = site + +class HistoryDatasetAssociationSubset( object ): + def __init__(self, hda, subset, location): + self.hda = hda + self.subset = subset + self.location = location class Library( object, APIItem ): permitted_actions = get_permitted_actions( filter='LIBRARY' ) @@ -1536,21 +1556,26 @@ class LibraryDatasetDatasetAssociation( DatasetInstance ): else: return template.get_widgets( trans.user ) return [] - def templates_dict( self ): + def templates_dict( self, use_name=False ): """ Returns a dict of template info """ + #TODO: Should have a method that allows names and labels to be returned together in a structured way template_data = {} for temp_info in self.info_association: template = temp_info.template content = temp_info.info.content tmp_dict = {} for field in template.fields: - tmp_dict[field['label']] = content[field['name']] + if use_name: + name = field[ 'name' ] + else: + name = field[ 'label' ] + tmp_dict[ name ] = content.get( field[ 'name' ] ) template_data[template.name] = tmp_dict return template_data - def templates_json( self ): - return simplejson.dumps( self.templates_dict() ) + def templates_json( self, use_name=False ): + return simplejson.dumps( self.templates_dict( use_name=use_name ) ) def get_display_name( self ): """ @@ -1617,12 +1642,14 @@ class ImplicitlyConvertedDatasetAssociation( object ): self.purged = purged self.metadata_safe = metadata_safe - def clear( self, purge = False ): + def clear( self, purge = False, delete_dataset = True ): self.deleted = True if self.dataset: - self.dataset.deleted = True - self.dataset.purged = purge - if purge: #do something with purging + if delete_dataset: + self.dataset.deleted = True + if purge: + self.dataset.purged = True + if purge and self.dataset.deleted: #do something with purging self.purged = True try: os.unlink( self.file_name ) except Exception, e: print "Failed to purge associated file (%s) from disk: %s" % ( self.file_name, e ) @@ -2620,8 +2647,8 @@ class APIKeys( object ): class ToolShedRepository( object ): def __init__( self, id=None, create_time=None, tool_shed=None, name=None, description=None, owner=None, installed_changeset_revision=None, - changeset_revision=None, metadata=None, includes_datatypes=False, update_available=False, deleted=False, uninstalled=False, - dist_to_shed=False ): + changeset_revision=None, ctx_rev=None, metadata=None, includes_datatypes=False, update_available=False, deleted=False, + uninstalled=False, dist_to_shed=False ): self.id = id self.create_time = create_time self.tool_shed = tool_shed @@ -2630,6 +2657,7 @@ class ToolShedRepository( object ): self.owner = owner self.installed_changeset_revision = installed_changeset_revision self.changeset_revision = changeset_revision + self.ctx_rev = ctx_rev self.metadata = metadata self.includes_datatypes = includes_datatypes self.update_available = update_available diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index e8fbcf36886..1f347d4cbf0 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -76,6 +76,7 @@ UserOpenID.table = Table( "galaxy_user_openid", metadata, Column( "session_id", Integer, ForeignKey( "galaxy_session.id" ), index=True ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), Column( "openid", TEXT, index=True, unique=True ), + Column( "provider", TrimmedString( 255 ) ), ) History.table = Table( "history", metadata, @@ -143,6 +144,12 @@ HistoryDatasetAssociationDisplayAtAuthorization.table = Table( "history_dataset_ Column( "history_dataset_association_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), Column( "site", TrimmedString( 255 ) ) ) + +HistoryDatasetAssociationSubset.table = Table( "history_dataset_association_subset", metadata, + Column( "id", Integer, primary_key=True ), + Column( "history_dataset_association_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ), + Column( "history_dataset_association_subset_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ), + Column( "location", Unicode(255), index=True) ) ImplicitlyConvertedDatasetAssociation.table = Table( "implicitly_converted_dataset_association", metadata, Column( "id", Integer, primary_key=True ), @@ -375,6 +382,7 @@ ToolShedRepository.table = Table( "tool_shed_repository", metadata, Column( "owner", TrimmedString( 255 ), index=True ), Column( "installed_changeset_revision", TrimmedString( 255 ) ), Column( "changeset_revision", TrimmedString( 255 ), index=True ), + Column( "ctx_rev", TrimmedString( 10 ) ), Column( "metadata", JSONType, nullable=True ), Column( "includes_datatypes", Boolean, index=True, default=False ), Column( "update_available", Boolean, default=False ), @@ -421,7 +429,8 @@ Job.table = Table( "job", metadata, Column( "job_runner_external_id", String( 255 ) ), Column( "object_store_id", TrimmedString( 255 ), index=True ), Column( "imported", Boolean, default=False, index=True ), - Column( "params", TrimmedString(255), index=True ) ) + Column( "params", TrimmedString(255), index=True ), + Column( "handler", TrimmedString( 255 ), index=True ) ) JobParameter.table = Table( "job_parameter", metadata, Column( "id", Integer, primary_key=True ), @@ -483,6 +492,19 @@ JobImportHistoryArchive.table = Table( "job_import_history_archive", metadata, Column( "archive_dir", TEXT ) ) +GenomeIndexToolData.table = Table( "genome_index_tool_data", metadata, + Column( "id", Integer, primary_key=True ), + Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ), + Column( "deferred_job_id", Integer, ForeignKey( "deferred_job.id" ), index=True ), + Column( "transfer_job_id", Integer, ForeignKey( "transfer_job.id" ), index=True ), + Column( "dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ), + Column( "fasta_path", String( 255 ) ), + Column( "created_time", DateTime, default=now ), + Column( "modified_time", DateTime, default=now, onupdate=now ), + Column( "indexer", String( 64 ) ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), + ) + Task.table = Table( "task", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), @@ -1196,6 +1218,9 @@ assign_mapper( context, HistoryDatasetAssociation, HistoryDatasetAssociation.tab implicitly_converted_datasets=relation( ImplicitlyConvertedDatasetAssociation, primaryjoin=( ImplicitlyConvertedDatasetAssociation.table.c.hda_parent_id == HistoryDatasetAssociation.table.c.id ) ), + implicitly_converted_parent_datasets=relation( + ImplicitlyConvertedDatasetAssociation, + primaryjoin=( ImplicitlyConvertedDatasetAssociation.table.c.hda_id == HistoryDatasetAssociation.table.c.id ) ), children=relation( HistoryDatasetAssociation, primaryjoin=( HistoryDatasetAssociation.table.c.parent_id == HistoryDatasetAssociation.table.c.id ), @@ -1231,6 +1256,13 @@ assign_mapper( context, Dataset, Dataset.table, assign_mapper( context, HistoryDatasetAssociationDisplayAtAuthorization, HistoryDatasetAssociationDisplayAtAuthorization.table, properties=dict( history_dataset_association = relation( HistoryDatasetAssociation ), user = relation( User ) ) ) + +assign_mapper( context, HistoryDatasetAssociationSubset, HistoryDatasetAssociationSubset.table, + properties=dict( hda = relation( HistoryDatasetAssociation, + primaryjoin=( HistoryDatasetAssociationSubset.table.c.history_dataset_association_id == HistoryDatasetAssociation.table.c.id ) ), + subset = relation( HistoryDatasetAssociation, + primaryjoin=( HistoryDatasetAssociationSubset.table.c.history_dataset_association_subset_id == HistoryDatasetAssociation.table.c.id ) ) + ) ) assign_mapper( context, ImplicitlyConvertedDatasetAssociation, ImplicitlyConvertedDatasetAssociation.table, properties=dict( parent_hda=relation( @@ -1493,6 +1525,13 @@ assign_mapper( context, JobExportHistoryArchive, JobExportHistoryArchive.table, assign_mapper( context, JobImportHistoryArchive, JobImportHistoryArchive.table, properties=dict( job = relation( Job ), history = relation( History ) ) ) +assign_mapper( context, GenomeIndexToolData, GenomeIndexToolData.table, + properties=dict( job = relation( Job ), + dataset = relation( Dataset ), + user = relation( User ), + deferred = relation( DeferredJob, backref='deferred_job' ), + transfer = relation( TransferJob, backref='transfer_job' ) ) ) + assign_mapper( context, PostJobAction, PostJobAction.table, properties=dict(workflow_step = relation( WorkflowStep, backref='post_job_actions', primaryjoin=(WorkflowStep.table.c.id == PostJobAction.table.c.workflow_step_id)))) diff --git a/lib/galaxy/model/migrate/versions/0094_add_job_handler_col.py b/lib/galaxy/model/migrate/versions/0094_add_job_handler_col.py new file mode 100644 index 00000000000..ffdcc872c55 --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0094_add_job_handler_col.py @@ -0,0 +1,49 @@ +""" +Migration script to create "handler" column in job table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +import logging +log = logging.getLogger( __name__ ) + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +# Column to add. +handler_col = Column( "handler", TrimmedString(255), index=True ) + +def display_migration_details(): + print "" + print "This migration script adds a 'handler' column to the Job table." + +def upgrade(): + print __doc__ + metadata.reflect() + + # Add column to Job table. + try: + Job_table = Table( "job", metadata, autoload=True ) + handler_col.create( Job_table ) + assert handler_col is Job_table.c.handler + + except Exception, e: + print str(e) + log.debug( "Adding column 'handler' to job table failed: %s" % str( e ) ) + +def downgrade(): + metadata.reflect() + + # Drop column from Job table. + try: + Job_table = Table( "job", metadata, autoload=True ) + handler_col = Job_table.c.handler + handler_col.drop() + except Exception, e: + log.debug( "Dropping column 'handler' from job table failed: %s" % ( str( e ) ) ) diff --git a/lib/galaxy/model/migrate/versions/0095_hda_subsets.py b/lib/galaxy/model/migrate/versions/0095_hda_subsets.py new file mode 100644 index 00000000000..f7b7473ad6f --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0095_hda_subsets.py @@ -0,0 +1,44 @@ +""" +Migration script to create table for tracking history_dataset_association subsets. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +import logging +log = logging.getLogger( __name__ ) + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +# Table to add. + +HistoryDatasetAssociationSubset_table = Table( "history_dataset_association_subset", metadata, + Column( "id", Integer, primary_key=True ), + Column( "history_dataset_association_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ), + Column( "history_dataset_association_subset_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ), + Column( "location", Unicode(255), index=True) +) + +def upgrade(): + print __doc__ + metadata.reflect() + + # Create history_dataset_association_subset. + try: + HistoryDatasetAssociationSubset_table.create() + except Exception, e: + print str(e) + log.debug( "Creating history_dataset_association_subset table failed: %s" % str( e ) ) + +def downgrade(): + metadata.reflect() + + # Drop history_dataset_association_subset table. + try: + HistoryDatasetAssociationSubset_table.drop() + except Exception, e: + print str(e) + log.debug( "Dropping history_dataset_association_subset table failed: %s" % str( e ) ) \ No newline at end of file diff --git a/lib/galaxy/model/migrate/versions/0096_openid_provider.py b/lib/galaxy/model/migrate/versions/0096_openid_provider.py new file mode 100644 index 00000000000..b839fef1635 --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0096_openid_provider.py @@ -0,0 +1,45 @@ +""" +Migration script to add column to openid table for provider. +Remove any OpenID entries with nonunique GenomeSpace Identifier +""" + +BAD_IDENTIFIER = 'https://identity.genomespace.org/identityServer/xrd.jsp' +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * +from galaxy.model.custom_types import TrimmedString + +import logging +log = logging.getLogger( __name__ ) + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +def upgrade(): + print __doc__ + metadata.reflect() + + try: + OpenID_table = Table( "galaxy_user_openid", metadata, autoload=True ) + c = Column( "provider", TrimmedString( 255 ) ) + c.create( OpenID_table ) + assert c is OpenID_table.c.provider + except Exception, e: + print "Adding provider column to galaxy_user_openid table failed: %s" % str( e ) + log.debug( "Adding provider column to galaxy_user_openid table failed: %s" % str( e ) ) + + try: + cmd = "DELETE FROM galaxy_user_openid WHERE openid='%s'" % ( BAD_IDENTIFIER ) + db_session.execute( cmd ) + except Exception, e: + log.debug( "Deleting bad Identifiers from galaxy_user_openid failed: %s" % str( e ) ) + +def downgrade(): + metadata.reflect() + try: + OpenID_table = Table( "galaxy_user_openid", metadata, autoload=True ) + OpenID_table.c.provider.drop() + except Exception, e: + print "Dropping provider column from galaxy_user_openid table failed: %s" % str( e ) + log.debug( "Dropping provider column from galaxy_user_openid table failed: %s" % str( e ) ) \ No newline at end of file diff --git a/lib/galaxy/model/migrate/versions/0097_add_ctx_rev_column.py b/lib/galaxy/model/migrate/versions/0097_add_ctx_rev_column.py new file mode 100644 index 00000000000..68f6f1f1bc2 --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0097_add_ctx_rev_column.py @@ -0,0 +1,43 @@ +""" +Migration script to add the ctx_rev column to the tool_shed_repository table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +import datetime +now = datetime.datetime.utcnow +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +def upgrade(): + print __doc__ + metadata.reflect() + ToolShedRepository_table = Table( "tool_shed_repository", metadata, autoload=True ) + col = Column( "ctx_rev", TrimmedString( 10 ) ) + try: + col.create( ToolShedRepository_table ) + assert col is ToolShedRepository_table.c.ctx_rev + except Exception, e: + print "Adding ctx_rev column to the tool_shed_repository table failed: %s" % str( e ) +def downgrade(): + metadata.reflect() + ToolShedRepository_table = Table( "tool_shed_repository", metadata, autoload=True ) + try: + ToolShedRepository_table.c.ctx_rev.drop() + except Exception, e: + print "Dropping column ctx_rev from the tool_shed_repository table failed: %s" % str( e ) diff --git a/lib/galaxy/model/migrate/versions/0098_genome_index_tool_data_table.py b/lib/galaxy/model/migrate/versions/0098_genome_index_tool_data_table.py new file mode 100644 index 00000000000..5a57e237b65 --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0098_genome_index_tool_data_table.py @@ -0,0 +1,53 @@ +""" +Migration script to create the genome_index_tool_data table. +""" + +from sqlalchemy import * +from migrate import * + +import datetime +now = datetime.datetime.utcnow + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData( migrate_engine ) + +# New table in changeset TODO:TODO +GenomeIndexToolData_table = Table( "genome_index_tool_data", metadata, + Column( "id", Integer, primary_key=True ), + Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ), + Column( "dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ), + Column( "deferred_job_id", Integer, ForeignKey( "deferred_job.id" ), index=True ), + Column( "transfer_job_id", Integer, ForeignKey( "transfer_job.id" ), index=True ), + Column( "fasta_path", String( 255 ) ), + Column( "created_time", DateTime, default=now ), + Column( "modified_time", DateTime, default=now, onupdate=now ), + Column( "indexer", String( 64 ) ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), + ) + +def upgrade(): + print __doc__ + + metadata.reflect() + try: + GenomeIndexToolData_table.create() + except Exception, e: + log.debug( "Creating genome_index_tool_data table failed: %s" % str( e ) ) + +def downgrade(): + metadata.reflect() + try: + GenomeIndexToolData_table.drop() + except Exception, e: + log.debug( "Dropping genome_index_tool_data table failed: %s" % str( e ) ) diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index 748c74257c9..b2eb4334193 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -223,7 +223,7 @@ class DiskObjectStore(ObjectStore): if not os.path.exists(path): return self._construct_path(obj, base_dir=base_dir, dir_only=dir_only, extra_dir=extra_dir, extra_dir_at_root=extra_dir_at_root, alt_name=alt_name) - def _construct_path(self, obj, old_style=False, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None): + def _construct_path(self, obj, old_style=False, base_dir=None, dir_only=False, extra_dir=None, extra_dir_at_root=False, alt_name=None, **kwargs): """ Construct the expected absolute path for accessing the object identified by `obj`.id. @@ -963,6 +963,9 @@ class DistributedObjectStore(ObjectStore): return self.__call_method('get_filename', obj, ObjectNotFound, True, **kwargs) def update_from_file(self, obj, **kwargs): + if kwargs.get('create', False): + self.create(obj, **kwargs) + kwargs['create'] = False return self.__call_method('update_from_file', obj, ObjectNotFound, True, **kwargs) def get_object_url(self, obj, **kwargs): diff --git a/lib/galaxy/openid/providers.py b/lib/galaxy/openid/providers.py index b31f0073a10..7e14eafb366 100644 --- a/lib/galaxy/openid/providers.py +++ b/lib/galaxy/openid/providers.py @@ -9,6 +9,9 @@ from galaxy.util.odict import odict log = logging.getLogger( __name__ ) +NO_PROVIDER_ID = 'None' +RESERVED_PROVIDER_IDS = [ NO_PROVIDER_ID ] + class OpenIDProvider( object ): '''An OpenID Provider object.''' @classmethod @@ -22,7 +25,9 @@ class OpenIDProvider( object ): op_endpoint_url = provider_elem.find( 'op_endpoint_url' ) if op_endpoint_url is not None: op_endpoint_url = op_endpoint_url.text + never_associate_with_user = string_as_bool( provider_elem.get( 'never_associate_with_user', 'False' ) ) assert (provider_id and provider_name and op_endpoint_url), Exception( "OpenID Provider improperly configured" ) + assert provider_id not in RESERVED_PROVIDER_IDS, Exception( 'Specified OpenID Provider uses a reserved id: %s' % ( provider_id ) ) sreg_required = [] sreg_optional = [] use_for = {} @@ -45,8 +50,8 @@ class OpenIDProvider( object ): sreg_required = None sreg_optional = None use_for = None - return cls( provider_id, provider_name, op_endpoint_url, sreg_required, sreg_optional, use_for, store_user_preference ) - def __init__( self, id, name, op_endpoint_url, sreg_required=None, sreg_optional=None, use_for=None, store_user_preference=None ): + return cls( provider_id, provider_name, op_endpoint_url, sreg_required=sreg_required, sreg_optional=sreg_optional, use_for=use_for, store_user_preference=store_user_preference, never_associate_with_user=never_associate_with_user ) + def __init__( self, id, name, op_endpoint_url, sreg_required=None, sreg_optional=None, use_for=None, store_user_preference=None, never_associate_with_user=None ): '''When sreg options are not specified, defaults are used.''' self.id = id self.name = name @@ -71,6 +76,10 @@ class OpenIDProvider( object ): self.store_user_preference = store_user_preference else: self.store_user_preference = {} + if never_associate_with_user: + self.never_associate_with_user = True + else: + self.never_associate_with_user = False def post_authentication( self, trans, openid_manager, info ): sreg_attributes = openid_manager.get_sreg( info ) for store_pref_name, store_pref_value_name in self.store_user_preference.iteritems(): @@ -80,9 +89,12 @@ class OpenIDProvider( object ): raise Exception( 'Only sreg is currently supported.' ) trans.sa_session.add( trans.user ) trans.sa_session.flush() + def has_post_authentication_actions( self ): + return bool( self.store_user_preference ) class OpenIDProviders( object ): '''Collection of OpenID Providers''' + NO_PROVIDER_ID = NO_PROVIDER_ID @classmethod def from_file( cls, filename ): try: @@ -107,6 +119,7 @@ class OpenIDProviders( object ): self.providers = providers else: self.providers = odict() + self._banned_identifiers = [ provider.op_endpoint_url for provider in self.providers.itervalues() if provider.never_associate_with_user ] def __iter__( self ): for provider in self.providers.itervalues(): yield provider @@ -115,3 +128,5 @@ class OpenIDProviders( object ): return self.providers[ name ] else: return default + def new_provider_from_identifier( self, identifier ): + return OpenIDProvider( None, identifier, identifier, never_associate_with_user = identifier in self._banned_identifiers ) diff --git a/lib/galaxy/security/validate_user_input.py b/lib/galaxy/security/validate_user_input.py index 47489dcc8e2..0d3b68921ac 100644 --- a/lib/galaxy/security/validate_user_input.py +++ b/lib/galaxy/security/validate_user_input.py @@ -1,6 +1,8 @@ import re VALID_PUBLICNAME_RE = re.compile( "^[a-z0-9\-]+$" ) +VALID_PUBLICNAME_SUB = re.compile( "[^a-z0-9\-]" ) +FILL_CHAR = '-' def validate_email( trans, email, user=None, check_dup=True ): message = '' @@ -31,6 +33,20 @@ def validate_publicname( trans, publicname, user=None ): return "Public name is taken; please choose another" return '' +def transform_publicname( trans, publicname, user=None ): + # User names must be at least four characters in length and contain only lower-case + # letters, numbers, and the '-' character. + #TODO: Enhance to allow generation of semi-random publicnnames e.g., when valid but taken + if user and user.username == publicname: + return publicname + elif publicname not in [ 'None', None, '' ]: + publicname = publicname.lower() + publicname = re.sub( VALID_PUBLICNAME_SUB, FILL_CHAR, publicname ) + publicname = publicname.ljust( 4, FILL_CHAR )[:255] + if not trans.sa_session.query( trans.app.model.User ).filter_by( username=publicname ).first(): + return publicname + return '' + def validate_password( trans, password, confirm ): if len( password ) < 6: return "Use a password of at least 6 characters" diff --git a/lib/galaxy/tool_shed/install_manager.py b/lib/galaxy/tool_shed/install_manager.py index 44bfa6d2cec..c3fe0916358 100644 --- a/lib/galaxy/tool_shed/install_manager.py +++ b/lib/galaxy/tool_shed/install_manager.py @@ -7,6 +7,7 @@ from galaxy.tools import ToolSection from galaxy.util.json import from_json_string, to_json_string from galaxy.util.shed_util import * from galaxy.util.odict import odict + log = logging.getLogger( __name__ ) class InstallManager( object ): @@ -119,7 +120,7 @@ class InstallManager( object ): is_displayed = True return is_displayed, tool_sections def handle_repository_contents( self, current_working_dir, repository_clone_url, relative_install_dir, repository_elem, repository_name, description, - changeset_revision, tmp_name ): + changeset_revision, ctx_rev ): # Generate the metadata for the installed tool shed repository, among other things. It is critical that the installed repository is # updated to the desired changeset_revision before metadata is set because the process for setting metadata uses the repository files on disk. # The values for the keys in each of the following dictionaries will be a list to allow for the same tool to be displayed in multiple places @@ -144,6 +145,7 @@ class InstallManager( object ): repository_name, description, changeset_revision, + ctx_rev, repository_clone_url, metadata_dict, dist_to_shed=True ) @@ -154,9 +156,15 @@ class InstallManager( object ): # Handle missing data table entries for tool parameters that are dynamically generated select lists. repository_tools_tups = handle_missing_data_table_entry( self.app, self.tool_path, sample_files, repository_tools_tups ) # Handle missing index files for tool parameters that are dynamically generated select lists. - repository_tools_tups = handle_missing_index_file( self.app, self.tool_path, sample_files, repository_tools_tups ) - # Handle tools that use fabric scripts to install dependencies. - handle_tool_dependencies( current_working_dir, relative_install_dir, repository_tools_tups ) + repository_tools_tups, sample_files_copied = handle_missing_index_file( self.app, self.tool_path, sample_files, repository_tools_tups ) + # Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance. + copy_sample_files( self.app, sample_files, sample_files_copied=sample_files_copied ) + if 'tool_dependencies_config' in metadata_dict: + # Install tool dependencies. + status, message = handle_tool_dependencies( self.app, repository_clone_url, metadata_dict[ 'tool_dependencies_config' ] ) + if status != 'ok' and message: + print 'The following error occurred while installing tool dependencies:' + print message add_to_tool_panel( self.app, repository_name, repository_clone_url, @@ -166,11 +174,6 @@ class InstallManager( object ): self.migrated_tools_config, tool_panel_dict=tool_panel_dict_for_display, new_install=True ) - # Remove the temporary file - try: - os.unlink( tmp_name ) - except: - pass if 'datatypes_config' in metadata_dict: datatypes_config = os.path.abspath( metadata_dict[ 'datatypes_config' ] ) # Load proprietary data types required by tools. The value of override is not important here since the Galaxy server will be started @@ -193,7 +196,7 @@ class InstallManager( object ): self.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) return tool_shed_repository, metadata_dict def install_repository( self, repository_elem ): - # Install a single repository, loading contained tools into the tool config. + # Install a single repository, loading contained tools into the tool panel. name = repository_elem.get( 'name' ) description = repository_elem.get( 'description' ) changeset_revision = repository_elem.get( 'changeset_revision' ) @@ -206,65 +209,55 @@ class InstallManager( object ): tool_shed_url = self.__get_url_from_tool_shed( self.tool_shed ) repository_clone_url = os.path.join( tool_shed_url, 'repos', self.repository_owner, name ) relative_install_dir = os.path.join( clone_dir, name ) - returncode, tmp_name = clone_repository( name, clone_dir, current_working_dir, repository_clone_url ) - if returncode == 0: - returncode, tmp_name = update_repository( current_working_dir, relative_install_dir, changeset_revision ) - if returncode == 0: - tool_shed_repository, metadata_dict = self.handle_repository_contents( current_working_dir, - repository_clone_url, - relative_install_dir, - repository_elem, - name, - description, - changeset_revision, - tmp_name ) - if 'tools' in metadata_dict: - # Get the tool_versions from the tool shed for each tool in the installed change set. - url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ - ( tool_shed_url, tool_shed_repository.name, self.repository_owner, changeset_revision ) - response = urllib2.urlopen( url ) - text = response.read() - response.close() - if text: - tool_version_dicts = from_json_string( text ) - handle_tool_versions( self.app, tool_version_dicts, tool_shed_repository ) - else: - # Set the tool versions since they seem to be missing for this repository in the tool shed. - # CRITICAL NOTE: These default settings may not properly handle all parent/child associations. - for tool_dict in metadata_dict[ 'tools' ]: - flush_needed = False - tool_id = tool_dict[ 'guid' ] - old_tool_id = tool_dict[ 'id' ] - tool_version = tool_dict[ 'version' ] - tool_version_using_old_id = get_tool_version( self.app, old_tool_id ) - tool_version_using_guid = get_tool_version( self.app, tool_id ) - if not tool_version_using_old_id: - tool_version_using_old_id = self.app.model.ToolVersion( tool_id=old_tool_id, - tool_shed_repository=tool_shed_repository ) - self.app.sa_session.add( tool_version_using_old_id ) - self.app.sa_session.flush() - if not tool_version_using_guid: - tool_version_using_guid = self.app.model.ToolVersion( tool_id=tool_id, - tool_shed_repository=tool_shed_repository ) - self.app.sa_session.add( tool_version_using_guid ) - self.app.sa_session.flush() - # Associate the two versions as parent / child. - tool_version_association = get_tool_version_association( self.app, - tool_version_using_old_id, - tool_version_using_guid ) - if not tool_version_association: - tool_version_association = self.app.model.ToolVersionAssociation( tool_id=tool_version_using_guid.id, - parent_id=tool_version_using_old_id.id ) - self.app.sa_session.add( tool_version_association ) - self.app.sa_session.flush() + ctx_rev = get_ctx_rev( tool_shed_url, name, self.repository_owner, changeset_revision ) + clone_repository( repository_clone_url, os.path.abspath( relative_install_dir ), ctx_rev ) + tool_shed_repository, metadata_dict = self.handle_repository_contents( current_working_dir, + repository_clone_url, + relative_install_dir, + repository_elem, + name, + description, + changeset_revision, + ctx_rev ) + if 'tools' in metadata_dict: + # Get the tool_versions from the tool shed for each tool in the installed change set. + url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy&no_reset=true' % \ + ( tool_shed_url, tool_shed_repository.name, self.repository_owner, changeset_revision ) + response = urllib2.urlopen( url ) + text = response.read() + response.close() + if text: + tool_version_dicts = from_json_string( text ) + handle_tool_versions( self.app, tool_version_dicts, tool_shed_repository ) else: - tmp_stderr = open( tmp_name, 'rb' ) - print "Error updating repository ', name, "': ', str( tmp_stderr.read() ) - tmp_stderr.close() - else: - tmp_stderr = open( tmp_name, 'rb' ) - print "Error cloning repository '", name, "': ", str( tmp_stderr.read() ) - tmp_stderr.close() + # Set the tool versions since they seem to be missing for this repository in the tool shed. + # CRITICAL NOTE: These default settings may not properly handle all parent/child associations. + for tool_dict in metadata_dict[ 'tools' ]: + flush_needed = False + tool_id = tool_dict[ 'guid' ] + old_tool_id = tool_dict[ 'id' ] + tool_version = tool_dict[ 'version' ] + tool_version_using_old_id = get_tool_version( self.app, old_tool_id ) + tool_version_using_guid = get_tool_version( self.app, tool_id ) + if not tool_version_using_old_id: + tool_version_using_old_id = self.app.model.ToolVersion( tool_id=old_tool_id, + tool_shed_repository=tool_shed_repository ) + self.app.sa_session.add( tool_version_using_old_id ) + self.app.sa_session.flush() + if not tool_version_using_guid: + tool_version_using_guid = self.app.model.ToolVersion( tool_id=tool_id, + tool_shed_repository=tool_shed_repository ) + self.app.sa_session.add( tool_version_using_guid ) + self.app.sa_session.flush() + # Associate the two versions as parent / child. + tool_version_association = get_tool_version_association( self.app, + tool_version_using_old_id, + tool_version_using_guid ) + if not tool_version_association: + tool_version_association = self.app.model.ToolVersionAssociation( tool_id=tool_version_using_guid.id, + parent_id=tool_version_using_old_id.id ) + self.app.sa_session.add( tool_version_association ) + self.app.sa_session.flush() @property def non_shed_tool_panel_configs( self ): # Get the non-shed related tool panel config file names from the Galaxy config - the default is tool_conf.xml. @@ -276,7 +269,6 @@ class InstallManager( object ): root = tree.getroot() tool_path = root.get( 'tool_path', None ) if tool_path is None: - # There will be a problem here if the user has defined 2 non-shed related configs. config_filenames.append( config_filename ) return config_filenames def __get_url_from_tool_shed( self, tool_shed ): diff --git a/lib/galaxy/tool_shed/migrate/check.py b/lib/galaxy/tool_shed/migrate/check.py index bded6b310b8..2f883570787 100644 --- a/lib/galaxy/tool_shed/migrate/check.py +++ b/lib/galaxy/tool_shed/migrate/check.py @@ -54,51 +54,52 @@ def verify_tools( app, url, galaxy_config_file, engine_options={} ): config_arg = '' if os.path.abspath( os.path.join( os.getcwd(), 'universe_wsgi.ini' ) ) != galaxy_config_file: config_arg = ' -c %s' % galaxy_config_file.replace( os.path.abspath( os.getcwd() ), '.' ) - # Automatically update the value of the migrate_tools.version database table column. - cmd = 'sh manage_tools.sh%s upgrade' % config_arg - proc = subprocess.Popen( args=cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) - return_code = proc.wait() - output = proc.stdout.read( 32768 ) - if return_code != 0: - raise Exception( "Error attempting to update the value of migrate_tools.version: %s" % output ) - elif missing_tool_configs: - if len( tool_panel_configs ) == 1: - plural = '' - tool_panel_config_file_names = tool_panel_configs[ 0 ] - else: - plural = 's' - tool_panel_config_file_names = ', '.join( tool_panel_configs ) - msg = "\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" - msg += "\n\nThe list of files at the end of this message refers to tools that are configured to load into the tool panel for\n" - msg += "this Galaxy instance, but have been removed from the Galaxy distribution. These tools can be automatically installed\n" - msg += "from the Galaxy tool shed at http://toolshed.g2.bx.psu.edu.\n\n" - msg += "To skip this process, attempt to start your Galaxy server again (e.g., sh run.sh or whatever you use). If you do this,\n" - msg += "be aware that these tools will no longer be available in your Galaxy tool panel, and entries for each of them should\n" - msg += "be removed from your file%s named %s.\n\n" % ( plural, tool_panel_config_file_names ) - msg += "CRITICAL NOTE IF YOU PLAN TO INSTALL\n" - msg += "The location in which the tool repositories will be installed is the value of the 'tool_path' attribute in the \n" - msg += 'tag of the file named ./migrated_tool_conf.xml (i.e., ). The default location\n' - msg += "setting is '../shed_tools', which may be problematic for some cluster environments, so make sure to change it before\n" - msg += "you execute the installation process if appropriate. The configured location must be outside of the Galaxy installation\n" - msg += "directory or it must be in a sub-directory protected by a properly configured .hgignore file if the directory is within\n" - msg += "the Galaxy installation directory hierarchy. This is because tool shed repositories will be installed using mercurial's\n" - msg += "clone feature, which creates .hg directories and associated mercurial repository files. Not having .hgignore properly\n" - msg += "configured could result in undesired behavior when modifying or updating your local Galaxy instance or the tool shed\n" - msg += "repositories if they are in directories that pose conflicts. See mercurial's .hgignore documentation at the following\n" - msg += "URL for details.\n\nhttp://mercurial.selenic.com/wiki/.hgignore\n\n" - msg += output - msg += "After the installation process finishes, you can start your Galaxy server. As part of this installation process,\n" - msg += "entries for each of the following tool config files will be added to the file named ./migrated_tool_conf.xml, so these\n" - msg += "tools will continue to be loaded into your tool panel. Because of this, existing entries for these files should be\n" - msg += "removed from your file%s named %s, but only after the installation process finishes.\n\n" % ( plural, tool_panel_config_file_names ) - for i, missing_tool_config in enumerate( missing_tool_configs ): - msg += "%s\n" % missing_tool_config - # Should we do the following? - #if i > 10: - # msg += "\n...and %d more tools...\n" % ( len( missing_tool_configs ) - ( i + 1 ) ) - # break - msg += "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n" - raise Exception( msg ) + if not app.config.running_functional_tests: + # Automatically update the value of the migrate_tools.version database table column. + cmd = 'sh manage_tools.sh%s upgrade' % config_arg + proc = subprocess.Popen( args=cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) + return_code = proc.wait() + output = proc.stdout.read( 32768 ) + if return_code != 0: + raise Exception( "Error attempting to update the value of migrate_tools.version: %s" % output ) + elif missing_tool_configs: + if len( tool_panel_configs ) == 1: + plural = '' + tool_panel_config_file_names = tool_panel_configs[ 0 ] + else: + plural = 's' + tool_panel_config_file_names = ', '.join( tool_panel_configs ) + msg = "\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" + msg += "\n\nThe list of files at the end of this message refers to tools that are configured to load into the tool panel for\n" + msg += "this Galaxy instance, but have been removed from the Galaxy distribution. These tools can be automatically installed\n" + msg += "from the Galaxy tool shed at http://toolshed.g2.bx.psu.edu.\n\n" + msg += "To skip this process, attempt to start your Galaxy server again (e.g., sh run.sh or whatever you use). If you do this,\n" + msg += "be aware that these tools will no longer be available in your Galaxy tool panel, and entries for each of them should\n" + msg += "be removed from your file%s named %s.\n\n" % ( plural, tool_panel_config_file_names ) + msg += "CRITICAL NOTE IF YOU PLAN TO INSTALL\n" + msg += "The location in which the tool repositories will be installed is the value of the 'tool_path' attribute in the \n" + msg += 'tag of the file named ./migrated_tool_conf.xml (i.e., ). The default location\n' + msg += "setting is '../shed_tools', which may be problematic for some cluster environments, so make sure to change it before\n" + msg += "you execute the installation process if appropriate. The configured location must be outside of the Galaxy installation\n" + msg += "directory or it must be in a sub-directory protected by a properly configured .hgignore file if the directory is within\n" + msg += "the Galaxy installation directory hierarchy. This is because tool shed repositories will be installed using mercurial's\n" + msg += "clone feature, which creates .hg directories and associated mercurial repository files. Not having .hgignore properly\n" + msg += "configured could result in undesired behavior when modifying or updating your local Galaxy instance or the tool shed\n" + msg += "repositories if they are in directories that pose conflicts. See mercurial's .hgignore documentation at the following\n" + msg += "URL for details.\n\nhttp://mercurial.selenic.com/wiki/.hgignore\n\n" + msg += output + msg += "After the installation process finishes, you can start your Galaxy server. As part of this installation process,\n" + msg += "entries for each of the following tool config files will be added to the file named ./migrated_tool_conf.xml, so these\n" + msg += "tools will continue to be loaded into your tool panel. Because of this, existing entries for these files should be\n" + msg += "removed from your file%s named %s, but only after the installation process finishes.\n\n" % ( plural, tool_panel_config_file_names ) + for i, missing_tool_config in enumerate( missing_tool_configs ): + msg += "%s\n" % missing_tool_config + # Should we do the following? + #if i > 10: + # msg += "\n...and %d more tools...\n" % ( len( missing_tool_configs ) - ( i + 1 ) ) + # break + msg += "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n" + raise Exception( msg ) else: log.info( "At migrate_tools version %d" % db_schema.version ) diff --git a/lib/galaxy/tool_shed/update_manager.py b/lib/galaxy/tool_shed/update_manager.py index eeba4f2de12..95c0396fb05 100644 --- a/lib/galaxy/tool_shed/update_manager.py +++ b/lib/galaxy/tool_shed/update_manager.py @@ -34,7 +34,7 @@ class UpdateManager( object ): log.info( 'Transfer job restarter shutting down...' ) def check_for_update( self, repository ): tool_shed_url = get_url_from_repository_tool_shed( self.app, repository ) - url = '%s/repository/check_for_updates?name=%s&owner=%s&changeset_revision=%s&webapp=update_manager' % \ + url = '%s/repository/check_for_updates?name=%s&owner=%s&changeset_revision=%s&webapp=update_manager&no_reset=true' % \ ( tool_shed_url, repository.name, repository.owner, repository.changeset_revision ) response = urllib2.urlopen( url ) text = response.read() diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 98b1ee847d3..af5da5294e7 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -5,7 +5,7 @@ import pkg_resources pkg_resources.require( "simplejson" ) -import logging, os, string, sys, tempfile, glob, shutil, types, urllib, subprocess +import logging, os, string, sys, tempfile, glob, shutil, types, urllib, subprocess, random import simplejson import binascii from UserDict import DictMixin @@ -31,6 +31,7 @@ from cgi import FieldStorage from galaxy.util.hash_util import * from galaxy.util import listify from galaxy.util.shed_util import * +from galaxy.web import url_for from galaxy.visualization.tracks.visual_analytics import TracksterConfig @@ -114,17 +115,17 @@ class ToolBox( object ): tool_path = self.tool_root_dir # Only load the panel_dict under certain conditions. load_panel_dict = not self.integrated_tool_panel_config_has_contents - for elem in root: + for index, elem in enumerate( root ): if parsing_shed_tool_conf: config_elems.append( elem ) if elem.tag == 'tool': - self.load_tool_tag_set( elem, self.tool_panel, self.integrated_tool_panel, tool_path, load_panel_dict, guid=elem.get( 'guid' ) ) + self.load_tool_tag_set( elem, self.tool_panel, self.integrated_tool_panel, tool_path, load_panel_dict, guid=elem.get( 'guid' ), index=index ) elif elem.tag == 'workflow': - self.load_workflow_tag_set( elem, self.tool_panel, self.integrated_tool_panel, load_panel_dict ) + self.load_workflow_tag_set( elem, self.tool_panel, self.integrated_tool_panel, load_panel_dict, index=index ) elif elem.tag == 'section': - self.load_section_tag_set( elem, tool_path, load_panel_dict ) + self.load_section_tag_set( elem, tool_path, load_panel_dict, index=index ) elif elem.tag == 'label': - self.load_label_tag_set( elem, self.tool_panel, self.integrated_tool_panel ) + self.load_label_tag_set( elem, self.tool_panel, self.integrated_tool_panel, load_panel_dict, index=index ) if parsing_shed_tool_conf: shed_tool_conf_dict = dict( config_filename=config_filename, tool_path=tool_path, @@ -286,7 +287,7 @@ class ToolBox( object ): self.app.model.ToolShedRepository.table.c.owner == owner, self.app.model.ToolShedRepository.table.c.installed_changeset_revision == installed_changeset_revision ) ) \ .first() - def load_tool_tag_set( self, elem, panel_dict, integrated_panel_dict, tool_path, load_panel_dict, guid=None ): + def load_tool_tag_set( self, elem, panel_dict, integrated_panel_dict, tool_path, load_panel_dict, guid=None, index=None ): try: path = elem.get( "file" ) if guid is None: @@ -354,10 +355,13 @@ class ToolBox( object ): if load_panel_dict: panel_dict[ key ] = tool # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file. - integrated_panel_dict[ key ] = tool + if key in integrated_panel_dict or index is None: + integrated_panel_dict[ key ] = tool + else: + integrated_panel_dict.insert( index, key, tool ) except: log.exception( "Error reading tool from path: %s" % path ) - def load_workflow_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict ): + def load_workflow_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ): try: # TODO: should id be encoded? workflow_id = elem.get( 'id' ) @@ -367,16 +371,22 @@ class ToolBox( object ): if load_panel_dict: panel_dict[ key ] = workflow # Always load workflows into the integrated_panel_dict. - integrated_panel_dict[ key ] = workflow + if key in integrated_panel_dict or index is None: + integrated_panel_dict[ key ] = workflow + else: + integrated_panel_dict.insert( index, key, workflow ) except: log.exception( "Error loading workflow: %s" % workflow_id ) - def load_label_tag_set( self, elem, panel_dict, integrated_panel_dict ): + def load_label_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ): label = ToolSectionLabel( elem ) key = 'label_' + label.id - if not self.integrated_tool_panel_config_has_contents: + if load_panel_dict: panel_dict[ key ] = label - integrated_panel_dict[ key ] = label - def load_section_tag_set( self, elem, tool_path, load_panel_dict ): + if key in integrated_panel_dict or index is None: + integrated_panel_dict[ key ] = label + else: + integrated_panel_dict.insert( index, key, label ) + def load_section_tag_set( self, elem, tool_path, load_panel_dict, index=None ): key = 'section_' + elem.get( "id" ) if key in self.tool_panel: section = self.tool_panel[ key ] @@ -390,17 +400,20 @@ class ToolBox( object ): else: integrated_section = ToolSection( elem ) integrated_elems = integrated_section.elems - for sub_elem in elem: + for sub_index, sub_elem in enumerate( elem ): if sub_elem.tag == 'tool': - self.load_tool_tag_set( sub_elem, elems, integrated_elems, tool_path, load_panel_dict, guid=sub_elem.get( 'guid' ) ) + self.load_tool_tag_set( sub_elem, elems, integrated_elems, tool_path, load_panel_dict, guid=sub_elem.get( 'guid' ), index=sub_index ) elif sub_elem.tag == 'workflow': - self.load_workflow_tag_set( sub_elem, elems, integrated_elems, load_panel_dict ) + self.load_workflow_tag_set( sub_elem, elems, integrated_elems, load_panel_dict, index=sub_index ) elif sub_elem.tag == 'label': - self.load_label_tag_set( sub_elem, elems, integrated_elems ) + self.load_label_tag_set( sub_elem, elems, integrated_elems, load_panel_dict, index=sub_index ) if load_panel_dict: self.tool_panel[ key ] = section # Always load sections into the integrated_tool_panel. - self.integrated_tool_panel[ key ] = integrated_section + if key in self.integrated_tool_panel or index is None: + self.integrated_tool_panel[ key ] = integrated_section + else: + self.integrated_tool_panel.insert( index, key, integrated_section ) def load_tool( self, config_file, guid=None ): """Load a single tool from the file named by `config_file` and return an instance of `Tool`.""" # Parse XML configuration file and get the root element @@ -474,17 +487,119 @@ class ToolBox( object ): Returns a SQLAlchemy session """ return self.app.model.context + + def to_dict( self, trans, in_panel=True, trackster=False ): + + def filter_for_panel( item, filters ): + """ + Filters tool panel elements so that only those that are compatible + with provided filters are kept. + """ + def _apply_filter( filter_item, filter_list ): + for filter_method in filter_list: + if not filter_method( filter_item ): + return False + return True + if isinstance( item, Tool ): + if _apply_filter( item, filters[ 'tool' ] ): + return item + elif isinstance( item, ToolSectionLabel ): + if _apply_filter( item, filters[ 'label' ] ): + return item + elif isinstance( item, ToolSection ): + # Filter section item-by-item. Only show a label if there are + # non-filtered tools below it. + + if _apply_filter( item, filters[ 'section' ] ): + cur_label_key = None + tools_under_label = False + filtered_elems = item.elems.copy() + for key, section_item in item.elems.items(): + if isinstance( section_item, Tool ): + # Filter tool. + if _apply_filter( section_item, filters[ 'tool' ] ): + tools_under_label = True + else: + del filtered_elems[ key ] + elif isinstance( section_item, ToolSectionLabel ): + # If there is a label and it does not have tools, + # remove it. + if ( cur_label_key and not tools_under_label ) or not _apply_filter( section_item, filters[ 'label' ] ): + del filtered_elems[ cur_label_key ] + + # Reset attributes for new label. + cur_label_key = key + tools_under_label = False + + + # Handle last label. + if cur_label_key and not tools_under_label: + del filtered_elems[ cur_label_key ] + + # Only return section if there are elements. + if len( filtered_elems ) != 0: + copy = item.copy() + copy.elems = filtered_elems + return copy + + return None + + # + # Dictify toolbox. + # + + if in_panel: + panel_elts = [ val for val in self.tool_panel.itervalues() ] + + # Filter if necessary. + filters = dict( tool=[ lambda x: not x._is_hidden_for_user( trans.user ) ], section=[], label=[] ) #hidden tools filter + if trackster: + filters[ 'tool' ].append( lambda x: x.trackster_conf ) # If tool has a trackster config, it can be used in Trackster. + filtered_panel_elts = [] + for index, elt in enumerate( panel_elts ): + elt = filter_for_panel( elt, filters ) + if elt: + filtered_panel_elts.append( elt ) + panel_elts = filtered_panel_elts + + # Produce panel. + rval = [] + for elt in panel_elts: + rval.append( elt.to_dict( trans, for_link=True ) ) + else: + tools = [] + for id, tool in self.app.toolbox.tools_by_id.items(): + tools.append( tool.to_dict( trans ) ) + rval = tools + + return rval class ToolSection( object ): """ A group of tools with similar type/purpose that will be displayed as a group in the user interface. """ - def __init__( self, elem ): - self.name = elem.get( "name" ) - self.id = elem.get( "id" ) - self.version = elem.get( "version" ) or '' + def __init__( self, elem=None ): + f = lambda elem, val: elem is not None and elem.get( val ) or '' + self.name = f( elem, 'name' ) + self.id = f( elem, 'id' ) + self.version = f( elem, 'version' ) self.elems = odict() + + def copy( self ): + copy = ToolSection() + copy.name = self.name + copy.id = self.id + copy.version = self.version + copy.elems = self.elems.copy() + return copy + + def to_dict( self, trans, for_link=False ): + """ Return a dict that includes section's attributes. """ + section_elts = [] + for key, val in self.elems.items(): + section_elts.append( val.to_dict( trans, for_link=for_link ) ) + return { 'type': 'section', 'id': self.id, 'name': self.name, 'version': self.version, 'elems': section_elts } class ToolSectionLabel( object ): """ @@ -495,6 +610,10 @@ class ToolSectionLabel( object ): self.text = elem.get( "text" ) self.id = elem.get( "id" ) self.version = elem.get( "version" ) or '' + + def to_dict( self, trans, **kwargs ): + """ Return a dict that includes label's attributes. """ + return { 'type': 'label', 'id': self.id, 'name': self.text, 'version': self.version } class DefaultToolState( object ): """ @@ -572,22 +691,24 @@ class ToolOutput( object ): def __iter__( self ): return iter( ( self.format, self.metadata_source, self.parent ) ) + + def to_dict( self ): + return { + 'name': self.name, + 'format': self.format, + 'label': self.label, + 'hidden': self.hidden + } class ToolRequirement( object ): """ - Represents an external requirement that must be available for the tool to - run (for example, a program, package, or library). Requirements can - optionally assert a specific version, or reference a command to execute a - fabric script. If fabric is used, the type is 'fabfile' and the version - attribute is not used since the fabric script includes all necessary - information for automatic dependency installation. + Represents an external requirement that must be available for the tool to run (for example, a program, package, or library). + Requirements can optionally assert a specific version. """ - def __init__( self, name=None, type=None, version=None, fabfile=None, method=None ): + def __init__( self, name=None, type=None, version=None ): self.name = name self.type = type self.version = version - self.fabfile = fabfile - self.method = method class ToolParallelismInfo(object): """ @@ -625,6 +746,8 @@ class Tool: self.check_values = True self.nginx_upload = False self.input_required = False + self.display_interface = True + self.require_login = False # Define a place to keep track of all input parameters. These # differ from the inputs dictionary in that inputs can be page # elements like conditionals, but input_params are basic form @@ -669,31 +792,35 @@ class Tool: if tool_version: return tool_version.get_version_ids( self.app ) return [] - def get_job_runner( self, job_params=None ): - # Look through runners to find one with matching parameters. - selected_runner = None - if len( self.job_runners ) == 1: - # Most tools have a single runner. - selected_runner = self.job_runners[0] + def __get_job_run_config( self, run_configs, key, job_params=None ): + # Look through runners/handlers to find one with matching parameters. + available_configs = [] + if len( run_configs ) == 1: + # Most tools have a single config. + return run_configs[0][ key ] # return to avoid random when this will be the case most of the time elif job_params is None: - # Use job runner with no params - for runner in self.job_runners: - if "params" not in runner: - selected_runner = runner + # Use job config with no params + for config in run_configs: + if "params" not in config: + available_configs.append( config ) else: - # Find runner with matching parameters. - for runner in self.job_runners: - if "params" in runner: + # Find config with matching parameters. + for config in run_configs: + if "params" in config: match = True - runner_params = runner[ "params" ] + config_params = config[ "params" ] for param, value in job_params.items(): - if param not in runner_params or \ - runner_params[ param ] != job_params[ param ]: + if param not in config_params or \ + config_params[ param ] != job_params[ param ]: match = False break if match: - selected_runner = runner - return selected_runner[ "url" ] + available_configs.append( config ) + return random.choice( available_configs )[ key ] + def get_job_runner( self, job_params=None ): + return self.__get_job_run_config( self.job_runners, key='url', job_params=job_params ) + def get_job_handler( self, job_params=None ): + return self.__get_job_run_config( self.job_handlers, key='name', job_params=job_params ) def parse( self, root, guid=None ): """ Read tool configuration from the element `root` and fill in `self`. @@ -721,6 +848,8 @@ class Tool: # Useful i.e. when an indeterminate number of outputs are created by # a tool. self.force_history_refresh = util.string_as_bool( root.get( 'force_history_refresh', 'False' ) ) + self.display_interface = util.string_as_bool( root.get( 'display_interface', str( self.display_interface ) ) ) + self.require_login = util.string_as_bool( root.get( 'require_login', str( self.require_login ) ) ) # Load input translator, used by datasource tools to change # names/values of incoming parameters self.input_translator = root.find( "request_param_translation" ) @@ -758,6 +887,12 @@ class Tool: self.parallelism = ToolParallelismInfo(parallelism) else: self.parallelism = None + # Set job handler(s). Each handler is a dict with 'url' and, optionally, 'params'. + self_id = self.id.lower() + self.job_handlers = [ { "name" : name } for name in self.app.config.default_job_handlers ] + # Set custom handler(s) if they're defined. + if self_id in self.app.config.tool_handlers: + self.job_handlers = self.app.config.tool_handlers[ self_id ] # Set job runner(s). Each runner is a dict with 'url' and, optionally, 'params'. if self.app.config.start_job_runners is None: # Jobs are always local regardless of tool config if no additional @@ -767,7 +902,6 @@ class Tool: # Set job runner to the cluster default self.job_runners = [ { "url" : self.app.config.default_cluster_job_runner } ] # Set custom runner(s) if they're defined. - self_id = self.id.lower() if self_id in self.app.config.tool_runners: self.job_runners = self.app.config.tool_runners[ self_id ] # Is this a 'hidden' tool (hidden in tool menu) @@ -843,7 +977,7 @@ class Tool: self.is_workflow_compatible = self.check_workflow_compatible() # Trackster configuration. trackster_conf = root.find( "trackster_conf" ) - if trackster_conf: + if trackster_conf is not None: self.trackster_conf = TracksterConfig.parse( trackster_conf ) else: self.trackster_conf = None @@ -1199,19 +1333,8 @@ class Tool: for requirement_elem in requirements_elem.findall( 'requirement' ): name = util.xml_text( requirement_elem ) type = requirement_elem.get( "type", "package" ) - if type == 'fabfile': - # The fabric script will include all necessary information for - # automatically installing the tool dependencies. - fabfile = requirement_elem.get( "fabfile" ) - method = requirement_elem.get( "method" ) - version = None - else: - # For backward compatibility, requirements tag sets should not require the - # use of a fabric script. - version = requirement_elem.get( "version" ) - fabfile = None - method = None - requirement = ToolRequirement( name=name, type=type, version=version, fabfile=fabfile, method=method ) + version = requirement_elem.get( "version", None ) + requirement = ToolRequirement( name=name, type=type, version=version ) self.requirements.append( requirement ) def check_workflow_compatible( self ): """ @@ -1326,6 +1449,8 @@ class Tool: # on the standard run form) or "URL" (a parameter provided by # external data source tools). if "runtool_btn" not in incoming and "URL" not in incoming: + if not self.display_interface: + return 'message.mako', dict( status='info', message="The interface for this tool cannot be displayed", refresh_frames=['everything'] ) return "tool_form.mako", dict( errors={}, tool_state=state, param_values={}, incoming={} ) # Process incoming data if not( self.check_values ): @@ -1374,6 +1499,8 @@ class Tool: state.page += 1 # Fill in the default values for the next page self.fill_in_new_state( trans, self.inputs_by_page[ state.page ], state.inputs ) + if not self.display_interface: + return 'message.mako', dict( status='info', message="The interface for this tool cannot be displayed", refresh_frames=['everything'] ) return 'tool_form.mako', dict( errors=errors, tool_state=state ) else: try: @@ -1386,6 +1513,8 @@ class Tool: except: pass # Just a refresh, render the form with updated state and errors. + if not self.display_interface: + return 'message.mako', dict( status='info', message="The interface for this tool cannot be displayed", refresh_frames=['everything'] ) return 'tool_form.mako', dict( errors=errors, tool_state=state ) def find_fieldstorage( self, x ): if isinstance( x, FieldStorage ): @@ -2110,7 +2239,7 @@ class Tool: shutil.rmtree(temp_file_path) except: continue - def collect_child_datasets( self, output): + def collect_child_datasets( self, output, job_working_directory ): """ Look for child dataset files, create HDA and attach to parent. """ @@ -2118,7 +2247,12 @@ class Tool: # Loop through output file names, looking for generated children in # form of 'child_parentId_designation_visibility_extension' for name, outdata in output.items(): - for filename in glob.glob(os.path.join(self.app.config.new_file_path,"child_%i_*" % outdata.id) ): + filenames = [] + if 'new_file_path' in self.app.config.collect_outputs_from: + filenames.extend( glob.glob(os.path.join(self.app.config.new_file_path,"child_%i_*" % outdata.id) ) ) + if 'job_working_directory' in self.app.config.collect_outputs_from: + filenames.extend( glob.glob(os.path.join(job_working_directory,"child_%i_*" % outdata.id) ) ) + for filename in filenames: if not name in children: children[name] = {} fields = os.path.basename(filename).split("_") @@ -2138,7 +2272,7 @@ class Tool: sa_session=self.sa_session ) self.app.security_agent.copy_dataset_permissions( outdata.dataset, child_dataset.dataset ) # Move data from temp location to dataset location - self.app.object_store.update_from_file(child_dataset.dataset, filename, create=True) + self.app.object_store.update_from_file(child_dataset.dataset, file_name=filename, create=True) self.sa_session.add( child_dataset ) self.sa_session.flush() child_dataset.set_size() @@ -2170,7 +2304,7 @@ class Tool: self.sa_session.add( child_dataset ) self.sa_session.flush() return children - def collect_primary_datasets( self, output): + def collect_primary_datasets( self, output, job_working_directory ): """ Find any additional datasets generated by a tool and attach (for cases where number of outputs is not known in advance). @@ -2180,7 +2314,12 @@ class Tool: # datasets in form of: # 'primary_associatedWithDatasetID_designation_visibility_extension(_DBKEY)' for name, outdata in output.items(): - for filename in glob.glob(os.path.join(self.app.config.new_file_path,"primary_%i_*" % outdata.id) ): + filenames = [] + if 'new_file_path' in self.app.config.collect_outputs_from: + filenames.extend( glob.glob(os.path.join(self.app.config.new_file_path,"primary_%i_*" % outdata.id) ) ) + if 'job_working_directory' in self.app.config.collect_outputs_from: + filenames.extend( glob.glob(os.path.join(job_working_directory,"primary_%i_*" % outdata.id) ) ) + for filename in filenames: if not name in primary_datasets: primary_datasets[name] = {} fields = os.path.basename(filename).split("_") @@ -2205,7 +2344,7 @@ class Tool: self.sa_session.add( primary_data ) self.sa_session.flush() # Move data from temp location to dataset location - self.app.object_store.update_from_file(primary_data.dataset, filename, create=True) + self.app.object_store.update_from_file(primary_data.dataset, file_name=filename, create=True) primary_data.set_size() primary_data.name = "%s (%s)" % ( outdata.name, designation ) primary_data.info = outdata.info @@ -2238,6 +2377,54 @@ class Tool: self.sa_session.add( new_data ) self.sa_session.flush() return primary_datasets + + def _is_hidden_for_user( self, user ): + if self.hidden or ( not user and self.require_login ): + return True + return False + + def to_dict( self, trans, for_link=False, for_display=False ): + """ Returns dict of tool. """ + + # Basic information + tool_dict = { 'id': self.id, 'name': self.name, + 'version': self.version, 'description': self.description } + + if for_link: + # Create tool link. + if not self.tool_type.startswith( 'data_source' ): + link = url_for( controller='tool_runner', tool_id=self.id ) + else: + link = url_for( self.action, **self.get_static_param_values( trans ) ) + + # Basic information + tool_dict.update( { 'type': 'tool', 'link': link, + 'min_width': self.uihints.get( 'minwidth', -1 ), + 'target': self.target } ) + + if for_display: + # Dictify inputs. + inputs = [] + for name, input in self.inputs.items(): + param_dict = { 'name' : name, 'label' : input.label } + if isinstance( input, DataToolParameter ): + param_dict.update( { 'type' : 'data', 'html' : urllib.quote( input.get_html( trans ) ) } ) + elif isinstance( input, SelectToolParameter ): + param_dict.update( { 'type' : 'select', 'html' : urllib.quote( input.get_html( trans ) ) } ) + elif isinstance( input, Conditional ): + # TODO. + pass + else: + param_dict.update( { 'type' : '??', 'init_value' : input.value, \ + 'html' : urllib.quote( input.get_html( trans ) ) } ) + inputs.append( param_dict ) + + tool_dict[ 'inputs' ] = inputs + + # Dictify outputs. + pass + + return tool_dict class DataSourceTool( Tool ): """ @@ -2369,6 +2556,9 @@ class ExportHistoryTool( Tool ): class ImportHistoryTool( Tool ): tool_type = 'import_history' +class GenomeIndexTool( Tool ): + tool_type = 'index_genome' + # Populate tool_type to ToolClass mappings tool_types = {} for tool_class in [ Tool, DataDestinationTool, SetMetadataTool, DataSourceTool, AsyncDataSourceTool ]: diff --git a/lib/galaxy/tools/actions/index_genome.py b/lib/galaxy/tools/actions/index_genome.py new file mode 100644 index 00000000000..08e7f698965 --- /dev/null +++ b/lib/galaxy/tools/actions/index_genome.py @@ -0,0 +1,65 @@ +import tempfile +from __init__ import ToolAction +from galaxy.util.odict import odict +from galaxy.tools.genome_index import * + +import logging +log = logging.getLogger( __name__ ) + +class GenomeIndexToolAction( ToolAction ): + """Tool action used for exporting a history to an archive. """ + + def execute( self, tool, trans, *args, **kwargs ): + # + # Get genome to index. + # + incoming = kwargs['incoming'] + # + # Create the job and output dataset objects + # + job = trans.app.model.Job() + job.tool_id = tool.id + job.user_id = incoming['user'] + start_job_state = job.state # should be job.states.NEW + job.state = job.states.WAITING # we need to set job state to something other than NEW, or else when tracking jobs in db it will be picked up before we have added input / output parameters + trans.sa_session.add( job ) + + # Create dataset that will serve as archive. + temp_dataset = trans.app.model.Dataset( state=trans.app.model.Dataset.states.NEW ) + trans.sa_session.add( temp_dataset ) + + trans.sa_session.flush() # ensure job.id and archive_dataset.id are available + trans.app.object_store.create( temp_dataset ) # set the object store id, create dataset (because galaxy likes having datasets) + + # + # Setup job and job wrapper. + # + + # Add association for keeping track of job, history, archive relationship. + user = trans.sa_session.query( trans.app.model.User ).get( int( incoming['user'] ) ) + assoc = trans.app.model.GenomeIndexToolData( job=job, dataset=temp_dataset, fasta_path=incoming['path'], \ + indexer=incoming['indexer'], user=user, \ + deferred_job=kwargs['deferred'], transfer_job=kwargs['transfer'] ) + trans.sa_session.add( assoc ) + + job_wrapper = GenomeIndexToolWrapper( job ) + cmd_line = job_wrapper.setup_job( assoc ) + # + # Add parameters to job_parameter table. + # + + # Set additional parameters. + incoming[ '__GENOME_INDEX_COMMAND__' ] = cmd_line + for name, value in tool.params_to_strings( incoming, trans.app ).iteritems(): + job.add_parameter( name, value ) + + job.state = start_job_state # job inputs have been configured, restore initial job state + trans.sa_session.flush() + + + # Queue the job for execution + trans.app.job_queue.put( job.id, tool ) + log.info( "Added genome index job to the job queue, id: %s" % str( job.id ) ) + + return job, odict() + diff --git a/lib/galaxy/tools/actions/metadata.py b/lib/galaxy/tools/actions/metadata.py index 000a1b9c855..ed4c9494793 100644 --- a/lib/galaxy/tools/actions/metadata.py +++ b/lib/galaxy/tools/actions/metadata.py @@ -1,6 +1,7 @@ from __init__ import ToolAction from galaxy.datatypes.metadata import JobExternalOutputMetadataWrapper from galaxy.util.odict import odict +from galaxy.util.json import to_json_string import logging log = logging.getLogger( __name__ ) @@ -8,7 +9,7 @@ log = logging.getLogger( __name__ ) class SetMetadataToolAction( ToolAction ): """Tool action used for setting external metadata on an existing dataset""" - def execute( self, tool, trans, incoming = {}, set_output_hid = False, overwrite = True, history=None ): + def execute( self, tool, trans, incoming = {}, set_output_hid = False, overwrite = True, history=None, job_params=None ): for name, value in incoming.iteritems(): if isinstance( value, trans.app.model.HistoryDatasetAssociation ): dataset = value @@ -30,6 +31,8 @@ class SetMetadataToolAction( ToolAction ): job.tool_id = tool.id if trans.user: job.user_id = trans.user.id + if job_params: + job.params = to_json_string( job_params ) start_job_state = job.state #should be job.states.NEW try: # For backward compatibility, some tools may not have versions yet. diff --git a/lib/galaxy/tools/genome_index/__init__.py b/lib/galaxy/tools/genome_index/__init__.py new file mode 100644 index 00000000000..91411f82100 --- /dev/null +++ b/lib/galaxy/tools/genome_index/__init__.py @@ -0,0 +1,184 @@ +import os, shutil, logging, tempfile, json, tarfile +from galaxy import model, util +from galaxy.web.framework.helpers import to_unicode +from galaxy.model.item_attrs import UsesAnnotations +from galaxy.util.json import * +from galaxy.web.base.controller import UsesHistory +from galaxy.tools.data import ToolDataTableManager + +log = logging.getLogger(__name__) + +def load_genome_index_tools( toolbox ): + """ Adds tools for indexing genomes via the main job runner. """ + # Use same process as that used in load_external_metadata_tool; see that + # method for why create tool description files on the fly. + tool_xml_text = """ + + + + $__GENOME_INDEX_COMMAND__ $output_file $output_file.files_path + + + + + + + + """ + + # Load export tool. + tmp_name = tempfile.NamedTemporaryFile() + tmp_name.write( tool_xml_text ) + tmp_name.flush() + genome_index_tool = toolbox.load_tool( tmp_name.name ) + toolbox.tools_by_id[ genome_index_tool.id ] = genome_index_tool + log.debug( "Loaded genome index tool: %s", genome_index_tool.id ) + +class GenomeIndexToolWrapper( object ): + """ Provides support for performing jobs that index a genome. """ + def __init__( self, job_id ): + self.locations = dict() + self.job_id = job_id + + def setup_job( self, genobj ): + """ Perform setup for job to index a genome and return an archive. Method generates + attribute files, sets the corresponding attributes in the associated database + object, and returns a command line for running the job. The command line + includes the command, inputs, and options; it does not include the output + file because it must be set at runtime. """ + + # + # Create and return command line for running tool. + # + scriptpath = os.path.join( os.path.abspath( os.getcwd() ), "lib/galaxy/tools/genome_index/index_genome.py" ) + return "python %s %s %s" % ( scriptpath, genobj.indexer, genobj.fasta_path ) + + def postprocessing( self, sa_session, app ): + """ Finish the job, move the finished indexes to their final resting place, + and update the .loc files where applicable. """ + gitd = sa_session.query( model.GenomeIndexToolData ).filter_by( job_id=self.job_id ).first() + indexdirs = dict( bfast='bfast_index', bowtie='bowtie_index', bowtie2='bowtie2_index', + bwa='bwa_index', perm='perm_%s_index', picard='srma_index', sam='sam_index' ) + + + if gitd: + destination = None + tdtman = ToolDataTableManager() + xmltree = tdtman.load_from_config_file(app.config.tool_data_table_config_path) + for node in xmltree: + table = node.get('name') + location = node.findall('file')[0].get('path') + self.locations[table] = os.path.abspath( location ) + locbase = os.path.abspath( os.path.split( self.locations['all_fasta'] )[0] ) + deferred = sa_session.query( model.DeferredJob ).filter_by( id=gitd.deferred_job_id ).first() + params = deferred.params + dbkey = params[ 'dbkey' ] + basepath = os.path.join( os.path.abspath( app.config.genome_data_path ), dbkey ) + intname = params[ 'intname' ] + indexer = gitd.indexer + workingdir = os.path.abspath( gitd.dataset.extra_files_path ) + fp = open( gitd.dataset.get_file_name(), 'r' ) + logloc = json.load( fp ) + fp.close() + location = [] + indexdata = gitd.dataset.extra_files_path + if indexer == '2bit': + indexdata = os.path.join( workingdir, '%s.2bit' % dbkey ) + destination = os.path.join( basepath, 'seq', '%s.2bit' % dbkey ) + location.append( dict( line='\t'.join( [ 'seq', dbkey, os.path.join( destination, '%s.2bit' % dbkey ) ] ), file= os.path.join( locbase, 'alignseq.loc' ) ) ) + elif indexer == 'bowtie': + self._ex_tar( workingdir, 'cs.tar' ) + destination = os.path.join( basepath, 'bowtie_index' ) + for var in [ 'nt', 'cs' ]: + for line in logloc[ var ]: + idx = line + if var == 'nt': + locfile = self.locations[ 'bowtie_indexes' ] + locdir = os.path.join( destination, idx ) + else: + locfile = self.locations[ 'bowtie_indexes_color' ] + locdir = os.path.join( destination, var, idx ) + location.append( dict( line='\t'.join( [ dbkey, dbkey, intname, locdir ] ), file=locfile ) ) + elif indexer == 'bowtie2': + destination = os.path.join( basepath, 'bowtie2_index' ) + for line in logloc[ 'nt' ]: + idx = line + locfile = self.locations[ 'bowtie2_indexes' ] + locdir = os.path.join( destination, idx ) + location.append( dict( line='\t'.join( [ dbkey, dbkey, intname, locdir ] ), file=locfile ) ) + elif indexer == 'bwa': + self._ex_tar( workingdir, 'cs.tar' ) + destination = os.path.join( basepath, 'bwa_index' ) + for var in [ 'nt', 'cs' ]: + for line in logloc[ var ]: + idx = line + if var == 'nt': + locfile = self.locations[ 'bwa_indexes' ] + locdir = os.path.join( destination, idx ) + else: + locfile = self.locations[ 'bwa_indexes_color' ] + locdir = os.path.join( destination, var, idx ) + location.append( dict( line='\t'.join( [ dbkey, dbkey, intname, locdir ] ), file=locfile ) ) + elif indexer == 'perm': + self._ex_tar( workingdir, 'cs.tar' ) + destination = os.path.join( basepath, 'perm_index' ) + for var in [ 'nt', 'cs' ]: + for line in logloc[ var ]: + idx = line.pop() + if var == 'nt': + locfile = self.locations[ 'perm_base_indexes' ] + locdir = os.path.join( destination, idx ) + else: + locfile = self.locations[ 'perm_color_indexes' ] + locdir = os.path.join( destination, var, idx ) + line.append( locdir ) + location.append( dict( line='\t'.join( line ), file=locfile ) ) + elif indexer == 'picard': + destination = os.path.join( basepath, 'srma_index' ) + for var in [ 'nt' ]: + for line in logloc[ var ]: + idx = line + locfile = self.locations[ 'picard_indexes' ] + locdir = os.path.join( destination, idx ) + location.append( dict( line='\t'.join( [ dbkey, dbkey, intname, locdir ] ), file=locfile ) ) + elif indexer == 'sam': + destination = os.path.join( basepath, 'sam_index' ) + for var in [ 'nt' ]: + for line in logloc[ var ]: + locfile = self.locations[ 'sam_fa_indexes' ] + locdir = os.path.join( destination, line ) + location.append( dict( line='\t'.join( [ 'index', dbkey, locdir ] ), file=locfile ) ) + + if destination is not None and os.path.exists( os.path.split( destination )[0] ) and not os.path.exists( destination ): + log.debug( 'Moving %s to %s' % ( indexdata, destination ) ) + shutil.move( indexdata, destination ) + if indexer not in [ '2bit' ]: + genome = '%s.fa' + target = os.path.join( destination, genome ) + farel = os.path.relpath( os.path.join( basepath, 'seq', genome ), destination ) + os.symlink( farel, target ) + if os.path.exists( os.path.join( destination, 'cs' ) ): + target = os.path.join( destination, 'cs', genome ) + farel = os.path.relpath( os.path.join( basepath, 'seq', genome ), os.path.join( destination, 'cs' ) ) + os.symlink( os.path.join( farel, target ) ) + for line in location: + self._add_line( line[ 'file' ], line[ 'line' ] ) + + def _ex_tar( self, directory, filename ): + fh = tarfile.open( os.path.join( directory, filename ) ) + fh.extractall( path=directory ) + fh.close() + os.remove( os.path.join( directory, filename ) ) + + def _add_line( self, locfile, newline ): + filepath = locfile + origlines = [] + output = [] + comments = [] + with open( filepath, 'r' ) as destfile: + for line in destfile: + origlines.append( line.strip() ) + if newline not in origlines: + origlines.append( newline ) + with open( filepath, 'w+' ) as destfile: + destfile.write( '\n'.join( origlines ) ) diff --git a/lib/galaxy/tools/genome_index/index_genome.py b/lib/galaxy/tools/genome_index/index_genome.py new file mode 100644 index 00000000000..96c54d16c02 --- /dev/null +++ b/lib/galaxy/tools/genome_index/index_genome.py @@ -0,0 +1,315 @@ +#!/usr/bin/env python +""" +Export a history to an archive file using attribute files. + +usage: %prog history_attrs dataset_attrs job_attrs out_file + -G, --gzip: gzip archive file +""" +import optparse, sys, os, tempfile, time, subprocess, shlex, json, tarfile, shutil + +class ManagedIndexer(): + def __init__( self, output_file, infile, workingdir ): + self.workingdir = os.path.abspath( workingdir ) + self.outfile = open( os.path.abspath( output_file ), 'w' ) + self.basedir = os.path.split( self.workingdir )[0] + self.fasta = os.path.abspath( infile ) + self.locations = dict( nt=[], cs=[] ) + self.log = [] + self.indexers = { + 'bwa': '_bwa', + 'bowtie': '_bowtie', + 'bowtie2': '_bowtie2', + '2bit': '_twobit', + 'perm': '_perm', + 'bfast': '_bfast', + 'picard': '_picard', + 'sam': '_sam' + } + if not os.path.exists( self.workingdir ): + os.makedirs( self.workingdir ) + self.logfile = open( os.path.join( self.workingdir, 'ManagedIndexer.log' ), 'w+' ) + + def run_indexer( self, indexer ): + self.fapath = self.fasta + self.fafile = os.path.basename( self.fapath ) + with WithChDir( self.basedir ): + if indexer not in self.indexers: + raise KeyError, 'The requested indexing function does not exist' + else: + with WithChDir( self.workingdir ): + self._log( 'Running indexer %s.' % indexer ) + result = getattr( self, self.indexers[ indexer ] )() + if result is None: + self._log( 'Error running indexer %s.' % indexer ) + self._flush_files() + raise Exception + else: + self._log( 'Indexer %s completed successfully.' % indexer ) + self._flush_files() + + def _flush_files( self ): + json.dump( self.locations, self.outfile ) + self.outfile.close() + self.logfile.close() + + def _log( self, stuff ): + timestamp = time.strftime('%Y-%m-%d %H:%M:%S %z') + self.logfile.write( "[%s] %s\n" % (timestamp, stuff) ) + + def _bwa( self ): + with WithChDir( self.workingdir ): + if not os.path.exists( self.fafile ): + os.symlink( os.path.relpath( self.fapath ), self.fafile ) + command = shlex.split( 'bwa index -a bwtsw %s' % self.fafile ) + result = subprocess.call( command, stderr=self.logfile, stdout=self.logfile ) + if result != 0: + newcommand = shlex.split( 'bwa index -c %s' % self.fafile ) + result = call( newcommand, stderr=self.logfile, stdout=self.logfile ) + if result == 0: + self.locations[ 'nt' ].append( self.fafile ) + os.remove( self.fafile ) + os.makedirs( 'cs' ) + with WithChDir( 'cs' ): + if not os.path.exists( self.fafile ): + os.symlink( os.path.relpath( self.fapath ), self.fafile ) + command = shlex.split( 'bwa index -a bwtsw -c %s' % self.fafile ) + result = subprocess.call( command, stderr=self.logfile, stdout=self.logfile ) + if result != 0: + newcommand = shlex.split( 'bwa index -c %s' % self.fafile ) + result = call( newcommand, stderr=self.logfile, stdout=self.logfile ) + if result == 0: + self.locations[ 'cs' ].append( self.fafile ) + os.remove( self.fafile ) + else: + return False + else: + self.locations[ 'cs' ].append( self.fafile ) + os.remove( self.fafile ) + temptar = tarfile.open( 'cs.tar', 'w' ) + temptar.add( 'cs' ) + temptar.close() + shutil.rmtree( 'cs' ) + return True + else: + return False + + def _bowtie( self ): + ref_base = os.path.splitext(self.fafile)[0] + if not os.path.exists( self.fafile ): + os.symlink( os.path.relpath( self.fapath ), self.fafile ) + command = shlex.split( 'bowtie-build -f %s %s' % ( self.fafile, ref_base ) ) + result = subprocess.call( command, stderr=self.logfile, stdout=self.logfile ) + if result == 0: + self.locations[ 'nt' ].append( ref_base ) + os.remove( self.fafile ) + indexdir = os.path.join( os.getcwd(), 'cs' ) + os.makedirs( indexdir ) + with WithChDir( indexdir ): + ref_base = os.path.splitext(self.fafile)[0] + if not os.path.exists( self.fafile ): + os.symlink( os.path.relpath( self.fapath ), self.fafile ) + command = shlex.split( 'bowtie-build -C -f %s %s' % ( self.fafile, ref_base ) ) + result = subprocess.call( command, stderr=self.logfile, stdout=self.logfile ) + if result == 0: + self.locations[ 'cs' ].append( ref_base ) + else: + return False + os.remove( os.path.join( indexdir, self.fafile ) ) + temptar = tarfile.open( 'cs.tar', 'w' ) + temptar.add( 'cs' ) + temptar.close() + shutil.rmtree( 'cs' ) + return True + else: + return False + + def _bowtie2( self ): + ref_base = os.path.splitext(self.fafile)[0] + if not os.path.exists( self.fafile ): + os.symlink( os.path.relpath( self.fapath ), self.fafile ) + command = shlex.split( 'bowtie2-build %s %s' % ( self.fafile, ref_base ) ) + result = subprocess.call( command, stderr=self.logfile, stdout=self.logfile ) + if result == 0: + self.locations[ 'nt' ].append( ref_base ) + os.remove( self.fafile ) + return True + else: + return False + + def _twobit( self ): + """Index reference files using 2bit for random access. + """ + ref_base = os.path.splitext(self.fafile)[0] + out_file = "%s.2bit" % ref_base + if not os.path.exists( self.fafile ): + os.symlink( os.path.relpath( self.fapath ), self.fafile ) + command = shlex.split( 'faToTwoBit %s %s' % ( self.fafile, out_file ) ) + result = subprocess.call( command, stderr=self.logfile, stdout=self.logfile ) + if result == 0: + self.locations['nt'].append( out_file ) + os.remove( self.fafile ) + return True + else: + return False + + def _perm( self ): + local_ref = self.fafile + if not os.path.exists( local_ref ): + os.symlink( os.path.relpath( self.fapath ), self.fafile ) + genome = os.path.splitext( local_ref )[0] + read_length = 50 + for seed in [ 'F3', 'F4' ]: + key = '%s_%s_%s' % (genome, seed, read_length) + desc = '%s: seed=%s, read length=%s' % (genome, seed, read_length) + index = "%s_base_%s_%s.index" % (genome, seed, read_length) + command = shlex.split("PerM %s %s --readFormat fastq --seed %s -m -s %s" % (local_ref, read_length, seed, index)) + result = subprocess.call( command ) + if result == 0: + self.locations[ 'nt' ].append( [ key, desc, index ] ) + else: + return False + os.remove( local_ref ) + os.makedirs( 'cs' ) + with WithChDir( 'cs' ): + if not os.path.exists( local_ref ): + os.symlink( os.path.relpath( self.fapath ), self.fafile ) + for seed in [ 'F3', 'F4' ]: + key = '%s_%s_%s' % (genome, seed, read_length) + desc = '%s: seed=%s, read length=%s' % (genome, seed, read_length) + index = "%s_color_%s_%s.index" % (genome, seed, read_length) + command = shlex.split("PerM %s %s --readFormat csfastq --seed %s -m -s %s" % (local_ref, read_length, seed, index)) + result = subprocess.call( command, stderr=self.logfile, stdout=self.logfile ) + if result == 0: + self.locations[ 'cs' ].append( [ key, desc, index ] ) + else: + return False + os.remove( local_ref ) + temptar = tarfile.open( 'cs.tar', 'w' ) + temptar.add( 'cs' ) + temptar.close() + shutil.rmtree( 'cs' ) + return True + + def _bfast( self ): + """Indexes bfast in color and nucleotide space for longer reads. + + This preps for 40+bp sized reads, which is bfast's strength. + """ + dir_name_nt = 'nt' + dir_name_cs = 'cs' + window_size = 14 + bfast_nt_masks = [ + "1111111111111111111111", + "1111101110111010100101011011111", + "1011110101101001011000011010001111111", + "10111001101001100100111101010001011111", + "11111011011101111011111111", + "111111100101001000101111101110111", + "11110101110010100010101101010111111", + "111101101011011001100000101101001011101", + "1111011010001000110101100101100110100111", + "1111010010110110101110010110111011", + ] + bfast_color_masks = [ + "1111111111111111111111", + "111110100111110011111111111", + "10111111011001100011111000111111", + "1111111100101111000001100011111011", + "111111110001111110011111111", + "11111011010011000011000110011111111", + "1111111111110011101111111", + "111011000011111111001111011111", + "1110110001011010011100101111101111", + "111111001000110001011100110001100011111", + ] + local_ref = self.fafile + os.makedirs( dir_name_nt ) + os.makedirs( dir_name_cs ) + if not os.path.exists( self.fafile ): + os.symlink( os.path.relpath( self.fapath ), self.fafile ) + with WithChDir( dir_name_nt ): + if not os.path.exists( self.fafile ): + os.symlink( os.path.relpath( self.fapath ), self.fafile ) + # nucleotide space + command = shlex.split( "bfast fasta2brg -f %s -A 0" % local_ref ) + result = subprocess.call( command, stderr=self.logfile ) + for i, mask in enumerate( bfast_nt_masks ): + command = shlex.split("bfast index -d 1 -n 4 -f %s -A 0 -m %s -w %s -i %s" % + ( local_ref, mask, window_size, i + 1 ) ) + result = subprocess.call( command, stderr=self.logfile, stdout=self.logfile ) + os.remove( self.fafile ) + if result != 0: + return False + else: + os.remove( self.fafile ) + with WithChDir( dir_name_cs ): + if not os.path.exists( self.fafile ): + os.symlink( os.path.relpath( self.fapath ), self.fafile ) + # colorspace + command = shlex.split( "bfast fasta2brg -f %s -A 1" % local_ref ) + result = subprocess.call( command, stderr=self.logfile, stdout=self.logfile ) + for i, mask in enumerate( bfast_color_masks ): + command = shlex.split( "bfast index -d 1 -n 4 -f %s -A 1 -m %s -w %s -i %s" % + ( local_ref, mask, window_size, i + 1 ) ) + result = subprocess.call( command, stderr=self.logfile, stdout=self.logfile ) + if result != 0: + return False + else: + os.remove( self.fafile ) + self.locations = None + return True + + def _picard( self ): + local_ref = self.fafile + srma = '/Users/dave/srma.jar' + genome = os.path.splitext( self.fafile )[0] + if not os.path.exists( self.fafile ): + os.symlink( os.path.relpath( self.fapath ), self.fafile ) + command = shlex.split( 'samtools faidx %s' % self.fafile ) + subprocess.call( command, stderr=self.logfile ) + os.rename( '%s.fai' % self.fafile, '%s.fai' % genome ) + command = shlex.split( "java -cp %s net.sf.picard.sam.CreateSequenceDictionary R=%s O=%s/%s.dict URI=%s" \ + % ( srma, local_ref, os.curdir, genome, local_ref ) ) + result = subprocess.call( command, stderr=self.logfile, stdout=self.logfile ) + if result != 0: + return False + else: + self.locations[ 'nt' ].append( self.fafile ) + #os.remove( '%s.fai' % genome ) + os.remove( self.fafile ) + return True + + def _sam( self ): + local_ref = self.fafile + local_file = os.path.splitext( self.fafile )[ 0 ] + if not os.path.exists( local_ref ): + os.symlink( os.path.relpath( self.fapath ), self.fafile ) + command = shlex.split("samtools faidx %s" % local_ref) + result = subprocess.call( command, stderr=self.logfile ) + if result != 0: + return False + else: + self.locations[ 'nt' ].append( local_ref ) + os.remove( local_ref ) + return True + +class WithChDir(): + def __init__( self, target ): + self.working = target + self.previous = os.getcwd() + def __enter__( self ): + os.chdir( self.working ) + def __exit__( self, *args ): + os.chdir( self.previous ) + + +if __name__ == "__main__": + # Parse command line. + parser = optparse.OptionParser() + (options, args) = parser.parse_args() + indexer, infile, outfile, working_dir = args + + # Create archive. + idxobj = ManagedIndexer( outfile, infile, working_dir ) + idxobj.run_indexer( indexer ) + \ No newline at end of file diff --git a/lib/galaxy/tools/imp_exp/__init__.py b/lib/galaxy/tools/imp_exp/__init__.py index 5c3f43484b5..c38e2cecbd9 100644 --- a/lib/galaxy/tools/imp_exp/__init__.py +++ b/lib/galaxy/tools/imp_exp/__init__.py @@ -1,5 +1,6 @@ import os, shutil, logging, tempfile, simplejson from galaxy import model +from galaxy.tools.parameters.basic import UnvalidatedValue from galaxy.web.framework.helpers import to_unicode from galaxy.model.item_attrs import UsesAnnotations from galaxy.util.json import * @@ -324,6 +325,8 @@ class JobExportHistoryArchiveWrapper( object, UsesHistory, UsesAnnotations ): "annotation" : to_unicode( getattr( obj, 'annotation', '' ) ), "tags" : get_item_tag_dict( obj ), } + if isinstance( obj, UnvalidatedValue ): + return obj.__str__() return simplejson.JSONEncoder.default( self, obj ) # @@ -415,8 +418,13 @@ class JobExportHistoryArchiveWrapper( object, UsesHistory, UsesAnnotations ): params_dict[ name ] = value job_attrs[ 'params' ] = params_dict - # Get input, output datasets. - input_datasets = [ assoc.dataset.hid for assoc in job.input_datasets ] + # -- Get input, output datasets. -- + + input_datasets = [] + for assoc in job.input_datasets: + # Optional data inputs will not have a dataset. + if assoc.dataset: + input_datasets.append( assoc.dataset.hid ) job_attrs[ 'input_datasets' ] = input_datasets output_datasets = [ assoc.dataset.hid for assoc in job.output_datasets ] job_attrs[ 'output_datasets' ] = output_datasets diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 1ff75d9b4b0..4eff947c9da 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1056,26 +1056,35 @@ class DrillDownSelectToolParameter( SelectToolParameter ): ... ... ''' ) ) >>> print p.get_html() -
    -
  • [+]Heading 1 -
      -
    • Option 1 -
      • -
    • Option 2 -
      • -
    • [+]Heading 1 -
        -
      • Option 3 -
        • -
      • Option 4 -
        • -
      -
      • -
    -
    • -
  • Option 5 -
    • -
+
+
+ + Heading 1 +
+
+ Option 1 +
+
+ Option 2 +
+
+ + Heading 1 +
+
+ Option 3 +
+
+ Option 4 +
+
+
+
+
+
+ Option 5 +
+
>>> p = DrillDownSelectToolParameter( None, XML( ... ''' ... @@ -1093,26 +1102,35 @@ class DrillDownSelectToolParameter( SelectToolParameter ): ... ... ''' ) ) >>> print p.get_html() -
    -
  • [+]Heading 1 -
      -
    • Option 1 -
      • -
    • Option 2 -
      • -
    • [+]Heading 1 -
        -
      • Option 3 -
        • -
      • Option 4 -
        • -
      -
      • -
    -
    • -
  • Option 5 -
    • -
+
+
+ + Heading 1 +
+
+ Option 1 +
+
+ Option 2 +
+
+ + Heading 1 +
+
+ Option 3 +
+
+ Option 4 +
+
+
+
+
+
+ Option 5 +
+
>>> print p.options [{'selected': False, 'name': 'Heading 1', 'value': 'heading1', 'options': [{'selected': False, 'name': 'Option 1', 'value': 'option1', 'options': []}, {'selected': False, 'name': 'Option 2', 'value': 'option2', 'options': []}, {'selected': False, 'name': 'Heading 1', 'value': 'heading1', 'options': [{'selected': False, 'name': 'Option 3', 'value': 'option3', 'options': []}, {'selected': False, 'name': 'Option 4', 'value': 'option4', 'options': []}]}]}, {'selected': False, 'name': 'Option 5', 'value': 'option5', 'options': []}] """ @@ -1133,7 +1151,9 @@ class DrillDownSelectToolParameter( SelectToolParameter ): from_file = os.path.join( tool.app.config.tool_data_path, from_file ) elem = XML( "%s" % open( from_file ).read() ) self.is_dynamic = False - self.dynamic_options = None #backwards compatibility with SelectToolParameter's old dynamic options and late validation + self.dynamic_options = elem.get( 'dynamic_options' , None ) + if self.dynamic_options: + self.is_dynamic = True self.options = [] self.filtered = {} if elem.find( 'filter' ): @@ -1148,12 +1168,23 @@ class DrillDownSelectToolParameter( SelectToolParameter ): if filter.get( 'value' ) not in self.filtered[filter.get( 'data_ref' )][filter.get( 'meta_key' )]: self.filtered[filter.get( 'data_ref' )][filter.get( 'meta_key' )][filter.get( 'value' )] = [] recurse_option_elems( self.filtered[filter.get( 'data_ref' )][filter.get( 'meta_key' )][filter.get( 'value' )], filter.find( 'options' ).findall( 'option' ) ) - else: + elif not self.dynamic_options: recurse_option_elems( self.options, elem.find( 'options' ).findall( 'option' ) ) + def _get_options_from_code( self, trans=None, value=None, other_values=None ): + assert self.dynamic_options, Exception( "dynamic_options was not specifed" ) + call_other_values = { '__trans__': trans, '__value__': value } + if other_values: + call_other_values.update( other_values.dict ) + return eval( self.dynamic_options, self.tool.code_namespace, call_other_values ) + + def get_options( self, trans=None, value=None, other_values={} ): if self.is_dynamic: - options = [] + if self.dynamic_options: + options = self._get_options_from_code( trans=trans, value=value, other_values=other_values ) + else: + options = [] for filter_key, filter_value in self.filtered.iteritems(): dataset = other_values[filter_key] if dataset.__class__.__name__.endswith( "DatasetFilenameWrapper" ): #this is a bad way to check for this, but problems importing class ( due to circular imports? ) diff --git a/lib/galaxy/util/inflection.py b/lib/galaxy/util/inflection.py index 2d73ac63057..67342b05593 100644 --- a/lib/galaxy/util/inflection.py +++ b/lib/galaxy/util/inflection.py @@ -14,7 +14,7 @@ class Base: def cond_plural(self, number_of_records, word) : '''Returns the plural form of a word if first parameter is greater than 1''' - if number_of_records > 1 : + if number_of_records != 1: return self.pluralize(word) else : return word @@ -385,4 +385,4 @@ class Inflector: # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THIS SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THIS SOFTWARE. \ No newline at end of file +# THIS SOFTWARE. diff --git a/lib/galaxy/util/odict.py b/lib/galaxy/util/odict.py index 94c29624aa2..005996d3684 100644 --- a/lib/galaxy/util/odict.py +++ b/lib/galaxy/util/odict.py @@ -11,23 +11,22 @@ class odict(UserDict): This dictionary class extends UserDict to record the order in which items are added. Calling keys(), values(), items(), etc. will return results in this order. - - I've added iterkeys, itervalues, iteritems """ - def __init__(self, dict = None): + def __init__( self, dict = None ): self._keys = [] - UserDict.__init__(self, dict) + UserDict.__init__( self, dict ) - def __delitem__(self, key): - UserDict.__delitem__(self, key) - self._keys.remove(key) + def __delitem__( self, key ): + UserDict.__delitem__( self, key ) + self._keys.remove( key ) - def __setitem__(self, key, item): - UserDict.__setitem__(self, key, item) - if key not in self._keys: self._keys.append(key) + def __setitem__( self, key, item ): + UserDict.__setitem__( self, key, item ) + if key not in self._keys: + self._keys.append( key ) - def clear(self): - UserDict.clear(self) + def clear( self ): + UserDict.clear( self ) self._keys = [] def copy(self): @@ -35,49 +34,43 @@ class odict(UserDict): new.update( self ) return new - def items(self): - return zip(self._keys, self.values()) + def items( self ): + return zip( self._keys, self.values() ) - def keys(self): + def keys( self ): return self._keys[:] - def popitem(self): + def popitem( self ): try: key = self._keys[-1] except IndexError: - raise KeyError('dictionary is empty') + raise KeyError( 'dictionary is empty' ) + val = self[ key ] + del self[ key ] + return ( key, val ) - val = self[key] - del self[key] + def setdefault( self, key, failobj=None ): + if key not in self._keys: + self._keys.append( key ) + return UserDict.setdefault( self, key, failobj ) - return (key, val) + def update( self, dict ): + for ( key, val ) in dict.items(): + self.__setitem__( key, val ) - def setdefault(self, key, failobj = None): - if key not in self._keys: self._keys.append(key) - return UserDict.setdefault(self, key, failobj) + def values( self ): + return map( self.get, self._keys ) - def update(self, dict): - UserDict.update(self, dict) - for key in dict.keys(): - if key not in self._keys: self._keys.append(key) - - def update(self, dict): - for (key,val) in dict.items(): - self.__setitem__(key,val) - - def values(self): - return map(self.get, self._keys) - - def iterkeys(self): + def iterkeys( self ): return iter( self._keys ) - def itervalues(self): + def itervalues( self ): for key in self._keys: - yield self.get(key) + yield self.get( key ) - def iteritems(self): + def iteritems( self ): for key in self._keys: - yield key, self.get(key) + yield key, self.get( key ) def __iter__( self ): for key in self._keys: @@ -86,3 +79,7 @@ class odict(UserDict): def reverse( self ): self._keys.reverse() + def insert( self, index, key, item ): + if key not in self._keys: + self._keys.insert( index, key ) + UserDict.__setitem__( self, key, item ) diff --git a/lib/galaxy/util/shed_util.py b/lib/galaxy/util/shed_util.py index 0ee1074bed3..d1b852d8f87 100644 --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -1,18 +1,107 @@ -import os, tempfile, shutil, subprocess, logging +import sys, os, tempfile, shutil, subprocess, logging, string, urllib2 from datetime import date, datetime, timedelta -from time import strftime +from time import strftime, gmtime from galaxy import util from galaxy.datatypes.checkers import * from galaxy.util.json import * from galaxy.tools.search import ToolBoxSearch from galaxy.model.orm import * +from galaxy import eggs +import pkg_resources + +pkg_resources.require( 'mercurial' ) +from mercurial import ui, commands + pkg_resources.require( 'elementtree' ) from elementtree import ElementTree, ElementInclude from elementtree.ElementTree import Element, SubElement log = logging.getLogger( __name__ ) +# Characters that must be html escaped +MAPPED_CHARS = { '>' :'>', + '<' :'<', + '"' : '"', + '&' : '&', + '\'' : ''' } +VALID_CHARS = set( string.letters + string.digits + "'\"-=_.()/+*^,:?!#[]%\\$@;{}" ) + +class ShedCounter( object ): + def __init__( self, model ): + self.model = model + self.generation_time = strftime( "%b %d, %Y", gmtime() ) + self.repositories = 0 + self.new_repositories = 0 + self.deleted_repositories = 0 + self.invalid_tools = 0 + self.valid_tools = 0 + self.workflows = 0 + self.proprietary_datatypes = 0 + self.total_clones = 0 + self.generate_statistics() + @property + def sa_session( self ): + """Returns a SQLAlchemy session""" + return self.model.context + def generate_statistics( self ): + self.repositories = 0 + self.new_repositories = 0 + self.deleted_repositories = 0 + self.invalid_tools = 0 + self.valid_tools = 0 + self.workflows = 0 + self.proprietary_datatypes = 0 + self.total_clones = 0 + for repository in self.sa_session.query( self.model.Repository ): + self.repositories += 1 + self.total_clones += repository.times_downloaded + is_deleted = repository.deleted + is_new = repository.is_new + if is_deleted and is_new: + self.deleted_repositories += 1 + self.new_repositories += 1 + elif is_deleted: + self.deleted_repositories += 1 + elif is_new: + self.new_repositories += 1 + else: + processed_guids = [] + processed_invalid_tool_configs = [] + processed_relative_workflow_paths = [] + processed_datatypes = [] + for downloadable_revision in repository.downloadable_revisions: + metadata = downloadable_revision.metadata + if 'tools' in metadata: + tool_dicts = metadata[ 'tools' ] + for tool_dict in tool_dicts: + if 'guid' in tool_dict: + guid = tool_dict[ 'guid' ] + if guid not in processed_guids: + self.valid_tools += 1 + processed_guids.append( guid ) + if 'invalid_tools' in metadata: + invalid_tool_configs = metadata[ 'invalid_tools' ] + for invalid_tool_config in invalid_tool_configs: + if invalid_tool_config not in processed_invalid_tool_configs: + self.invalid_tools += 1 + processed_invalid_tool_configs.append( invalid_tool_config ) + if 'datatypes' in metadata: + datatypes = metadata[ 'datatypes' ] + for datatypes_dict in datatypes: + if 'extension' in datatypes_dict: + extension = datatypes_dict[ 'extension' ] + if extension not in processed_datatypes: + self.proprietary_datatypes += 1 + processed_datatypes.append( extension ) + if 'workflows' in metadata: + workflows = metadata[ 'workflows' ] + for workflow_tup in workflows: + relative_path, exported_workflow_dict = workflow_tup + if relative_path not in processed_relative_workflow_paths: + self.workflows += 1 + processed_relative_workflow_paths.append( relative_path ) + self.generation_time = strftime( "%b %d, %Y", gmtime() ) def add_to_shed_tool_config( app, shed_tool_conf_dict, elem_list ): # A tool shed repository is being installed so change the shed_tool_conf file. Parse the config file to generate the entire list # of config_elems instead of using the in-memory list since it will be a subset of the entire list if one or more repositories have @@ -129,13 +218,15 @@ def alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_ # The value of proprietary_path must be an absolute path due to job_working_directory. elem.attrib[ 'proprietary_path' ] = os.path.abspath( relative_head ) elem.attrib[ 'proprietary_datatype_module' ] = proprietary_datatype_module - sniffers = datatypes_config_root.find( 'sniffers' ) + else: + sniffers = None fd, proprietary_datatypes_config = tempfile.mkstemp() os.write( fd, '\n' ) os.write( fd, '\n' ) os.write( fd, '%s' % util.xml_to_string( registration ) ) - os.write( fd, '%s' % util.xml_to_string( sniffers ) ) + if sniffers: + os.write( fd, '%s' % util.xml_to_string( sniffers ) ) os.write( fd, '\n' ) os.close( fd ) os.chmod( proprietary_datatypes_config, 0644 ) @@ -179,31 +270,38 @@ def clean_tool_shed_url( tool_shed_url ): # Eliminate the port, if any, since it will result in an invalid directory name. return tool_shed_url.split( ':' )[ 0 ] return tool_shed_url.rstrip( '/' ) -def clone_repository( name, clone_dir, current_working_dir, repository_clone_url ): - log.debug( "Installing repository '%s'" % name ) - if not os.path.exists( clone_dir ): - os.makedirs( clone_dir ) - log.debug( 'Cloning %s' % repository_clone_url ) - cmd = 'hg clone %s' % repository_clone_url - tmp_name = tempfile.NamedTemporaryFile().name - tmp_stderr = open( tmp_name, 'wb' ) - os.chdir( clone_dir ) - proc = subprocess.Popen( args=cmd, shell=True, stderr=tmp_stderr.fileno() ) - returncode = proc.wait() - os.chdir( current_working_dir ) - tmp_stderr.close() - return returncode, tmp_name -def copy_sample_loc_file( app, filename ): - """Copy xxx.loc.sample to ~/tool-data/xxx.loc.sample and ~/tool-data/xxx.loc""" - head, sample_loc_file = os.path.split( filename ) - loc_file = sample_loc_file.replace( '.sample', '' ) - tool_data_path = os.path.abspath( app.config.tool_data_path ) +def clone_repository( repository_clone_url, repository_file_dir, ctx_rev ): + """ + Clone the repository up to the specified changeset_revision. No subsequent revisions will be present in the cloned repository. + """ + commands.clone( get_configured_ui(), + repository_clone_url, + dest=repository_file_dir, + pull=True, + noupdate=False, + rev=[ ctx_rev ] ) +def copy_sample_file( app, filename, dest_path=None ): + """ + Copy xxx.loc.sample to dest_path/xxx.loc.sample and dest_path/xxx.loc. The default value for dest_path is ~/tool-data. + """ + if dest_path is None: + dest_path = os.path.abspath( app.config.tool_data_path ) + sample_file_path, sample_file_name = os.path.split( filename ) + copied_file = sample_file_name.replace( '.sample', '' ) # It's ok to overwrite the .sample version of the file. - shutil.copy( os.path.abspath( filename ), os.path.join( tool_data_path, sample_loc_file ) ) - # Only create the .loc file if it does not yet exist. We don't - # overwrite it in case it contains stuff proprietary to the local instance. - if not os.path.exists( os.path.join( tool_data_path, loc_file ) ): - shutil.copy( os.path.abspath( filename ), os.path.join( tool_data_path, loc_file ) ) + shutil.copy( os.path.abspath( filename ), os.path.join( dest_path, sample_file_name ) ) + # Only create the .loc file if it does not yet exist. We don't overwrite it in case it contains stuff proprietary to the local instance. + if not os.path.exists( os.path.join( dest_path, copied_file ) ): + shutil.copy( os.path.abspath( filename ), os.path.join( dest_path, copied_file ) ) +def copy_sample_files( app, sample_files, sample_files_copied=None, dest_path=None ): + """ + Copy all files to dest_path in the local Galaxy environment that have not already been copied. Those that have been copied + are contained in sample_files_copied. The default value for dest_path is ~/tool-data. + """ + sample_files_copied = util.listify( sample_files_copied ) + for filename in sample_files: + if filename not in sample_files_copied: + copy_sample_file( app, filename, dest_path=dest_path ) def create_repository_dict_for_proprietary_datatypes( tool_shed, name, owner, installed_changeset_revision, tool_dicts, converter_path=None, display_path=None ): return dict( tool_shed=tool_shed, repository_name=name, @@ -212,7 +310,8 @@ def create_repository_dict_for_proprietary_datatypes( tool_shed, name, owner, in tool_dicts=tool_dicts, converter_path=converter_path, display_path=display_path ) -def create_or_update_tool_shed_repository( app, name, description, changeset_revision, repository_clone_url, metadata_dict, owner='', dist_to_shed=False ): +def create_or_update_tool_shed_repository( app, name, description, changeset_revision, ctx_rev, repository_clone_url, metadata_dict, + owner='', dist_to_shed=False ): # The received value for dist_to_shed will be True if the InstallManager is installing a repository that contains tools or datatypes that used # to be in the Galaxy distribution, but have been moved to the main Galaxy tool shed. sa_session = app.model.context.current @@ -225,6 +324,7 @@ def create_or_update_tool_shed_repository( app, name, description, changeset_rev if tool_shed_repository: tool_shed_repository.description = description tool_shed_repository.changeset_revision = changeset_revision + tool_shed_repository.ctx_rev = ctx_rev tool_shed_repository.metadata = metadata_dict tool_shed_repository.includes_datatypes = includes_datatypes tool_shed_repository.deleted = False @@ -236,6 +336,7 @@ def create_or_update_tool_shed_repository( app, name, description, changeset_rev owner=owner, installed_changeset_revision=changeset_revision, changeset_revision=changeset_revision, + ctx_rev=ctx_rev, metadata=metadata_dict, includes_datatypes=includes_datatypes, dist_to_shed=dist_to_shed ) @@ -247,7 +348,7 @@ def generate_clone_url( trans, repository ): tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) return '%s/repos/%s/%s' % ( tool_shed_url, repository.owner, repository.name ) def generate_datatypes_metadata( datatypes_config, metadata_dict ): - """Update the received metadata_dict with changes that have been applied to the received datatypes_config.""" + """Update the received metadata_dict with information from the parsed datatypes_config.""" tree = ElementTree.parse( datatypes_config ) root = tree.getroot() ElementInclude.include( root ) @@ -325,12 +426,21 @@ def generate_metadata( toolbox, relative_install_dir, repository_clone_url ): if not ( check_binary( full_path ) or check_image( full_path ) or check_gzip( full_path )[ 0 ] or check_bz2( full_path )[ 0 ] or check_zip( full_path ) ): try: - tool = toolbox.load_tool( full_path ) + # Make sure we're looking at a tool config and not a display application config or something else. + element_tree = util.parse_xml( full_path ) + element_tree_root = element_tree.getroot() + is_tool = element_tree_root.tag == 'tool' except Exception, e: - tool = None - if tool is not None: - tool_config = os.path.join( root, name ) - metadata_dict = generate_tool_metadata( tool_config, tool, repository_clone_url, metadata_dict ) + log.debug( "Error parsing %s, exception: %s" % ( full_path, str( e ) ) ) + is_tool = False + if is_tool: + try: + tool = toolbox.load_tool( full_path ) + except Exception, e: + tool = None + if tool is not None: + tool_config = os.path.join( root, name ) + metadata_dict = generate_tool_metadata( tool_config, tool, repository_clone_url, metadata_dict ) # Find all exported workflows elif name.endswith( '.ga' ): relative_path = os.path.join( root, name ) @@ -356,30 +466,30 @@ def generate_tool_metadata( tool_config, tool, repository_clone_url, metadata_di # Handle tool.requirements. tool_requirements = [] for tr in tool.requirements: - name=tr.name - type=tr.type - if type == 'fabfile': - version = None - fabfile = tr.fabfile - method = tr.method - else: - version = tr.version - fabfile = None - method = None - requirement_dict = dict( name=name, - type=type, - version=version, - fabfile=fabfile, - method=method ) + requirement_dict = dict( name=tr.name, + type=tr.type, + version=tr.version ) tool_requirements.append( requirement_dict ) # Handle tool.tests. tool_tests = [] if tool.tests: for ttb in tool.tests: + required_files = [] + for required_file in ttb.required_files: + value, extra = required_file + required_files.append( ( value ) ) + inputs = [] + for input in ttb.inputs: + name, value, extra = input + inputs.append( ( name, value ) ) + outputs = [] + for output in ttb.outputs: + name, file_name, extra = output + outputs.append( ( name, os.path.split( file_name )[ 1 ] ) ) test_dict = dict( name=ttb.name, - required_files=ttb.required_files, - inputs=ttb.inputs, - outputs=ttb.outputs ) + required_files=required_files, + inputs=inputs, + outputs=outputs ) tool_tests.append( test_dict ) tool_dict = dict( id=tool.id, guid=guid, @@ -570,6 +680,15 @@ def generate_workflow_metadata( relative_path, exported_workflow_dict, metadata_ else: metadata_dict[ 'workflows' ] = [ ( relative_path, exported_workflow_dict ) ] return metadata_dict +def get_configured_ui(): + # Configure any desired ui settings. + _ui = ui.ui() + # The following will suppress all messages. This is + # the same as adding the following setting to the repo + # hgrc file' [ui] section: + # quiet = True + _ui.setconfig( 'ui', 'quiet', True ) + return _ui def get_converter_and_display_paths( registration_elem, relative_install_dir ): """Find the relative path to data type converters and display applications included in installed tool shed repositories.""" converter_path = None @@ -612,6 +731,12 @@ def get_converter_and_display_paths( registration_elem, relative_install_dir ): if converter_path and display_path: break return converter_path, display_path +def get_ctx_rev( tool_shed_url, name, owner, changeset_revision ): + url = '%s/repository/get_ctx_rev?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy&no_reset=true' % ( tool_shed_url, name, owner, changeset_revision ) + response = urllib2.urlopen( url ) + ctx_rev = response.read() + response.close() + return ctx_rev def get_shed_tool_conf_dict( app, shed_tool_conf ): """ Return the in-memory version of the shed_tool_conf file, which is stored in the config_elems entry @@ -768,27 +893,27 @@ def handle_missing_data_table_entry( app, tool_path, sample_files, repository_to return repository_tools_tups def handle_missing_index_file( app, tool_path, sample_files, repository_tools_tups ): """Inspect each tool to see if it has any input parameters that are dynamically generated select lists that depend on a .loc file.""" - missing_files_handled = [] + sample_files_copied = [] for index, repository_tools_tup in enumerate( repository_tools_tups ): tup_path, guid, repository_tool = repository_tools_tup params_with_missing_index_file = repository_tool.params_with_missing_index_file for param in params_with_missing_index_file: options = param.options - missing_head, missing_tail = os.path.split( options.missing_index_file ) - if missing_tail not in missing_files_handled: + missing_file_path, missing_file_name = os.path.split( options.missing_index_file ) + if missing_file_name not in sample_files_copied: # The repository must contain the required xxx.loc.sample file. for sample_file in sample_files: - sample_head, sample_tail = os.path.split( sample_file ) - if sample_tail == '%s.sample' % missing_tail: - copy_sample_loc_file( app, sample_file ) + sample_file_path, sample_file_name = os.path.split( sample_file ) + if sample_file_name == '%s.sample' % missing_file_name: + copy_sample_file( app, sample_file ) if options.tool_data_table and options.tool_data_table.missing_index_file: options.tool_data_table.handle_found_index_file( options.missing_index_file ) - missing_files_handled.append( missing_tail ) + sample_files_copied.append( options.missing_index_file ) break # Reload the tool into the local list of repository_tools_tups. repository_tool = app.toolbox.load_tool( os.path.join( tool_path, tup_path ), guid=guid ) repository_tools_tups[ index ] = ( tup_path, guid, repository_tool ) - return repository_tools_tups + return repository_tools_tups, sample_files_copied def handle_sample_tool_data_table_conf_file( app, filename ): """ Parse the incoming filename and add new entries to the in-memory @@ -912,8 +1037,9 @@ def load_datatype_items( app, repository, relative_install_dir, deactivate=False if display_path: # Load or deactivate proprietary datatype display applications app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict, deactivate=deactivate ) -def load_repository_contents( trans, repository_name, description, owner, changeset_revision, tool_path, repository_clone_url, - relative_install_dir, current_working_dir, tmp_name, tool_shed=None, tool_section=None, shed_tool_conf=None ): +def load_repository_contents( trans, repository_name, description, owner, changeset_revision, ctx_rev, tool_path, repository_clone_url, + relative_install_dir, current_working_dir, tool_shed=None, tool_section=None, shed_tool_conf=None, + install_tool_dependencies=False ): """Generate the metadata for the installed tool shed repository, among other things.""" # It is critical that the installed repository is updated to the desired changeset_revision before metadata is set because the # process for setting metadata uses the repository files on disk. This method is called when an admin is installing a new repository @@ -926,6 +1052,7 @@ def load_repository_contents( trans, repository_name, description, owner, change repository_name, description, changeset_revision, + ctx_rev, repository_clone_url, metadata_dict, dist_to_shed=False ) @@ -937,8 +1064,9 @@ def load_repository_contents( trans, repository_name, description, owner, change # Handle missing data table entries for tool parameters that are dynamically generated select lists. repository_tools_tups = handle_missing_data_table_entry( trans.app, tool_path, sample_files, repository_tools_tups ) # Handle missing index files for tool parameters that are dynamically generated select lists. - repository_tools_tups = handle_missing_index_file( trans.app, tool_path, sample_files, repository_tools_tups ) - # Handle tools that use fabric scripts to install dependencies. + repository_tools_tups, sample_files_copied = handle_missing_index_file( trans.app, tool_path, sample_files, repository_tools_tups ) + # Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance. + copy_sample_files( trans.app, sample_files, sample_files_copied=sample_files_copied ) handle_tool_dependencies( current_working_dir, relative_install_dir, repository_tools_tups ) add_to_tool_panel( app=trans.app, repository_name=repository_name, @@ -949,11 +1077,6 @@ def load_repository_contents( trans, repository_name, description, owner, change shed_tool_conf=shed_tool_conf, tool_panel_dict=tool_panel_dict, new_install=True ) - # Remove the temporary file - try: - os.unlink( tmp_name ) - except: - pass if 'datatypes_config' in metadata_dict: datatypes_config = os.path.abspath( metadata_dict[ 'datatypes_config' ] ) # Load data types required by tools. @@ -987,18 +1110,12 @@ def panel_entry_per_tool( tool_section_dict ): if k not in [ 'id', 'version', 'name' ]: return True return False -def pull_repository( current_working_dir, repo_files_dir, name ): - # Pull the latest possible contents to the repository. - log.debug( "Pulling latest updates to the repository named '%s'" % name ) - cmd = 'hg pull' - tmp_name = tempfile.NamedTemporaryFile().name - tmp_stderr = open( tmp_name, 'wb' ) - os.chdir( repo_files_dir ) - proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() ) - returncode = proc.wait() - os.chdir( current_working_dir ) - tmp_stderr.close() - return returncode, tmp_name +def pull_repository( repo, repository_clone_url, ctx_rev ): + """Pull changes from a remote repository to a local one.""" + commands.pull( get_configured_ui(), + repo, + source=repository_clone_url, + rev=ctx_rev ) def remove_from_shed_tool_config( trans, shed_tool_conf_dict, guids_to_remove ): # A tool shed repository is being uninstalled so change the shed_tool_conf file. Parse the config file to generate the entire list # of config_elems instead of using the in-memory list since it will be a subset of the entire list if one or more repositories have @@ -1096,17 +1213,50 @@ def remove_from_tool_panel( trans, repository, shed_tool_conf, uninstall ): if uninstall: # Write the current in-memory version of the integrated_tool_panel.xml file to disk. trans.app.toolbox.write_integrated_tool_panel_config_file() -def update_repository( current_working_dir, repo_files_dir, changeset_revision ): - # Update the cloned repository to changeset_revision. It is imperative that the - # installed repository is updated to the desired changeset_revision before metadata - # is set because the process for setting metadata uses the repository files on disk. - log.debug( 'Updating cloned repository to revision "%s"' % changeset_revision ) - cmd = 'hg update -r %s' % changeset_revision - tmp_name = tempfile.NamedTemporaryFile().name - tmp_stderr = open( tmp_name, 'wb' ) - os.chdir( repo_files_dir ) - proc = subprocess.Popen( cmd, shell=True, stderr=tmp_stderr.fileno() ) - returncode = proc.wait() - os.chdir( current_working_dir ) - tmp_stderr.close() - return returncode, tmp_name +def to_html_escaped( text ): + """Translates the characters in text to html values""" + translated = [] + for c in text: + if c in [ '\r\n', '\n', ' ', '\t' ] or c in VALID_CHARS: + translated.append( c ) + elif c in MAPPED_CHARS: + translated.append( MAPPED_CHARS[ c ] ) + else: + translated.append( 'X' ) + return ''.join( translated ) +def to_html_str( text ): + """Translates the characters in text to sn html string""" + translated = [] + for c in text: + if c in VALID_CHARS: + translated.append( c ) + elif c in MAPPED_CHARS: + translated.append( MAPPED_CHARS[ c ] ) + elif c == ' ': + translated.append( ' ' ) + elif c == '\t': + translated.append( '    ' ) + elif c == '\n': + translated.append( '
' ) + elif c not in [ '\r' ]: + translated.append( 'X' ) + return ''.join( translated ) +def update_repository( repo, ctx_rev=None ): + """ + Update the cloned repository to changeset_revision. It is critical that the installed repository is updated to the desired + changeset_revision before metadata is set because the process for setting metadata uses the repository files on disk. + """ + # TODO: We may have files on disk in the repo directory that aren't being tracked, so they must be removed. + # The codes used to show the status of files are as follows. + # M = modified + # A = added + # R = removed + # C = clean + # ! = deleted, but still tracked + # ? = not tracked + # I = ignored + # It would be nice if we could use mercurial's purge extension to remove untracked files. The problem is that + # purging is not supported by the mercurial API. See the deprecated update_for_browsing() method in common.py. + commands.update( get_configured_ui(), + repo, + rev=ctx_rev ) diff --git a/lib/galaxy/visualization/tracks/data_providers.py b/lib/galaxy/visualization/tracks/data_providers.py index 79c5908a0ad..5a5fb7b5630 100644 --- a/lib/galaxy/visualization/tracks/data_providers.py +++ b/lib/galaxy/visualization/tracks/data_providers.py @@ -19,7 +19,7 @@ from galaxy.util.lrucache import LRUCache from galaxy.visualization.tracks.summary import * import galaxy_utils.sequence.vcf from galaxy.datatypes.tabular import Vcf -from galaxy.datatypes.interval import Bed, Gff, Gtf, ENCODEPeak +from galaxy.datatypes.interval import Interval, Bed, Gff, Gtf, ENCODEPeak from pysam import csamtools, ctabix @@ -67,8 +67,7 @@ class TracksDataProvider( object ): """ Write data in region defined by chrom, start, and end to a file. """ - # Override. - pass + raise Exception( "Unimplemented Function" ) def valid_chroms( self ): """ @@ -81,23 +80,19 @@ class TracksDataProvider( object ): Returns true if dataset has data in the specified genome window, false otherwise. """ - # Override. - pass + raise Exception( "Unimplemented Function" ) def get_iterator( self, chrom, start, end ): """ Returns an iterator that provides data in the region chrom:start-end """ - # Override. - pass + raise Exception( "Unimplemented Function" ) def process_data( self, iterator, start_val=0, max_vals=None, **kwargs ): """ Process data from an iterator to a format that can be provided to client. """ - # Override. - pass - + raise Exception( "Unimplemented Function" ) def get_data( self, chrom, start, end, start_val=0, max_vals=sys.maxint, **kwargs ): """ @@ -270,18 +265,96 @@ class TabixDataProvider( FilterableMixin, TracksDataProvider ): out.close() # -# -- BED data providers -- +# -- Interval data providers -- # -class BedDataProvider( TracksDataProvider ): +class IntervalDataProvider( TracksDataProvider ): """ - Abstract class that processes BED data from native format to payload format. + Processes BED data from native format to payload format. Payload format: [ uid (offset), start, end, name, strand, thick_start, thick_end, blocks ] """ def get_iterator( self, chrom, start, end ): - raise "Unimplemented Method" + raise Exception( "Unimplemented Function" ) + + def process_data( self, iterator, start_val=0, max_vals=None, **kwargs ): + """ + Provides + """ + # Build data to return. Payload format is: + # [ , , , , ] + # + # First three entries are mandatory, others are optional. + # + filter_cols = from_json_string( kwargs.get( "filter_cols", "[]" ) ) + no_detail = ( "no_detail" in kwargs ) + rval = [] + message = None + # Subtract one b/c columns are 1-based but indices are 0-based. + col_fn = lambda col: None if col is None else col - 1 + start_col = self.original_dataset.metadata.startCol - 1 + end_col = self.original_dataset.metadata.endCol - 1 + strand_col = col_fn( self.original_dataset.metadata.strandCol ) + name_col = col_fn( self.original_dataset.metadata.nameCol ) + for count, line in enumerate( iterator ): + if count < start_val: + continue + if max_vals and count-start_val >= max_vals: + message = ERROR_MAX_VALS % ( max_vals, "features" ) + break + + feature = line.split() + length = len(feature) + # Unique id is just a hash of the line + payload = [ hash(line), int( feature[start_col] ), int( feature [end_col] ) ] + + if no_detail: + rval.append( payload ) + continue + + # Name, strand. + if name_col: + payload.append( feature[name_col] ) + if strand_col: + # Put empty name as placeholder. + if not name_col: payload.append( "" ) + payload.append( feature[strand_col] ) + + # Score (filter data) + if length >= 5 and filter_cols and filter_cols[0] == "Score": + try: + payload.append( float( feature[4] ) ) + except: + payload.append( feature[4] ) + + rval.append( payload ) + + return { 'data': rval, 'message': message } + + def write_data_to_file( self, chrom, start, end, filename ): + raise Exception( "Unimplemented Function" ) + +class IntervalTabixDataProvider( TabixDataProvider, IntervalDataProvider ): + """ + Provides data from a BED file indexed via tabix. + """ + pass + + +# +# -- BED data providers -- +# + +class BedDataProvider( TracksDataProvider ): + """ + Processes BED data from native format to payload format. + + Payload format: [ uid (offset), start, end, name, strand, thick_start, thick_end, blocks ] + """ + + def get_iterator( self, chrom, start, end ): + raise Exception( "Unimplemented Method" ) def process_data( self, iterator, start_val=0, max_vals=None, **kwargs ): """ @@ -359,7 +432,7 @@ class BedTabixDataProvider( TabixDataProvider, BedDataProvider ): Provides data from a BED file indexed via tabix. """ pass - + class RawBedDataProvider( BedDataProvider ): """ Provide data from BED file. @@ -936,9 +1009,9 @@ class IntervalIndexDataProvider( FilterableMixin, TracksDataProvider ): return { 'data': results, 'message': message } -class GFFDataProvider( TracksDataProvider ): +class RawGFFDataProvider( TracksDataProvider ): """ - Provide data from GFF file. + Provide data from GFF file that has not been indexed. NOTE: this data provider does not use indices, and hence will be very slow for large datasets. @@ -951,7 +1024,7 @@ class GFFDataProvider( TracksDataProvider ): """ start, end = int( start ), int( end ) source = open( self.original_dataset.file_name ) - + def features_in_region_iter(): offset = 0 for feature in GFFReaderWrapper( source, fix_strand=True ): @@ -961,7 +1034,7 @@ class GFFDataProvider( TracksDataProvider ): yield feature, offset offset += feature.raw_size return features_in_region_iter() - + def process_data( self, iterator, start_val=0, max_vals=None, **kwargs ): """ Process data from an iterator to a format that can be provided to client. @@ -984,6 +1057,48 @@ class GFFDataProvider( TracksDataProvider ): return { 'data': results, 'message': message } + +class GtfTabixDataProvider( TabixDataProvider ): + """ + Returns data from GTF datasets that are indexed via tabix. + """ + + def process_data( self, iterator, start_val=0, max_vals=None, **kwargs ): + # Loop through lines and group by transcript_id; each group is a feature. + + # TODO: extend this code or use code in gff_util to process GFF/3 as well + # and then create a generic GFFDataProvider that can be used with both + # raw and tabix datasets. + features = {} + for count, line in enumerate( iterator ): + line_attrs = parse_gff_attributes( line.split('\t')[8] ) + transcript_id = line_attrs[ 'transcript_id' ] + if transcript_id in features: + feature = features[ transcript_id ] + else: + feature = [] + features[ transcript_id ] = feature + feature.append( GFFInterval( None, line.split( '\t') ) ) + + # Process data. + filter_cols = from_json_string( kwargs.get( "filter_cols", "[]" ) ) + no_detail = ( "no_detail" in kwargs ) + results = [] + message = None + + for count, intervals in enumerate( features.values() ): + if count < start_val: + continue + if count-start_val >= max_vals: + message = ERROR_MAX_VALS % ( max_vals, "reads" ) + break + + feature = GFFFeature( None, intervals=intervals ) + payload = package_gff_feature( feature, no_detail=no_detail, filter_cols=filter_cols ) + payload.insert( 0, feature.intervals[ 0 ].attributes[ 'transcript_id' ] ) + results.append( payload ) + + return { 'data': results, 'message': message } # # -- ENCODE Peak data providers. @@ -1113,7 +1228,13 @@ class ENCODEPeakTabixDataProvider( TabixDataProvider, ENCODEPeakDataProvider ): # type. First key is converted dataset type; if result is another dict, second key # is original dataset type. TODO: This needs to be more flexible. dataset_type_name_to_data_provider = { - "tabix": { Vcf: VcfTabixDataProvider, Bed: BedTabixDataProvider, ENCODEPeak: ENCODEPeakTabixDataProvider, "default" : TabixDataProvider }, + "tabix": { + Vcf: VcfTabixDataProvider, + Bed: BedTabixDataProvider, + Gtf: GtfTabixDataProvider, + ENCODEPeak: ENCODEPeakTabixDataProvider, + Interval: IntervalTabixDataProvider, + "default" : TabixDataProvider }, "interval_index": IntervalIndexDataProvider, "bai": BamDataProvider, "bam": SamDataProvider, @@ -1172,16 +1293,17 @@ def package_gff_feature( feature, no_detail=False, filter_cols=[] ): feature.end ] - # HACK: remove interval with name 'transcript' from feature. + # HACK: ignore interval with name 'transcript' from feature. # Cufflinks puts this interval in each of its transcripts, # and they mess up trackster by covering the feature's blocks. # This interval will always be a feature's first interval, - # and the GFF's third column is its feature name. + # and the GFF's third column is its feature name. + feature_intervals = feature.intervals if feature.intervals[0].fields[2] == 'transcript': - feature.intervals = feature.intervals[1:] + feature_intervals = feature.intervals[1:] # Add blocks. - block_sizes = [ (interval.end - interval.start ) for interval in feature.intervals ] - block_starts = [ ( interval.start - feature.start ) for interval in feature.intervals ] + block_sizes = [ (interval.end - interval.start ) for interval in feature_intervals ] + block_starts = [ ( interval.start - feature.start ) for interval in feature_intervals ] blocks = zip( block_sizes, block_starts ) payload.append( [ ( feature.start + block[1], feature.start + block[1] + block[0] ) for block in blocks ] ) diff --git a/lib/galaxy/web/api/datasets.py b/lib/galaxy/web/api/datasets.py new file mode 100644 index 00000000000..b11dc19fe77 --- /dev/null +++ b/lib/galaxy/web/api/datasets.py @@ -0,0 +1,44 @@ +""" +API operations on the contents of a dataset. +""" +import logging, os, string, shutil, urllib, re, socket +from cgi import escape, FieldStorage +from galaxy import util, datatypes, jobs, web, util +from galaxy.web.base.controller import * +from galaxy.util.sanitize_html import sanitize_html +from galaxy.model.orm import * + +log = logging.getLogger( __name__ ) + +class DatasetsController( BaseAPIController, UsesHistoryDatasetAssociation ): + + @web.expose_api + def index( self, trans, hda_id, **kwd ): + """ + GET /api/datasets + Lists datasets. + """ + pass + + @web.expose_api + def show( self, trans, id, deleted='False', **kwd ): + """ + GET /api/datasets/{encoded_dataset_id} + Displays information about and/or content of a dataset. + """ + + # Get HDA. + try: + hda = self.get_dataset( trans, id, check_ownership=True, check_accessible=True ) + except Exception, e: + return str( e ) + + # Return information about HDA. + rval = None + try: + rval = hda.get_api_value() + except Exception, e: + rval = "Error in dataset API at listing contents" + log.error( rval + ": %s" % str(e) ) + trans.response.status = 500 + return rval diff --git a/lib/galaxy/web/api/histories.py b/lib/galaxy/web/api/histories.py index 819a509430e..b46b89bc3c2 100644 --- a/lib/galaxy/web/api/histories.py +++ b/lib/galaxy/web/api/histories.py @@ -20,28 +20,27 @@ class HistoriesController( BaseAPIController, UsesHistory ): GET /api/histories GET /api/histories/deleted Displays a collection (list) of histories. - """ + """ rval = [] deleted = util.string_as_bool( deleted ) - try: - query = trans.sa_session.query( trans.app.model.History ).filter_by( user=trans.user, deleted=deleted ).order_by( - desc(trans.app.model.History.table.c.update_time)).all() - except Exception, e: - rval = "Error in history API" - log.error( rval + ": %s" % str(e) ) - trans.response.status = 500 - - if not rval: - try: + if trans.user: + query = trans.sa_session.query(trans.app.model.History ).filter_by( user=trans.user, deleted=deleted ).order_by( + desc(trans.app.model.History.table.c.update_time)).all() for history in query: item = history.get_api_value(value_mapper={'id':trans.security.encode_id}) item['url'] = url_for( 'history', id=trans.security.encode_id( history.id ) ) rval.append( item ) - except Exception, e: - rval = "Error in history API at constructing return list" - log.error( rval + ": %s" % str(e) ) - trans.response.status = 500 + elif trans.galaxy_session.current_history: + #No user, this must be session authentication with an anonymous user. + history = trans.galaxy_session.current_history + item = history.get_api_value(value_mapper={'id':trans.security.encode_id}) + item['url'] = url_for( 'history', id=trans.security.encode_id( history.id ) ) + rval.append(item) + except Exception, e: + rval = "Error in history API" + log.error( rval + ": %s" % str(e) ) + trans.response.status = 500 return rval @web.expose_api @@ -54,7 +53,7 @@ class HistoriesController( BaseAPIController, UsesHistory ): history_id = id params = util.Params( kwd ) deleted = util.string_as_bool( deleted ) - + def traverse( datasets ): rval = {} states = trans.app.model.Dataset.states @@ -65,15 +64,13 @@ class HistoriesController( BaseAPIController, UsesHistory ): if not item['deleted']: rval[item['state']] = rval[item['state']] + 1 return rval - try: history = self.get_history( trans, history_id, check_ownership=True, check_accessible=True, deleted=deleted ) except Exception, e: return str( e ) - try: item = history.get_api_value(view='element', value_mapper={'id':trans.security.encode_id}) - num_sets = len( [hda.id for hda in history.datasets if not hda.deleted] ) + num_sets = len( [hda.id for hda in history.datasets if not hda.deleted] ) states = trans.app.model.Dataset.states state = states.ERROR if num_sets == 0: @@ -87,7 +84,7 @@ class HistoriesController( BaseAPIController, UsesHistory ): elif summary[states.QUEUED] > 0: state = states.QUEUED elif summary[states.OK] == num_sets: - state = states.OK + state = states.OK item['contents_url'] = url_for( 'history_contents', history_id=history_id ) item['state_details'] = summary item['state'] = state @@ -108,7 +105,7 @@ class HistoriesController( BaseAPIController, UsesHistory ): if payload.get( 'name', None ): hist_name = util.restore_text( payload['name'] ) new_history = trans.app.model.History( user=trans.user, name=hist_name ) - + trans.sa_session.add( new_history ) trans.sa_session.flush() item = new_history.get_api_value(view='element', value_mapper={'id':trans.security.encode_id}) @@ -124,8 +121,8 @@ class HistoriesController( BaseAPIController, UsesHistory ): # a request body is optional here purge = False if kwd.get( 'payload', None ): - purge = util.string_as_bool( kwd['payload'].get( 'purge', False ) ) - + purge = util.string_as_bool( kwd['payload'].get( 'purge', False ) ) + try: history = self.get_history( trans, history_id, check_ownership=True, check_accessible=False, deleted=True ) except Exception, e: diff --git a/lib/galaxy/web/api/history_contents.py b/lib/galaxy/web/api/history_contents.py index c583e46410d..ddcb37d75fe 100644 --- a/lib/galaxy/web/api/history_contents.py +++ b/lib/galaxy/web/api/history_contents.py @@ -1,11 +1,9 @@ """ API operations on the contents of a history. """ -import logging, os, string, shutil, urllib, re, socket -from cgi import escape, FieldStorage -from galaxy import util, datatypes, jobs, web, util +import logging +from galaxy import web from galaxy.web.base.controller import * -from galaxy.util.sanitize_html import sanitize_html from galaxy.model.orm import * import pkg_resources @@ -21,12 +19,11 @@ class HistoryContentsController( BaseAPIController, UsesHistoryDatasetAssociatio """ GET /api/histories/{encoded_history_id}/contents Displays a collection (list) of history contents - """ + """ try: history = self.get_history( trans, history_id, check_ownership=True, check_accessible=True ) except Exception, e: return str( e ) - rval = [] try: for dataset in history.datasets: @@ -57,6 +54,8 @@ class HistoryContentsController( BaseAPIController, UsesHistoryDatasetAssociatio return str( e ) try: item = content.get_api_value( view='element' ) + if trans.user_is_admin() or trans.app.config.expose_dataset_path: + item['file_name'] = content.file_name if not item['deleted']: # Problem: Method url_for cannot use the dataset controller # Get the environment from DefaultWebTransaction and use default webapp mapper instead of webapp API mapper @@ -67,7 +66,7 @@ class HistoryContentsController( BaseAPIController, UsesHistoryDatasetAssociatio item = self.encode_all_ids( trans, item ) except Exception, e: item = "Error in history API at listing dataset" - log.error( item + ": %s" % str(e) ) + log.error( item + ": %s" % str(e) ) trans.response.status = 500 return item @@ -77,7 +76,6 @@ class HistoryContentsController( BaseAPIController, UsesHistoryDatasetAssociatio POST /api/libraries/{encoded_history_id}/contents Creates a new history content item (file, aka HistoryDatasetAssociation). """ - params = util.Params( payload ) from_ld_id = payload.get( 'from_ld_id', None ) try: @@ -95,7 +93,6 @@ class HistoryContentsController( BaseAPIController, UsesHistoryDatasetAssociatio except Exception, e: return str( e ) hda = ld.library_dataset_dataset_association.to_history_dataset_association( history, add_to_history=True ) - history.add_dataset( hda ) trans.sa_session.flush() return hda.get_api_value() else: diff --git a/lib/galaxy/web/api/libraries.py b/lib/galaxy/web/api/libraries.py index ce85114cd7d..cefdf7b3ebe 100644 --- a/lib/galaxy/web/api/libraries.py +++ b/lib/galaxy/web/api/libraries.py @@ -7,18 +7,27 @@ from galaxy import util, datatypes, jobs, web, util from galaxy.web.base.controller import * from galaxy.util.sanitize_html import sanitize_html from galaxy.model.orm import * +from paste.httpexceptions import * log = logging.getLogger( __name__ ) class LibrariesController( BaseAPIController ): @web.expose_api - def index( self, trans, **kwd ): + def index( self, trans, deleted='False', **kwd ): """ GET /api/libraries + GET /api/libraries/deleted Displays a collection (list) of libraries. """ - query = trans.sa_session.query( trans.app.model.Library ).filter( trans.app.model.Library.table.c.deleted == False ) + query = trans.sa_session.query( trans.app.model.Library ) + deleted = util.string_as_bool( deleted ) + if deleted: + route = 'deleted_library' + query = query.filter( trans.app.model.Library.table.c.deleted == True ) + else: + route = 'library' + query = query.filter( trans.app.model.Library.table.c.deleted == False ) current_user_role_ids = [ role.id for role in trans.get_current_user_roles() ] library_access_action = trans.app.security_agent.permitted_actions.LIBRARY_ACCESS.action restricted_library_ids = [ lp.library_id for lp in trans.sa_session.query( trans.model.LibraryPermissions ) \ @@ -32,31 +41,32 @@ class LibrariesController( BaseAPIController ): rval = [] for library in query: item = library.get_api_value() - item['url'] = url_for( 'library', id=trans.security.encode_id( library.id ) ) + item['url'] = url_for( route, id=trans.security.encode_id( library.id ) ) item['id'] = trans.security.encode_id( item['id'] ) rval.append( item ) return rval @web.expose_api - def show( self, trans, id, **kwd ): + def show( self, trans, id, deleted='False', **kwd ): """ GET /api/libraries/{encoded_library_id} + GET /api/libraries/deleted/{encoded_library_id} Displays information about a library. """ library_id = id + deleted = util.string_as_bool( deleted ) params = util.Params( kwd ) try: decoded_library_id = trans.security.decode_id( library_id ) except TypeError: - trans.response.status = 400 - return "Malformed library id ( %s ) specified, unable to decode." % str( library_id ) + raise HTTPBadRequest( detail='Malformed library id ( %s ) specified, unable to decode.' % id ) try: library = trans.sa_session.query( trans.app.model.Library ).get( decoded_library_id ) + assert library.deleted == deleted except: library = None if not library or not ( trans.user_is_admin() or trans.app.security_agent.can_access_library( trans.get_current_user_roles(), library ) ): - trans.response.status = 400 - return "Invalid library id ( %s ) specified." % str( library_id ) + raise HTTPBadRequest( detail='Invalid library id ( %s ) specified.' % id ) item = library.get_api_value( view='element' ) #item['contents_url'] = url_for( 'contents', library_id=library_id ) item['contents_url'] = url_for( 'library_contents', library_id=library_id ) @@ -69,13 +79,11 @@ class LibrariesController( BaseAPIController ): Creates a new library. """ if not trans.user_is_admin(): - trans.response.status = 403 - return "You are not authorized to create a new library." + raise HTTPForbidden( detail='You are not authorized to create a new library.' ) params = util.Params( payload ) name = util.restore_text( params.get( 'name', None ) ) if not name: - trans.response.status = 400 - return "Missing required parameter 'name'." + raise HTTPBadRequest( detail="Missing required parameter 'name'." ) description = util.restore_text( params.get( 'description', '' ) ) synopsis = util.restore_text( params.get( 'synopsis', '' ) ) if synopsis in [ 'None', None ]: @@ -91,3 +99,22 @@ class LibrariesController( BaseAPIController ): rval['name'] = name rval['id'] = encoded_id return [ rval ] + + @web.expose_api + def delete( self, trans, id, **kwd ): + if not trans.user_is_admin(): + raise HTTPForbidden( detail='You are not authorized to delete libraries.' ) + try: + decoded_id = trans.security.decode_id( id ) + except TypeError: + raise HTTPBadRequest( detail='Malformed library id ( %s ) specified, unable to decode.' % id ) + try: + library = trans.sa_session.query( trans.app.model.Library ).get( decoded_id ) + except: + library = None + if not library: + raise HTTPBadRequest( detail='Invalid library id ( %s ) specified.' % id ) + library.deleted = True + trans.sa_session.add( library ) + trans.sa_session.flush() + return library.get_api_value( view='element', value_mapper={ 'id' : trans.security.encode_id } ) diff --git a/lib/galaxy/web/api/requests.py b/lib/galaxy/web/api/requests.py index ce05aa59df8..453d99d7b11 100644 --- a/lib/galaxy/web/api/requests.py +++ b/lib/galaxy/web/api/requests.py @@ -1,19 +1,17 @@ """ API operations on a sample tracking system. """ -import logging, os, string, shutil, urllib, re, socket -from cgi import escape, FieldStorage -from galaxy import util, datatypes, jobs, web, util +import logging +from galaxy import util, web from galaxy.web.base.controller import * -from galaxy.util.sanitize_html import sanitize_html from galaxy.model.orm import * from galaxy.util.bunch import Bunch log = logging.getLogger( __name__ ) class RequestsAPIController( BaseAPIController ): - update_types = Bunch( REQUEST = 'request_state' ) - update_type_values = [v[1] for v in update_types.items()] + _update_types = Bunch( REQUEST = 'request_state' ) + _update_type_values = [v[1] for v in _update_types.items()] @web.expose_api def index( self, trans, **kwd ): """ @@ -77,7 +75,7 @@ class RequestsAPIController( BaseAPIController ): return "Missing required 'update_type' parameter. Please consult the API documentation for help." else: update_type = payload.pop( 'update_type' ) - if update_type not in self.update_type_values: + if update_type not in self._update_type_values: trans.response.status = 400 return "Invalid value for 'update_type' parameter ( %s ) specified. Please consult the API documentation for help." % update_type try: diff --git a/lib/galaxy/web/api/tools.py b/lib/galaxy/web/api/tools.py new file mode 100644 index 00000000000..b3106270b6e --- /dev/null +++ b/lib/galaxy/web/api/tools.py @@ -0,0 +1,75 @@ +from galaxy import config, tools, web, util +from galaxy.web.base.controller import BaseController, BaseAPIController +from galaxy.util.bunch import Bunch + +messages = Bunch( + NO_TOOL = "no tool" +) + +class ToolsController( BaseAPIController ): + """ + RESTful controller for interactions with tools. + """ + + @web.expose_api + def index( self, trans, **kwds ): + """ + GET /api/tools: returns a list of tools defined by parameters + parameters: + in_panel - if true, tools are returned in panel structure, + including sections and labels + trackster - if true, only tools that are compatible with + Trackster are returned + """ + + # Read params. + in_panel = util.string_as_bool( kwds.get( 'in_panel', 'True' ) ) + trackster = util.string_as_bool( kwds.get( 'trackster', 'False' ) ) + + # Create return value. + return self.app.toolbox.to_dict( trans, in_panel=in_panel, trackster=trackster ) + + @web.json + def show( self, trans, id, **kwd ): + """ + GET /api/tools/{tool_id} + Returns tool information, including parameters and inputs. + """ + return self.app.toolbox.tools_by_id[ id ].to_dict( trans, for_display=True ) + + @web.expose_api + def create( self, trans, payload, **kwd ): + """ + POST /api/tools + Executes tool using specified inputs, creating new history-dataset + associations, which are returned. + """ + + # TODO: set target history? + + # -- Execute tool. -- + + # Get tool. + tool_id = payload[ 'id' ] + tool = trans.app.toolbox.get_tool( tool_id ) + if not tool: + return { "message": { "type": "error", "text" : messages.NO_TOOL } } + + # Set up inputs. + inputs = payload[ 'inputs' ] + # HACK: add run button so that tool.handle_input will run tool. + inputs['runtool_btn'] = 'Execute' + # TODO: encode data ids and decode ids. + params = util.Params( inputs, sanitize = False ) + template, vars = tool.handle_input( trans, params.__dict__ ) + + # TODO: check for errors and ensure that output dataset(s) are available. + output_datasets = vars[ 'out_data' ].values() + rval = { + "outputs": [] + } + outputs = rval[ "outputs" ] + for output in output_datasets: + outputs.append( output.get_api_value() ) + return rval + \ No newline at end of file diff --git a/lib/galaxy/web/api/workflows.py b/lib/galaxy/web/api/workflows.py index 627612574d5..71de06dad38 100644 --- a/lib/galaxy/web/api/workflows.py +++ b/lib/galaxy/web/api/workflows.py @@ -18,9 +18,9 @@ class WorkflowsAPIController(BaseAPIController): def index(self, trans, **kwd): """ GET /api/workflows - Displays a collection of workflows + + Displays a collection of workflows. """ - # List parameters of a specific workflow rval = [] for wf in trans.sa_session.query(trans.app.model.StoredWorkflow).filter_by( user=trans.user, deleted=False).order_by( @@ -38,10 +38,12 @@ class WorkflowsAPIController(BaseAPIController): item['url'] = url_for('workflow', id=encoded_id) rval.append(item) return rval + @web.expose_api def show(self, trans, id, **kwd): """ GET /api/workflows/{encoded_workflow_id} + Displays information needed to run a workflow from the command line. """ workflow_id = id @@ -72,28 +74,29 @@ class WorkflowsAPIController(BaseAPIController): # p = step.get_required_parameters() item['inputs'] = inputs return item + @web.expose_api def create(self, trans, payload, **kwd): """ POST /api/workflows + We're not creating workflows from the api. Just execute for now. + However, we will import them if installed_repository_file is specified """ if 'workflow_id' not in payload: # create new if 'installed_repository_file' in payload: workflow_controller = trans.webapp.controllers[ 'workflow' ] - result = workflow_controller.import_workflow( trans=trans, + result = workflow_controller.import_workflow( trans=trans, cntrller='api', **payload) return result trans.response.status = 403 return "Either workflow_id or installed_repository_file must be specified" - if 'installed_repository_file' in payload: trans.response.status = 403 return "installed_repository_file may not be specified with workflow_id" - stored_workflow = trans.sa_session.query(self.app.model.StoredWorkflow).get( trans.security.decode_id(payload['workflow_id'])) if stored_workflow.user != trans.user and not trans.user_is_admin(): @@ -215,3 +218,4 @@ class WorkflowsAPIController(BaseAPIController): trans.sa_session.add( workflow_invocation ) trans.sa_session.flush() return rval + diff --git a/lib/galaxy/web/base/controller.py b/lib/galaxy/web/base/controller.py index 93f5aac6be7..2b35d2abd17 100644 --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -322,7 +322,7 @@ class UsesVisualization( SharableItemSecurity ): if hda_ldda == "hda": dataset = self.get_dataset( trans, dataset_id, check_ownership=False, check_accessible=True ) else: - dataset = trans.sa_session.query( trans.app.model.LibraryDatasetDatasetAssociation ).get( trans.security.decode_id(dataset_id) ) + dataset = trans.sa_session.query( trans.app.model.LibraryDatasetDatasetAssociation ).get( dataset_id ) try: prefs = track_dict['prefs'] @@ -1325,8 +1325,7 @@ class Admin( object ): message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) if webapp == 'galaxy': - cloned_repositories = trans.sa_session.query( trans.model.ToolShedRepository ) \ - .first() + cloned_repositories = trans.sa_session.query( trans.model.ToolShedRepository ).first() return trans.fill_template( '/webapps/galaxy/admin/index.mako', webapp=webapp, cloned_repositories=cloned_repositories, @@ -2381,8 +2380,8 @@ class Admin( object ): deleted = [] msg = None status = None - if not trans.app.config.get_bool( "enable_job_running", True ): - return trans.show_error_message( 'This Galaxy instance is not configured to run jobs. If using multiple servers, please directly access the job running instance to manage jobs.' ) + if not self.app.config.job_manager != self.app.config.server_name: + return trans.show_error_message( 'This Galaxy instance is not the job manager. If using multiple servers, please directly access the job manager instance to manage jobs.' ) job_ids = util.listify( stop ) if job_ids and stop_msg in [ None, '' ]: msg = 'Please enter an error message to display to the user describing why the job was terminated' diff --git a/lib/galaxy/web/buildapp.py b/lib/galaxy/web/buildapp.py index 597ffaad6f0..60bf45d5792 100644 --- a/lib/galaxy/web/buildapp.py +++ b/lib/galaxy/web/buildapp.py @@ -103,36 +103,38 @@ def app_factory( global_conf, **kwargs ): webapp.add_route( '/u/:username/h/:slug', controller='history', action='display_by_username_and_slug' ) webapp.add_route( '/u/:username/w/:slug', controller='workflow', action='display_by_username_and_slug' ) webapp.add_route( '/u/:username/v/:slug', controller='visualization', action='display_by_username_and_slug' ) - # If enabled, add the web API - if asbool( kwargs.get( 'enable_api', False ) ): - add_api_controllers( webapp, app ) - webapp.api_mapper.resource( 'content', - 'contents', - controller='library_contents', - name_prefix='library_', - path_prefix='/api/libraries/:library_id', - parent_resources=dict( member_name='library', collection_name='libraries' ) ) - webapp.api_mapper.resource( 'content', - 'contents', - controller='history_contents', - name_prefix='history_', - path_prefix='/api/histories/:history_id', - parent_resources=dict( member_name='history', collection_name='histories' ) ) - webapp.api_mapper.resource( 'permission', - 'permissions', - path_prefix='/api/libraries/:library_id', - parent_resources=dict( member_name='library', collection_name='libraries' ) ) - webapp.api_mapper.resource( 'library', 'libraries', path_prefix='/api' ) - webapp.api_mapper.resource( 'sample', 'samples', path_prefix='/api' ) - webapp.api_mapper.resource( 'request', 'requests', path_prefix='/api' ) - webapp.api_mapper.resource( 'form', 'forms', path_prefix='/api' ) - webapp.api_mapper.resource( 'request_type', 'request_types', path_prefix='/api' ) - webapp.api_mapper.resource( 'role', 'roles', path_prefix='/api' ) - webapp.api_mapper.resource_with_deleted( 'quota', 'quotas', path_prefix='/api' ) - webapp.api_mapper.resource_with_deleted( 'user', 'users', path_prefix='/api' ) - webapp.api_mapper.resource( 'workflow', 'workflows', path_prefix='/api' ) - webapp.api_mapper.resource_with_deleted( 'history', 'histories', path_prefix='/api' ) - #webapp.api_mapper.connect( 'run_workflow', '/api/workflow/{workflow_id}/library/{library_id}', controller='workflows', action='run', workflow_id=None, library_id=None, conditions=dict(method=["GET"]) ) + + # Add the web API + add_api_controllers( webapp, app ) + webapp.api_mapper.resource( 'content', + 'contents', + controller='library_contents', + name_prefix='library_', + path_prefix='/api/libraries/:library_id', + parent_resources=dict( member_name='library', collection_name='libraries' ) ) + webapp.api_mapper.resource( 'content', + 'contents', + controller='history_contents', + name_prefix='history_', + path_prefix='/api/histories/:history_id', + parent_resources=dict( member_name='history', collection_name='histories' ) ) + webapp.api_mapper.resource( 'permission', + 'permissions', + path_prefix='/api/libraries/:library_id', + parent_resources=dict( member_name='library', collection_name='libraries' ) ) + webapp.api_mapper.resource( 'dataset', 'datasets', path_prefix='/api' ) + webapp.api_mapper.resource_with_deleted( 'library', 'libraries', path_prefix='/api' ) + webapp.api_mapper.resource( 'sample', 'samples', path_prefix='/api' ) + webapp.api_mapper.resource( 'request', 'requests', path_prefix='/api' ) + webapp.api_mapper.resource( 'form', 'forms', path_prefix='/api' ) + webapp.api_mapper.resource( 'request_type', 'request_types', path_prefix='/api' ) + webapp.api_mapper.resource( 'role', 'roles', path_prefix='/api' ) + webapp.api_mapper.resource_with_deleted( 'quota', 'quotas', path_prefix='/api' ) + webapp.api_mapper.resource( 'tool', 'tools', path_prefix='/api' ) + webapp.api_mapper.resource_with_deleted( 'user', 'users', path_prefix='/api' ) + webapp.api_mapper.resource( 'workflow', 'workflows', path_prefix='/api' ) + webapp.api_mapper.resource_with_deleted( 'history', 'histories', path_prefix='/api' ) + #webapp.api_mapper.connect( 'run_workflow', '/api/workflow/{workflow_id}/library/{library_id}', controller='workflows', action='run', workflow_id=None, library_id=None, conditions=dict(method=["GET"]) ) webapp.finalize_config() # Wrap the webapp in some useful middleware diff --git a/lib/galaxy/web/controllers/admin_toolshed.py b/lib/galaxy/web/controllers/admin_toolshed.py index 9e1a9ff8c00..569a05ee3b0 100644 --- a/lib/galaxy/web/controllers/admin_toolshed.py +++ b/lib/galaxy/web/controllers/admin_toolshed.py @@ -2,10 +2,15 @@ import urllib2 from galaxy.web.controllers.admin import * from galaxy.util.json import from_json_string, to_json_string from galaxy.util.shed_util import * -from galaxy import tools +from galaxy import eggs, tools + +eggs.require( 'mercurial' ) +from mercurial import hg log = logging.getLogger( __name__ ) +MAX_CONTENT_SIZE = 32768 + class RepositoryListGrid( grids.Grid ): class NameColumn( grids.TextColumn ): def get_value( self, trans, grid, tool_shed_repository ): @@ -144,7 +149,7 @@ class AdminToolshed( AdminGalaxy ): def browse_tool_shed( self, trans, **kwd ): tool_shed_url = kwd[ 'tool_shed_url' ] galaxy_url = url_for( '/', qualified=True ) - url = '%s/repository/browse_valid_repositories?galaxy_url=%s&webapp=galaxy' % ( tool_shed_url, galaxy_url ) + url = '%srepository/browse_valid_repositories?galaxy_url=%s&webapp=galaxy&no_reset=true' % ( tool_shed_url, galaxy_url ) return trans.response.send_redirect( url ) @web.expose @web.require_admin @@ -162,7 +167,7 @@ class AdminToolshed( AdminGalaxy ): # Send a request to the relevant tool shed to see if there are any updates. repository = get_repository( trans, kwd[ 'id' ] ) tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) - url = '%s/repository/check_for_updates?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ + url = '%s/repository/check_for_updates?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s&webapp=galaxy&no_reset=true' % \ ( tool_shed_url, url_for( '/', qualified=True ), repository.name, repository.owner, repository.changeset_revision ) return trans.response.send_redirect( url ) @web.expose @@ -213,14 +218,14 @@ class AdminToolshed( AdminGalaxy ): def find_tools_in_tool_shed( self, trans, **kwd ): tool_shed_url = kwd[ 'tool_shed_url' ] galaxy_url = url_for( '/', qualified=True ) - url = '%s/repository/find_tools?galaxy_url=%s&webapp=galaxy' % ( tool_shed_url, galaxy_url ) + url = '%srepository/find_tools?galaxy_url=%s&webapp=galaxy&no_reset=true' % ( tool_shed_url, galaxy_url ) return trans.response.send_redirect( url ) @web.expose @web.require_admin def find_workflows_in_tool_shed( self, trans, **kwd ): tool_shed_url = kwd[ 'tool_shed_url' ] galaxy_url = url_for( '/', qualified=True ) - url = '%s/repository/find_workflows?galaxy_url=%s&webapp=galaxy' % ( tool_shed_url, galaxy_url ) + url = '%srepository/find_workflows?galaxy_url=%s&webapp=galaxy&no_reset=true' % ( tool_shed_url, galaxy_url ) return trans.response.send_redirect( url ) @web.expose @web.require_admin @@ -288,58 +293,45 @@ class AdminToolshed( AdminGalaxy ): current_working_dir = os.getcwd() installed_repository_names = [] for name, repo_info_tuple in repo_info_dict.items(): - description, repository_clone_url, changeset_revision = repo_info_tuple + description, repository_clone_url, changeset_revision, ctx_rev = repo_info_tuple clone_dir = os.path.join( tool_path, self.__generate_tool_path( repository_clone_url, changeset_revision ) ) relative_install_dir = os.path.join( clone_dir, name ) if os.path.exists( clone_dir ): # Repository and revision has already been cloned. message += 'Revision %s of repository %s was previously installed.
' % ( changeset_revision, name ) else: - returncode, tmp_name = clone_repository( name, clone_dir, current_working_dir, repository_clone_url ) - if returncode == 0: - returncode, tmp_name = update_repository( current_working_dir, relative_install_dir, changeset_revision ) - if returncode == 0: - owner = get_repository_owner( clean_repository_clone_url( repository_clone_url ) ) - tool_shed = clean_tool_shed_url( tool_shed_url ) - tool_shed_repository, metadata_dict = load_repository_contents( trans, - repository_name=name, - description=description, - owner=owner, - changeset_revision=changeset_revision, - tool_path=tool_path, - repository_clone_url=repository_clone_url, - relative_install_dir=relative_install_dir, - current_working_dir=current_working_dir, - tmp_name=tmp_name, - tool_shed=tool_shed, - tool_section=tool_section, - shed_tool_conf=shed_tool_conf ) - if 'tools' in metadata_dict: - # Get the tool_versions from the tool shed for each tool in the installed change set. - url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ - ( tool_shed_url, name, owner, changeset_revision ) - response = urllib2.urlopen( url ) - text = response.read() - response.close() - if text: - tool_version_dicts = from_json_string( text ) - handle_tool_versions( trans.app, tool_version_dicts, tool_shed_repository ) - else: - message += "Version information for the tools included in the %s repository is missing. " % name - message += "Reset all of this repository's metadata in the tool shed, then set the installed tool versions " - message += "from the installed repository's Repository Actions menu. " - status = 'error' - installed_repository_names.append( name ) + clone_repository( repository_clone_url, os.path.abspath( relative_install_dir ), ctx_rev ) + owner = get_repository_owner( clean_repository_clone_url( repository_clone_url ) ) + tool_shed = clean_tool_shed_url( tool_shed_url ) + tool_shed_repository, metadata_dict = load_repository_contents( trans, + repository_name=name, + description=description, + owner=owner, + changeset_revision=changeset_revision, + ctx_rev=ctx_rev, + tool_path=tool_path, + repository_clone_url=repository_clone_url, + relative_install_dir=relative_install_dir, + current_working_dir=current_working_dir, + tool_shed=tool_shed, + tool_section=tool_section, + shed_tool_conf=shed_tool_conf ) + if 'tools' in metadata_dict: + # Get the tool_versions from the tool shed for each tool in the installed change set. + url = '%srepository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy&no_reset=true' % \ + ( tool_shed_url, name, owner, changeset_revision ) + response = urllib2.urlopen( url ) + text = response.read() + response.close() + if text: + tool_version_dicts = from_json_string( text ) + handle_tool_versions( trans.app, tool_version_dicts, tool_shed_repository ) else: - tmp_stderr = open( tmp_name, 'rb' ) - message += '%s
' % tmp_stderr.read() - tmp_stderr.close() + message += "Version information for the tools included in the %s repository is missing. " % name + message += "Reset all of this repository's metadata in the tool shed, then set the installed tool versions " + message += "from the installed repository's Repository Actions menu. " status = 'error' - else: - tmp_stderr = open( tmp_name, 'rb' ) - message += '%s
' % tmp_stderr.read() - tmp_stderr.close() - status = 'error' + installed_repository_names.append( name ) if installed_repository_names: installed_repository_names.sort() num_repositories_installed = len( installed_repository_names ) @@ -367,6 +359,29 @@ class AdminToolshed( AdminGalaxy ): shed_tool_conf = shed_tool_conf.replace( './', '', 1 ) shed_tool_conf_select_field = None tool_panel_section_select_field = build_tool_panel_section_select_field( trans ) + if includes_tools: + # If we're installing a single repository that contains a readme file, get it's contents to display. + decoded_repo_info_dict = tool_shed_decode( repo_info_dict ) + if len( decoded_repo_info_dict ) == 1: + name = decoded_repo_info_dict.keys()[ 0 ] + repo_info_tuple = decoded_repo_info_dict[ name ] + description, repository_clone_url, changeset_revision, ctx_rev = repo_info_tuple + owner = get_repository_owner( clean_repository_clone_url( repository_clone_url ) ) + url = '%srepository/get_readme?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy&no_reset=true' % ( tool_shed_url, name, owner, changeset_revision ) + response = urllib2.urlopen( url ) + raw_text = response.read() + response.close() + readme_text = '' + for i, line in enumerate( raw_text ): + readme_text = '%s%s' % ( readme_text, to_html_str( line ) ) + if len( readme_text ) > MAX_CONTENT_SIZE: + large_str = '\nFile contents truncated because file size is larger than maximum viewing size of %s\n' % util.nice_size( MAX_CONTENT_SIZE ) + readme_text = '%s%s' % ( readme_text, to_html_str( large_str ) ) + break + else: + readme_text = '' + else: + readme_text = '' return trans.fill_template( '/admin/tool_shed_repository/select_tool_panel_section.mako', tool_shed_url=tool_shed_url, repo_info_dict=repo_info_dict, @@ -375,6 +390,7 @@ class AdminToolshed( AdminGalaxy ): shed_tool_conf_select_field=shed_tool_conf_select_field, tool_panel_section_select_field=tool_panel_section_select_field, new_tool_panel_section=new_tool_panel_section, + readme_text=readme_text, message=message, status=status ) @web.expose @@ -418,79 +434,82 @@ class AdminToolshed( AdminGalaxy ): repository_clone_url = generate_clone_url( trans, repository ) clone_dir = os.path.join( tool_path, self.__generate_tool_path( repository_clone_url, repository.installed_changeset_revision ) ) relative_install_dir = os.path.join( clone_dir, repository.name ) - returncode, tmp_name = clone_repository( repository.name, clone_dir, current_working_dir, repository_clone_url ) - if returncode == 0: - returncode, tmp_name = update_repository( current_working_dir, relative_install_dir, repository.installed_changeset_revision ) - if returncode == 0: - if repository.includes_tools: - # Get the location in the tool panel in which each tool was originally loaded. - metadata = repository.metadata - if 'tool_panel_section' in metadata: - tool_panel_dict = metadata[ 'tool_panel_section' ] - if not tool_panel_dict: - tool_panel_dict = generate_tool_panel_dict_for_new_install( metadata[ 'tools' ] ) + tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) + if not repository.ctx_rev: + # The ctx_rev column was introduced late, so may be null for some installed ToolShedRepositories. + ctx_rev = get_ctx_rev( tool_shed_url, repository.name, repository.owner, repository.installed_changeset_revision ) + else: + ctx_rev = repository.ctx_rev + clone_repository( repository_clone_url, os.path.abspath( relative_install_dir ), ctx_rev ) + if repository.includes_tools: + # Get the location in the tool panel in which each tool was originally loaded. + metadata = repository.metadata + if 'tool_panel_section' in metadata: + tool_panel_dict = metadata[ 'tool_panel_section' ] + if not tool_panel_dict: + tool_panel_dict = generate_tool_panel_dict_for_new_install( metadata[ 'tools' ] ) + else: + tool_panel_dict = generate_tool_panel_dict_for_new_install( metadata[ 'tools' ] ) + # TODO: Fix this to handle the case where the tools are distributed across in more than 1 ToolSection. The + # following assumes everything was loaded into 1 section (or no section) in the tool panel. + tool_section_dicts = tool_panel_dict[ tool_panel_dict.keys()[ 0 ] ] + tool_section_dict = tool_section_dicts[ 0 ] + original_section_id = tool_section_dict[ 'id' ] + original_section_name = tool_section_dict[ 'name' ] + if no_changes_checked: + if original_section_id in [ '' ]: + tool_section = None + else: + section_key = 'section_%s' % str( original_section_id ) + if section_key in trans.app.toolbox.tool_panel: + tool_section = trans.app.toolbox.tool_panel[ section_key ] else: - tool_panel_dict = generate_tool_panel_dict_for_new_install( metadata[ 'tools' ] ) - # TODO: Fix this to handle the case where the tools are distributed across in more than 1 ToolSection. The - # following assumes everything was loaded into 1 section (or no section) in the tool panel. - tool_section_dicts = tool_panel_dict[ tool_panel_dict.keys()[ 0 ] ] - tool_section_dict = tool_section_dicts[ 0 ] - original_section_id = tool_section_dict[ 'id' ] - original_section_name = tool_section_dict[ 'name' ] - if no_changes_checked: - if original_section_id in [ '' ]: - tool_section = None - else: - section_key = 'section_%s' % str( original_section_id ) - if section_key in trans.app.toolbox.tool_panel: - tool_section = trans.app.toolbox.tool_panel[ section_key ] - else: - # The section in which the tool was originally loaded used to be in the tool panel, but no longer is. - elem = Element( 'section' ) - elem.attrib[ 'name' ] = original_section_name - elem.attrib[ 'id' ] = original_section_id - elem.attrib[ 'version' ] = '' - tool_section = tools.ToolSection( elem ) - trans.app.toolbox.tool_panel[ section_key ] = tool_section + # The section in which the tool was originally loaded used to be in the tool panel, but no longer is. + elem = Element( 'section' ) + elem.attrib[ 'name' ] = original_section_name + elem.attrib[ 'id' ] = original_section_id + elem.attrib[ 'version' ] = '' + tool_section = tools.ToolSection( elem ) + trans.app.toolbox.tool_panel[ section_key ] = tool_section + else: + # The user elected to change the tool panel section to contain the tools. + new_tool_panel_section = kwd.get( 'new_tool_panel_section', '' ) + tool_panel_section = kwd.get( 'tool_panel_section', '' ) + if new_tool_panel_section: + section_id = new_tool_panel_section.lower().replace( ' ', '_' ) + new_section_key = 'section_%s' % str( section_id ) + if new_section_key in trans.app.toolbox.tool_panel: + # Appending a tool to an existing section in trans.app.toolbox.tool_panel + log.debug( "Appending to tool panel section: %s" % new_tool_panel_section ) + tool_section = trans.app.toolbox.tool_panel[ new_section_key ] else: - # The user elected to change the tool panel section to contain the tools. - new_tool_panel_section = kwd.get( 'new_tool_panel_section', '' ) - tool_panel_section = kwd.get( 'tool_panel_section', '' ) - if new_tool_panel_section: - section_id = new_tool_panel_section.lower().replace( ' ', '_' ) - new_section_key = 'section_%s' % str( section_id ) - if new_section_key in trans.app.toolbox.tool_panel: - # Appending a tool to an existing section in trans.app.toolbox.tool_panel - log.debug( "Appending to tool panel section: %s" % new_tool_panel_section ) - tool_section = trans.app.toolbox.tool_panel[ new_section_key ] - else: - # Appending a new section to trans.app.toolbox.tool_panel - log.debug( "Loading new tool panel section: %s" % new_tool_panel_section ) - elem = Element( 'section' ) - elem.attrib[ 'name' ] = new_tool_panel_section - elem.attrib[ 'id' ] = section_id - elem.attrib[ 'version' ] = '' - tool_section = tools.ToolSection( elem ) - trans.app.toolbox.tool_panel[ new_section_key ] = tool_section - elif tool_panel_section: - section_key = 'section_%s' % tool_panel_section - tool_section = trans.app.toolbox.tool_panel[ section_key ] - else: - tool_section = None - tool_shed_repository, metadata_dict = load_repository_contents( trans, - repository_name=repository.name, - description=repository.description, - owner=repository.owner, - changeset_revision=repository.installed_changeset_revision, - tool_path=tool_path, - repository_clone_url=repository_clone_url, - relative_install_dir=relative_install_dir, - current_working_dir=current_working_dir, - tmp_name=tmp_name, - tool_shed=repository.tool_shed, - tool_section=tool_section, - shed_tool_conf=shed_tool_conf ) - repository.uninstalled = False + # Appending a new section to trans.app.toolbox.tool_panel + log.debug( "Loading new tool panel section: %s" % new_tool_panel_section ) + elem = Element( 'section' ) + elem.attrib[ 'name' ] = new_tool_panel_section + elem.attrib[ 'id' ] = section_id + elem.attrib[ 'version' ] = '' + tool_section = tools.ToolSection( elem ) + trans.app.toolbox.tool_panel[ new_section_key ] = tool_section + elif tool_panel_section: + section_key = 'section_%s' % tool_panel_section + tool_section = trans.app.toolbox.tool_panel[ section_key ] + else: + tool_section = None + tool_shed_repository, metadata_dict = load_repository_contents( trans, + repository_name=repository.name, + description=repository.description, + owner=repository.owner, + changeset_revision=repository.installed_changeset_revision, + ctx_rev=ctx_rev, + tool_path=tool_path, + repository_clone_url=repository_clone_url, + relative_install_dir=relative_install_dir, + current_working_dir=current_working_dir, + tool_shed=repository.tool_shed, + tool_section=tool_section, + shed_tool_conf=shed_tool_conf ) + repository.uninstalled = False repository.deleted = False trans.sa_session.add( repository ) trans.sa_session.flush() @@ -545,7 +564,7 @@ class AdminToolshed( AdminGalaxy ): # Get the tool_versions from the tool shed for each tool in the installed change set. repository = get_repository( trans, kwd[ 'id' ] ) tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository ) - url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \ + url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy&no_reset=true' % \ ( tool_shed_url, repository.name, repository.owner, repository.changeset_revision ) response = urllib2.urlopen( url ) text = response.read() @@ -580,41 +599,32 @@ class AdminToolshed( AdminGalaxy ): owner = params.get( 'owner', None ) changeset_revision = params.get( 'changeset_revision', None ) latest_changeset_revision = params.get( 'latest_changeset_revision', None ) + latest_ctx_rev = params.get( 'latest_ctx_rev', None ) repository = get_repository_by_shed_name_owner_changeset_revision( trans.app, tool_shed_url, name, owner, changeset_revision ) - if changeset_revision and latest_changeset_revision: + if changeset_revision and latest_changeset_revision and latest_ctx_rev: if changeset_revision == latest_changeset_revision: message = "The cloned tool shed repository named '%s' is current (there are no updates available)." % name else: current_working_dir = os.getcwd() shed_tool_conf, tool_path, relative_install_dir = get_tool_panel_config_tool_path_install_dir( trans.app, repository ) if relative_install_dir: - repo_files_dir = os.path.join( relative_install_dir, name ) - returncode, tmp_name = pull_repository( current_working_dir, repo_files_dir, name ) - if returncode == 0: - returncode, tmp_name = update_repository( current_working_dir, repo_files_dir, latest_changeset_revision ) - if returncode == 0: - # Update the repository metadata. - repository_clone_url = os.path.join( tool_shed_url, 'repos', owner, name ) - tool_shed = clean_tool_shed_url( tool_shed_url ) - metadata_dict = generate_metadata( trans.app.toolbox, relative_install_dir, repository_clone_url ) - repository.metadata = metadata_dict - # Update the repository changeset_revision in the database. - repository.changeset_revision = latest_changeset_revision - repository.update_available = False - trans.sa_session.add( repository ) - trans.sa_session.flush() - message = "The cloned repository named '%s' has been updated to change set revision '%s'." % \ - ( name, latest_changeset_revision ) - else: - tmp_stderr = open( tmp_name, 'rb' ) - message = tmp_stderr.read() - tmp_stderr.close() - status = 'error' - else: - tmp_stderr = open( tmp_name, 'rb' ) - message = tmp_stderr.read() - tmp_stderr.close() - status = 'error' + repo_files_dir = os.path.abspath( os.path.join( relative_install_dir, name ) ) + repo = hg.repository( get_configured_ui(), path=repo_files_dir ) + repository_clone_url = os.path.join( tool_shed_url, 'repos', owner, name ) + pull_repository( repo, repository_clone_url, latest_ctx_rev ) + update_repository( repo, latest_ctx_rev ) + # Update the repository metadata. + tool_shed = clean_tool_shed_url( tool_shed_url ) + metadata_dict = generate_metadata( trans.app.toolbox, relative_install_dir, repository_clone_url ) + repository.metadata = metadata_dict + # Update the repository changeset_revision in the database. + repository.changeset_revision = latest_changeset_revision + repository.ctx_rev = latest_ctx_rev + repository.update_available = False + trans.sa_session.add( repository ) + trans.sa_session.flush() + message = "The cloned repository named '%s' has been updated to change set revision '%s'." % \ + ( name, latest_changeset_revision ) else: message = "The directory containing the cloned repository named '%s' cannot be found." % name status = 'error' diff --git a/lib/galaxy/web/controllers/cloud.py b/lib/galaxy/web/controllers/cloud.py index ea9c852beb2..69d9c3f21a2 100644 --- a/lib/galaxy/web/controllers/cloud.py +++ b/lib/galaxy/web/controllers/cloud.py @@ -6,10 +6,13 @@ BioCloudCentral Source: https://github.com/chapmanb/biocloudcentral """ -import boto import datetime import logging import time +from galaxy import eggs +import pkg_resources +pkg_resources.require('boto') +import boto from galaxy import web from galaxy.web.base.controller import BaseUIController from boto.ec2.regioninfo import RegionInfo diff --git a/lib/galaxy/web/controllers/data_admin.py b/lib/galaxy/web/controllers/data_admin.py new file mode 100644 index 00000000000..6a5726726be --- /dev/null +++ b/lib/galaxy/web/controllers/data_admin.py @@ -0,0 +1,175 @@ +import sys, ftplib, json +from galaxy import model, util +from galaxy.jobs import transfer_manager +from galaxy.web.base.controller import * +from galaxy.web.framework.helpers import time_ago, iff, grids +from galaxy.model.orm import * +from library_common import get_comptypes, lucene_search, whoosh_search + +# Older py compatibility +try: + set() +except: + from sets import Set as set + +import logging +log = logging.getLogger( __name__ ) + +class DataAdmin( BaseUIController ): + jobstyles = dict( + done='panel-done-message', + waiting='state-color-waiting', + running='state-color-running', + downloaded='state-color-running', + new='state-color-new', + ok='panel-done-message', + error='panel-error-message' + ) + + @web.expose + @web.require_admin + def manage_data( self, trans, **kwd ): + dbkeys = trans.db_builds + return trans.fill_template( '/admin/data_admin/data_form.mako', dbkeys=dbkeys ) + + @web.expose + @web.require_admin + def download_build( self, trans, **kwd ): + """Download a genome from a remote source and add it to the library.""" + params = util.Params( kwd ) + source = params.get('source', '') + longname = params.get('longname', None) + if not isinstance( params.get( 'indexers', None ), list ): + indexers = [ params.get( 'indexers', None ) ] + else: + indexers = params.get( 'indexers', None ) + if indexers is not None: + if indexers == [None]: + indexers = None + url = None + liftover = None + newlift = [] + dbkey = params.get( 'dbkey', None ) + dbkeys = dict() + protocol = 'http' + + if source == 'NCBI': + dbkey = params.get('dbkey', '')[0] + url = 'http://togows.dbcls.jp/entry/ncbi-nucleotide/%s.fasta' % dbkey + elif source == 'Broad': + dbkey = params.get('dbkey', '')[0] + url = 'ftp://ftp.broadinstitute.org/pub/seq/references/%s.fasta' % dbkey + elif source == 'UCSC': + longname = None + for build in trans.db_builds: + if dbkey[1] == build[0]: + dbkey = build[0] + longname = build[1] + break + assert dbkey is not '?', 'That build was not found' + ftp = ftplib.FTP('hgdownload.cse.ucsc.edu') + ftp.login('anonymous', 'user@example.com') + checker = [] + liftover = [] + newlift = [] + try: + ftp.retrlines('NLST /goldenPath/%s/liftOver/*.chain.gz' % dbkey, liftover.append) + for chain in liftover: + fname = chain.split( '/' )[-1] + target = fname.replace( '.over.chain.gz', '' ).split( 'To' )[1] + target = target[0].lower() + target[1:] + newlift.append( [ chain, dbkey, target ] ) + current = dbkey[0].upper() + dbkey[1:] + targetfile = '%sTo%s.over.chain.gz' % ( target, current ) + newlift.append( [ '/goldenPath/%s/liftOver/%s' % ( target, targetfile ), target, dbkey ] ) + except: + newlift = None + pass + ftp.retrlines('NLST /goldenPath/%s/bigZips/' % dbkey, checker.append) + for filename in [ dbkey, 'chromFa' ]: + for extension in [ '.tar.gz', '.tar.bz2', '.zip', '.fa.gz', '.fa.bz2' ]: + testfile = '/goldenPath/%s/bigZips/%s%s' % ( dbkey, filename, extension ) + if testfile in checker: + url = 'ftp://hgdownload.cse.ucsc.edu%s' % testfile + break; + else: + continue + if url is None: + message = u'The genome %s was not found on the UCSC server.' % dbkey + status = u'error' + return trans.fill_template( '/admin/data_admin/data_form.mako', + message=message, + status=status ) + elif source == 'Ensembl': + section = params.get('ensembl_section', '') + release1 = params.get('release_number', '') + organism = params.get('organism', '') + name = params.get('name', '') + longname = organism + dbkey = name + release2 = params.get('release2', '') + release2 = ".%s" % release2 if release2 else "" + if section == 'standard': + url = 'ftp://ftp.ensembl.org/pub/release-%s/fasta/%s/dna/%s.%s%s.dna.toplevel.fa.gz' % \ + (release1, organism.lower(), organism, name, release2) + else: + url = 'ftp://ftp.ensemblgenomes.org/pub/%s/release-%s/fasta/%s/dna/%s.%s%s.dna.toplevel.fa.gz' % \ + (section, release1, organism.lower(), organism, name, release2) + elif source == 'local': + url = 'http://127.0.0.1/%s.tar.gz' % dbkey + else: + raise ValueError + params = dict( protocol='http', name=dbkey, datatype='fasta', url=url, user=trans.user.id ) + jobid = trans.app.job_manager.deferred_job_queue.plugins['GenomeTransferPlugin'].create_job( trans, url, dbkey, longname, indexers ) + chainjob = [] + if newlift is not None: + for chain in newlift: + liftover_url = u'ftp://hgdownload.cse.ucsc.edu%s' % chain[0] + from_genome = chain[1] + to_genome = chain[2] + destfile = liftover_url.split('/')[-1].replace('.gz', '') + chainjob.append( trans.app.job_manager.deferred_job_queue.plugins['LiftOverTransferPlugin'].create_job( trans, liftover_url, dbkey, from_genome, to_genome, destfile ) ) + job = trans.app.job_manager.deferred_job_queue.plugins['GenomeTransferPlugin'].get_job_status( jobid ) + job.params['liftover'] = chainjob + trans.app.model.context.current.add( job ) + trans.app.model.context.current.flush() + return trans.response.send_redirect( web.url_for( controller='data_admin', + action='monitor_status', + job=jobid ) ) + + @web.expose + @web.require_admin + def monitor_status( self, trans, **kwd ): + params = util.Params( kwd ) + jobid = params.get( 'job', '' ) + chains = params.get( 'chains', [] ) + jobs = self._get_jobs( jobid, trans ) + return trans.fill_template( '/admin/data_admin/download_status.mako', mainjob=jobid, jobs=jobs ) + + @web.expose + @web.require_admin + def ajax_statusupdate( self, trans, **kwd ): + sa_session = trans.app.model.context.current + jobs = [] + params = util.Params( kwd ) + jobid = params.get( 'jobid', '' ) + jobs = self._get_jobs( jobid, trans ) + return trans.fill_template( '/admin/data_admin/ajax_statusupdate.mako', mainjob=jobid, jobs=jobs ) + + def _get_jobs( self, jobid, trans ): + jobs = [] + job = trans.app.job_manager.deferred_job_queue.plugins['GenomeTransferPlugin'].get_job_status( jobid ) + sa_session = trans.app.model.context.current + idxjobs = sa_session.query( model.GenomeIndexToolData ).filter_by( deferred_job_id=job.id, transfer_job_id=job.transfer_job.id ).all() + if job.params[ 'liftover' ] is not None: + for jobid in job.params[ 'liftover' ]: + lo_job = trans.app.job_manager.deferred_job_queue.plugins['LiftOverTransferPlugin'].get_job_status( jobid ) + jobs.append( dict( jobid=lo_job.id, state=lo_job.state, type='Download liftOver' ) ) + for idxjob in idxjobs: + jobentry = sa_session.query( model.Job ).filter_by( id=idxjob.job_id ).first() + jobs.append( dict( jobid=jobentry.id, state=jobentry.state, type='Index Genome' ) ) + jobs.append( dict ( jobid=job.id, state=job.state, type='Main Job' ) ) + jobs.append( dict ( jobid=job.transfer_job.id, state=job.transfer_job.state, type='Download Genome' ) ) + for je in jobs: + je[ 'style' ] = self.jobstyles[ je[ 'state' ] ] + return jobs diff --git a/lib/galaxy/web/controllers/dataset.py b/lib/galaxy/web/controllers/dataset.py index 2c62d359d91..be0bc09dcaf 100644 --- a/lib/galaxy/web/controllers/dataset.py +++ b/lib/galaxy/web/controllers/dataset.py @@ -1,17 +1,22 @@ -import logging, os, string, shutil, re, socket, mimetypes, urllib, tempfile, zipfile, glob, sys +import logging +import mimetypes +import os +import string +import sys +import tempfile +import urllib +import zipfile from galaxy.web.base.controller import * from galaxy.web.framework.helpers import time_ago, iff, grids -from galaxy import util, datatypes, jobs, web, model -from cgi import escape, FieldStorage +from galaxy import util, datatypes, web, model from galaxy.datatypes.display_applications.util import encode_dataset_user, decode_dataset_user from galaxy.util.sanitize_html import sanitize_html from galaxy.util import inflector from galaxy.model.item_attrs import * -from galaxy.model import LibraryDatasetDatasetAssociation, HistoryDatasetAssociation from galaxy.web.framework.helpers import to_unicode -import pkg_resources; +import pkg_resources; pkg_resources.require( "Paste" ) import paste.httpexceptions @@ -32,7 +37,7 @@ try: except RuntimeError: log.exception( "Compression error when testing zip compression. This option will be disabled for library downloads." ) except (TypeError, zipfile.LargeZipFile): # ZIP64 is only in Python2.5+. Remove TypeError when 2.4 support is dropped - log.warning( 'Max zip file size is 2GB, ZIP64 not supported' ) + log.warning( 'Max zip file size is 2GB, ZIP64 not supported' ) comptypes.append( 'zip' ) try: os.unlink( tmpf ) @@ -53,7 +58,7 @@ This is in reference to dataset id ${dataset_id} from history id ${history_id} ----------------------------------------------------------------------------- You should be able to view the history containing the related history item -${hid}: ${history_item_name} +${hid}: ${history_item_name} by logging in as a Galaxy admin user to the Galaxy instance referenced above and pointing your browser to the following link. @@ -90,7 +95,7 @@ class HistoryDatasetAssociationListGrid( grids.Grid ): class HistoryColumn( grids.GridColumn ): def get_value( self, trans, grid, hda): return hda.history.name - + class StatusColumn( grids.GridColumn ): def get_value( self, trans, grid, hda ): if hda.deleted: @@ -111,19 +116,19 @@ class HistoryDatasetAssociationListGrid( grids.Grid ): template='/dataset/grid.mako' default_sort_key = "-update_time" columns = [ - grids.TextColumn( "Name", key="name", + grids.TextColumn( "Name", key="name", # Link name to dataset's history. link=( lambda item: iff( item.history.deleted, None, dict( operation="switch", id=item.id ) ) ), filterable="advanced", attach_popup=True ), - HistoryColumn( "History", key="history", + HistoryColumn( "History", key="history", link=( lambda item: iff( item.history.deleted, None, dict( operation="switch_history", id=item.id ) ) ) ), grids.IndividualTagsColumn( "Tags", key="tags", model_tag_association_class=model.HistoryDatasetAssociationTagAssociation, filterable="advanced", grid_name="HistoryDatasetAssocationListGrid" ), StatusColumn( "Status", key="deleted", attach_popup=False ), grids.GridColumn( "Last Updated", key="update_time", format=time_ago ), ] - columns.append( - grids.MulticolFilterColumn( - "Search", - cols_to_filter=[ columns[0], columns[2] ], + columns.append( + grids.MulticolFilterColumn( + "Search", + cols_to_filter=[ columns[0], columns[2] ], key="free-text-search", visible=False, filterable="standard" ) ) operations = [ @@ -136,17 +141,17 @@ class HistoryDatasetAssociationListGrid( grids.Grid ): num_rows_per_page = 50 def build_initial_query( self, trans, **kwargs ): # Show user's datasets that are not deleted, not in deleted histories, and not hidden. - # To filter HDAs by user, need to join model class/HDA and History table so that it is - # possible to filter by user. However, for dictionary-based filtering to work, need a + # To filter HDAs by user, need to join model class/HDA and History table so that it is + # possible to filter by user. However, for dictionary-based filtering to work, need a # primary table for the query. return trans.sa_session.query( self.model_class ).select_from( self.model_class.table.join( model.History.table ) ) \ .filter( model.History.user == trans.user ) \ .filter( self.model_class.deleted==False ) \ .filter( model.History.deleted==False ) \ .filter( self.model_class.visible==True ) - + class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHistoryDatasetAssociation, UsesItemRatings ): - + stored_list_grid = HistoryDatasetAssociationListGrid() @web.expose @@ -202,7 +207,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist job_stdout=job.stdout, job_info=job.info, job_traceback=job.traceback, - email=email, + email=email, message=message ) frm = to_address # Check email a bit @@ -219,130 +224,45 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist return trans.show_ok_message( "Your error report has been sent" ) except Exception, e: return trans.show_error_message( "An error occurred sending the report by email: %s" % str( e ) ) - + @web.expose def default(self, trans, dataset_id=None, **kwd): return 'This link may not be followed from within Galaxy.' - - @web.expose - def archive_composite_dataset( self, trans, data=None, **kwd ): - # save a composite object into a compressed archive for downloading - params = util.Params( kwd ) - valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' - outfname = data.name[0:150] - outfname = ''.join(c in valid_chars and c or '_' for c in outfname) - if (params.do_action == None): - params.do_action = 'zip' # default - msg = util.restore_text( params.get( 'msg', '' ) ) - messagetype = params.get( 'messagetype', 'done' ) - if not data: - msg = "You must select at least one dataset" - messagetype = 'error' - else: - error = False - try: - if (params.do_action == 'zip'): - # Can't use mkstemp - the file must not exist first - tmpd = tempfile.mkdtemp() - tmpf = os.path.join( tmpd, 'library_download.' + params.do_action ) - if ziptype == '64': - archive = zipfile.ZipFile( tmpf, 'w', zipfile.ZIP_DEFLATED, True ) - else: - archive = zipfile.ZipFile( tmpf, 'w', zipfile.ZIP_DEFLATED ) - archive.add = lambda x, y: archive.write( x, y.encode('CP437') ) - elif params.do_action == 'tgz': - archive = util.streamball.StreamBall( 'w|gz' ) - elif params.do_action == 'tbz': - archive = util.streamball.StreamBall( 'w|bz2' ) - except (OSError, zipfile.BadZipFile): - error = True - log.exception( "Unable to create archive for download" ) - msg = "Unable to create archive for %s for download, please report this error" % outfname - messagetype = 'error' - if not error: - current_user_roles = trans.get_current_user_roles() - ext = data.extension - path = data.file_name - fname = os.path.split(path)[-1] - efp = data.extra_files_path - htmlname = os.path.splitext(outfname)[0] - if not htmlname.endswith(ext): - htmlname = '%s_%s' % (htmlname,ext) - archname = '%s.html' % htmlname # fake the real nature of the html file - try: - archive.add(data.file_name,archname) - except IOError: - error = True - log.exception( "Unable to add composite parent %s to temporary library download archive" % data.file_name) - msg = "Unable to create archive for download, please report this error" - messagetype = 'error' - for root, dirs, files in os.walk(efp): - for fname in files: - fpath = os.path.join(root,fname) - rpath = os.path.relpath(fpath,efp) - try: - archive.add( fpath,rpath ) - except IOError: - error = True - log.exception( "Unable to add %s to temporary library download archive" % rpath) - msg = "Unable to create archive for download, please report this error" - messagetype = 'error' - continue - if not error: - if params.do_action == 'zip': - archive.close() - tmpfh = open( tmpf ) - # CANNOT clean up - unlink/rmdir was always failing because file handle retained to return - must rely on a cron job to clean up tmp - trans.response.set_content_type( "application/x-zip-compressed" ) - trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s.zip"' % outfname - return tmpfh - else: - trans.response.set_content_type( "application/x-tar" ) - outext = 'tgz' - if params.do_action == 'tbz': - outext = 'tbz' - trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s.%s"' % (outfname,outext) - archive.wsgi_status = trans.response.wsgi_status() - archive.wsgi_headeritems = trans.response.wsgi_headeritems() - return archive.stream - return trans.show_error_message( msg ) - + @web.expose def get_metadata_file(self, trans, hda_id, metadata_name): """ Allows the downloading of metadata files associated with datasets (eg. bai index for bam files) """ data = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( trans.security.decode_id( hda_id ) ) if not data or not trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), data.dataset ): return trans.show_error_message( "You are not allowed to access this dataset" ) - + valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' fname = ''.join(c in valid_chars and c or '_' for c in data.name)[0:150] - + file_ext = data.metadata.spec.get(metadata_name).get("file_ext", metadata_name) trans.response.headers["Content-Type"] = "application/octet-stream" trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy%s-[%s].%s"' % (data.hid, fname, file_ext) return open(data.metadata.get(metadata_name).file_name) - - def _check_dataset(self, trans, dataset_id): + + def _check_dataset(self, trans, hda_id): # DEPRECATION: We still support unencoded ids for backward compatibility try: - data = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( trans.security.decode_id( dataset_id ) ) + data = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( trans.security.decode_id( hda_id) ) if data is None: - raise ValueError( 'Invalid reference dataset id: %s.' % dataset_id ) + raise ValueError( 'Invalid reference dataset id: %s.' % hda_id) except: try: - data = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( int( dataset_id ) ) + data = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( int( hda_id ) ) except: data = None if not data: - raise paste.httpexceptions.HTTPRequestRangeNotSatisfiable( "Invalid reference dataset id: %s." % str( dataset_id ) ) + raise paste.httpexceptions.HTTPRequestRangeNotSatisfiable( "Invalid reference dataset id: %s." % str( hda_id ) ) if not trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), data.dataset ): return trans.show_error_message( "You are not allowed to access this dataset" ) - if data.state == trans.model.Dataset.states.UPLOAD: return trans.show_error_message( "Please wait until this dataset finishes uploading before attempting to view it." ) - return data - + @web.expose @web.json def transfer_status(self, trans, dataset_id, filename=None): @@ -352,7 +272,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist if isinstance( data, basestring ): return data log.debug( "Checking transfer status for dataset %s..." % data.dataset.id ) - + # Pulling files in extra_files_path into cache is not handled via this # method but that's primarily because those files are typically linked to # through tool's output page anyhow so tying a JavaScript event that will @@ -361,63 +281,11 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist return True else: return trans.app.object_store.file_ready(data.dataset) - + @web.expose - def display(self, trans, dataset_id=None, preview=False, filename=None, to_ext=None, **kwd): - """Catches the dataset id and displays file contents as directed""" - composite_extensions = trans.app.datatypes_registry.get_composite_extensions( ) - composite_extensions.append('html') # for archiving composite datatypes + def display(self, trans, dataset_id=None, preview=False, filename=None, to_ext=None, chunk=None, **kwd): data = self._check_dataset(trans, dataset_id) - if isinstance( data, basestring ): - return data - - if filename and filename != "index": - # For files in extra_files_path - file_path = trans.app.object_store.get_filename(data.dataset, extra_dir='dataset_%s_files' % data.dataset.id, alt_name=filename) - if os.path.exists( file_path ): - if os.path.isdir( file_path ): - return trans.show_error_message( "Directory listing is not allowed." ) #TODO: Reconsider allowing listing of directories? - mime, encoding = mimetypes.guess_type( file_path ) - if not mime: - try: - mime = trans.app.datatypes_registry.get_mimetype_by_extension( ".".split( file_path )[-1] ) - except: - mime = "text/plain" - trans.response.set_content_type( mime ) - return open( file_path ) - else: - return trans.show_error_message( "Could not find '%s' on the extra files path %s." % ( filename, file_path ) ) - - trans.response.set_content_type(data.get_mime()) - trans.log_event( "Display dataset id: %s" % str( dataset_id ) ) - - if to_ext or isinstance(data.datatype, datatypes.binary.Binary): # Saving the file, or binary file - if data.extension in composite_extensions: - return self.archive_composite_dataset( trans, data, **kwd ) - else: - trans.response.headers['Content-Length'] = int( os.stat( data.file_name ).st_size ) - if not to_ext: - to_ext = data.extension - valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' - fname = ''.join(c in valid_chars and c or '_' for c in data.name)[0:150] - trans.response.set_content_type( "application/octet-stream" ) #force octet-stream so Safari doesn't append mime extensions to filename - trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy%s-[%s].%s"' % (data.hid, fname, to_ext) - return open( data.file_name ) - if not os.path.exists( data.file_name ): - raise paste.httpexceptions.HTTPNotFound( "File Not Found (%s)." % data.file_name ) - max_peek_size = 1000000 # 1 MB - if isinstance(data.datatype, datatypes.images.Html): - max_peek_size = 10000000 # 10 MB for html - if not preview or isinstance(data.datatype, datatypes.images.Image) or os.stat( data.file_name ).st_size < max_peek_size: - if trans.app.config.sanitize_all_html and trans.response.get_content_type() == "text/html": - # Sanitize anytime we respond with plain text/html content. - return sanitize_html(open( data.file_name ).read()) - return open( data.file_name ) - else: - trans.response.set_content_type( "text/html" ) - return trans.stream_template_mako( "/dataset/large_file.mako", - truncated_data = open( data.file_name ).read(max_peek_size), - data = data ) + return data.datatype.display_data(trans, data, preview, filename, to_ext, chunk, **kwd) @web.expose def edit(self, trans, dataset_id=None, filename=None, hid=None, **kwd): @@ -443,7 +311,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist # TODO: hid handling data = history.datasets[ int( hid ) - 1 ] id = None - elif dataset_id is not None: + elif dataset_id is not None: id = trans.app.security.decode_id( dataset_id ) data = trans.sa_session.query( self.app.model.HistoryDatasetAssociation ).get( id ) else: @@ -463,7 +331,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist # permission. In this case, we'll reset this permission to the hda user's private role. manage_permissions_action = trans.app.security_agent.get_action( trans.app.security_agent.permitted_actions.DATASET_MANAGE_PERMISSIONS.action ) permissions = { manage_permissions_action : [ trans.app.security_agent.get_private_user_role( data.history.user ) ] } - trans.app.security_agent.set_dataset_permission( data.dataset, permissions ) + trans.app.security_agent.set_dataset_permission( data.dataset, permissions ) if trans.app.security_agent.can_access_dataset( current_user_roles, data.dataset ): if data.state == trans.model.Dataset.states.UPLOAD: return trans.show_error_message( "Please wait until this dataset finishes uploading before attempting to edit its metadata." ) @@ -600,7 +468,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist refresh_frames=refresh_frames ) else: return trans.show_error_message( "You do not have permission to edit this dataset's ( id: %s ) information." % str( dataset_id ) ) - + @web.expose @web.require_login( "see all available datasets" ) def list( self, trans, **kwargs ): @@ -610,7 +478,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist if 'operation' in kwargs: operation = kwargs['operation'].lower() hda_ids = util.listify( kwargs.get( 'id', [] ) ) - + # Display no message by default status, message = None, None @@ -630,15 +498,15 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist if hdas: if operation == "switch" or operation == "switch_history": # Switch to a history that the HDA resides in. - + # Convert hda to histories. histories = [] for hda in hdas: histories.append( hda.history ) - + # Use history controller to switch the history. TODO: is this reasonable? status, message = trans.webapp.controllers['history']._list_switch( trans, histories ) - + # Current history changed, refresh history frame; if switching to a dataset, set hda seek. trans.template_context['refresh_frames'] = ['history'] if operation == "switch": @@ -648,35 +516,35 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist # Copy a dataset to the current history. target_histories = [ trans.get_history() ] status, message = self._copy_datasets( trans, hda_ids, target_histories ) - + # Current history changed, refresh history frame. trans.template_context['refresh_frames'] = ['history'] # Render the list view return self.stored_list_grid( trans, status=status, message=message, **kwargs ) - + @web.expose def imp( self, trans, dataset_id=None, **kwd ): """ Import another user's dataset via a shared URL; dataset is added to user's current history. """ msg = "" - + # Set referer message. referer = trans.request.referer if referer is not "": referer_message = "return to the previous page" % referer else: referer_message = "go to Galaxy's start page" % url_for( '/' ) - + # Error checking. if not dataset_id: return trans.show_error_message( "You must specify a dataset to import. You can %s." % referer_message, use_panels=True ) - + # Do import. cur_history = trans.get_history( create=True ) status, message = self._copy_datasets( trans, [ dataset_id ], [ cur_history ], imported=True ) message = "Dataset imported.
You can start using the dataset or %s." % ( url_for('/'), referer_message ) return trans.show_message( message, type=status, use_panels=True ) - + @web.expose @web.json @web.require_login( "use Galaxy datasets" ) @@ -685,7 +553,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist dataset = self.get_dataset( trans, id, False, True ) return_dict = { "name" : dataset.name, "link" : url_for( action="display_by_username_and_slug", username=dataset.history.user.username, slug=trans.security.encode_id( dataset.id ) ) } return return_dict - + @web.expose def get_embed_html_async( self, trans, id ): """ Returns HTML for embedding a dataset in a page. """ @@ -698,7 +566,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist def set_accessible_async( self, trans, id=None, accessible=False ): """ Does nothing because datasets do not have an importable/accessible attribute. This method could potentially set another attribute. """ return - + @web.expose @web.require_login( "rate items" ) @web.json @@ -713,7 +581,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist dataset_rating = self.rate_item( rate_item, trans.get_user(), dataset, rating ) return self.get_ave_item_rating_data( trans.sa_session, dataset ) - + @web.expose def display_by_username_and_slug( self, trans, username, slug, filename=None, preview=True ): """ Display dataset by username and slug; because datasets do not yet have slugs, the slug is the dataset's id. """ @@ -722,10 +590,10 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist # Filename used for composite types. if filename: return self.display( trans, dataset_id=slug, filename=filename) - + truncated, dataset_data = self.get_data( dataset, preview ) dataset.annotation = self.get_item_annotation_str( trans.sa_session, dataset.history.user, dataset ) - + # If data is binary or an image, stream without template; otherwise, use display template. # TODO: figure out a way to display images in display template. if isinstance(dataset.datatype, datatypes.binary.Binary) or isinstance(dataset.datatype, datatypes.images.Image) or isinstance(dataset.datatype, datatypes.images.Html): @@ -741,12 +609,12 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist else: user_item_rating = 0 ave_item_rating, num_ratings = self.get_ave_item_rating_data( trans.sa_session, dataset ) - + return trans.fill_template_mako( "/dataset/display.mako", item=dataset, item_data=dataset_data, truncated=truncated, user_item_rating = user_item_rating, ave_item_rating=ave_item_rating, num_ratings=num_ratings ) else: raise web.httpexceptions.HTTPNotFound() - + @web.expose def get_item_content_async( self, trans, id ): """ Returns item content in HTML format. """ @@ -758,7 +626,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist # Get annotation. dataset.annotation = self.get_item_annotation_str( trans.sa_session, trans.user, dataset ) return trans.stream_template_mako( "/dataset/item_content.mako", item=dataset, item_data=dataset_data, truncated=truncated ) - + @web.expose def annotate_async( self, trans, id, new_annotation=None, **kwargs ): dataset = self.get_dataset( trans, id, False, True ) @@ -770,7 +638,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist self.add_item_annotation( trans.sa_session, trans.get_user(), dataset, new_annotation ) trans.sa_session.flush() return new_annotation - + @web.expose def get_annotation_async( self, trans, id ): dataset = self.get_dataset( trans, id, False, True ) @@ -841,7 +709,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist if app_action in [ 'data', 'param' ]: assert action_param, "An action param must be provided for a data or param action" #data is used for things with filenames that could be passed off to a proxy - #in case some display app wants all files to be in the same 'directory', + #in case some display app wants all files to be in the same 'directory', #data can be forced to param, but not the other way (no filename for other direction) #get param name from url param name try: @@ -904,8 +772,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist if job.state in [ self.app.model.Job.states.QUEUED, self.app.model.Job.states.RUNNING, self.app.model.Job.states.NEW ]: # Are *all* of the job's other output datasets deleted? if job.check_if_output_datasets_deleted(): - job.mark_deleted( self.app.config.get_bool( 'enable_job_running', True ), - self.app.config.get_bool( 'track_jobs_in_database', False ) ) + job.mark_deleted( self.app.config.track_jobs_in_database ) self.app.job_manager.job_stop_queue.put( job.id ) trans.sa_session.flush() except Exception, e: @@ -961,7 +828,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist trans.log_event( "Dataset id %s has been unhidden" % str(id) ) return True return False - + def _purge( self, trans, dataset_id ): message = None status = 'done' @@ -1038,7 +905,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist return "OK" else: raise Exception( message ) - + @web.expose def unhide( self, trans, dataset_id, filename ): if self._unhide( trans, dataset_id ): @@ -1071,7 +938,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist """ Show the parameters used for an HDA """ - + def source_dataset_chain( dataset, lst ): try: cp_from_ldda = dataset.copied_from_library_dataset_dataset_association @@ -1085,13 +952,13 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist except: pass return lst - + hda = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( trans.security.decode_id( dataset_id ) ) if not hda: raise paste.httpexceptions.HTTPRequestRangeNotSatisfiable( "Invalid reference dataset id: %s." % str( dataset_id ) ) if not trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), hda.dataset ): return trans.show_error_message( "You are not allowed to access this dataset" ) - + # Get the associated job, if any. If this hda was copied from another, # we need to find the job that created the origial hda params_objects = None @@ -1103,7 +970,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist job = None for assoc in job_hda.creating_job_associations: job = assoc.job - break + break if job: # Get the tool object try: @@ -1114,10 +981,10 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist params_objects = job.get_param_values( trans.app ) except: pass - + inherit_chain = source_dataset_chain(hda, []) return trans.fill_template( "show_params.mako", inherit_chain=inherit_chain, history=trans.get_history(), hda=hda, tool=tool, params_objects=params_objects ) - + @web.expose def copy_datasets( self, trans, source_history=None, source_dataset_ids="", target_history_id=None, target_history_ids="", new_history_name="", do_copy=False, **kwd ): params = util.Params( kwd ) @@ -1176,7 +1043,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist if history in target_histories: refresh_frames = ['history'] trans.sa_session.flush() - hist_names_str = ", ".join( ['%s' % + hist_names_str = ", ".join( ['%s' % ( url_for( controller="history", action="switch_to_history", \ hist_id=trans.security.encode_id( hist.id ) ), hist.name ) \ for hist in target_histories ] ) @@ -1187,7 +1054,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist source_datasets = history.visible_datasets target_histories = [history] if user: - target_histories = user.active_histories + target_histories = user.active_histories return trans.fill_template( "/dataset/copy_view.mako", source_history = history, current_history = trans.get_history(), @@ -1205,7 +1072,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist """ Helper method for copying datasets. """ user = trans.get_user() done_msg = error_msg = "" - + invalid_datasets = 0 if not dataset_ids or not target_histories: error_msg = "You must provide both source datasets and target histories." @@ -1230,7 +1097,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHist done_msg = "%i dataset%s copied to %i histor%s." % \ ( num_datasets_copied, iff( num_datasets_copied == 1, "", "s"), len( target_histories ), iff( len ( target_histories ) == 1, "y", "ies") ) trans.sa_session.refresh( history ) - + if error_msg != "": status = ERROR message = error_msg diff --git a/lib/galaxy/web/controllers/requests_admin.py b/lib/galaxy/web/controllers/requests_admin.py index 9903a314f68..683e3618b9d 100644 --- a/lib/galaxy/web/controllers/requests_admin.py +++ b/lib/galaxy/web/controllers/requests_admin.py @@ -199,6 +199,11 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitions ): params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) + sample_id = params.get( 'sample_id', None ) + try: + sample = trans.sa_session.query( trans.model.Sample ).get( trans.security.decode_id ( sample_id ) ) + except: + return invalid_id_redirect( trans, 'requests_admin', sample_id, 'sample' ) if 'operation' in kwd: operation = kwd[ 'operation' ].lower() sample_dataset_id = params.get( 'id', None ) @@ -269,11 +274,6 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitions ): **kwd ) ) # Render the grid view - sample_id = params.get( 'sample_id', None ) - try: - sample = trans.sa_session.query( trans.model.Sample ).get( trans.security.decode_id ( sample_id ) ) - except: - return invalid_id_redirect( trans, 'requests_admin', sample_id, 'sample' ) request_id = trans.security.encode_id( sample.request.id ) library_id = trans.security.encode_id( sample.library.id ) self.datatx_grid.title = 'Manage "%s" datasets' % sample.name @@ -493,6 +493,7 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitions ): return trans.response.send_redirect( web.url_for( controller='requests_admin', action='select_datasets_to_transfer', request_id=trans.security.encode_id( request.id ), + external_service_id=trans.security.encode_id( external_service.id ), status=status, message=message ) ) def __create_sample_datasets( self, trans, sample, selected_datasets_to_transfer, external_service ): @@ -624,9 +625,6 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitions ): or not scp_configs.get( 'user_name', '' ) \ or not scp_configs.get( 'password', '' ): err_msg += "Error in external service login information. " - # Make sure web API is enabled and API key exists - if not trans.app.config.enable_api: - err_msg += "The 'enable_api = True' setting is not correctly set in the Galaxy config file. " if not trans.user.api_keys: err_msg += "Set your API Key in your User Preferences to transfer datasets. " # Check if library_import_dir is set diff --git a/lib/galaxy/web/controllers/requests_common.py b/lib/galaxy/web/controllers/requests_common.py index 5e989154baf..88bbc3320f8 100644 --- a/lib/galaxy/web/controllers/requests_common.py +++ b/lib/galaxy/web/controllers/requests_common.py @@ -1011,6 +1011,8 @@ class RequestsCommon( BaseUIController, UsesFormDefinitions ): sample = trans.sa_session.query( trans.model.Sample ).get( trans.security.decode_id( sample_id ) ) except: return invalid_id_redirect( trans, cntrller, sample_id, 'sample' ) + external_service_id = params.get( 'external_service_id', None ) + external_service = trans.sa_session.query( trans.model.ExternalService ).get( trans.security.decode_id( external_service_id ) ) # See if a library and folder have been set for this sample. if is_admin and not sample.library or not sample.folder: status = 'error' @@ -1043,6 +1045,7 @@ class RequestsCommon( BaseUIController, UsesFormDefinitions ): return trans.fill_template( '/requests/common/view_sample_datasets.mako', cntrller=cntrller, title=title, + external_service=external_service, sample=sample, sample_datasets=sample_datasets, transfer_status=transfer_status, diff --git a/lib/galaxy/web/controllers/root.py b/lib/galaxy/web/controllers/root.py index f70511550a0..c0c566f58de 100644 --- a/lib/galaxy/web/controllers/root.py +++ b/lib/galaxy/web/controllers/root.py @@ -250,7 +250,7 @@ class RootController( BaseUIController, UsesHistory, UsesAnnotations ): valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' fname = data.name fname = ''.join(c in valid_chars and c or '_' for c in fname)[0:150] - trans.response.headers["Content-Disposition"] = "attachment; filename=GalaxyHistoryItem-%s-[%s]%s" % (data.hid, fname, toext) + trans.response.headers["Content-Disposition"] = 'attachment; filename="GalaxyHistoryItem-%s-[%s]%s"' % (data.hid, fname, toext) trans.log_event( "Display dataset id: %s" % str(id) ) try: return open( data.file_name ) diff --git a/lib/galaxy/web/controllers/tool_runner.py b/lib/galaxy/web/controllers/tool_runner.py index 971d94cf27b..b43e7472e44 100644 --- a/lib/galaxy/web/controllers/tool_runner.py +++ b/lib/galaxy/web/controllers/tool_runner.py @@ -58,6 +58,8 @@ class ToolRunner( BaseUIController ): log.error( "index called with tool id '%s' but no such tool exists", tool_id ) trans.log_event( "Tool id '%s' does not exist" % tool_id ) return "Tool '%s' does not exist, kwd=%s " % (tool_id, kwd) + if tool.require_login and not trans.user: + return trans.response.send_redirect( url_for( controller='user', action='login', cntrller='user', message="You must be logged in to use this tool.", status="info", redirect=url_for( controller='/tool_runner', action='index', tool_id=tool_id, **kwd ) ) ) params = util.Params( kwd, sanitize = False ) #Sanitize parameters when substituting into command line via input wrappers #do param translation here, used by datasource tools if tool.input_translator: diff --git a/lib/galaxy/web/controllers/tracks.py b/lib/galaxy/web/controllers/tracks.py index b6dc56249f3..a20483130f1 100644 --- a/lib/galaxy/web/controllers/tracks.py +++ b/lib/galaxy/web/controllers/tracks.py @@ -486,7 +486,7 @@ class TracksController( BaseUIController, UsesVisualization, UsesHistoryDatasetA data = None # TODO: for raw data requests, map dataset type to provider using dict in data_providers.py if isinstance( dataset.datatype, Gff ): - data = GFFDataProvider( original_dataset=dataset ).get_data( chrom, low, high, **kwargs ) + data = RawGFFDataProvider( original_dataset=dataset ).get_data( chrom, low, high, **kwargs ) data[ 'dataset_type' ] = 'interval_index' data[ 'extra_info' ] = None elif isinstance( dataset.datatype, Bed ): @@ -932,6 +932,7 @@ class TracksController( BaseUIController, UsesVisualization, UsesHistoryDatasetA # Set input datasets for tool. If running on region, extract and use subset # when possible. # + location = "%s:%i-%i" % ( chrom, low, high ) for jida in original_job.input_datasets: # If param set previously by config actions, do nothing. if jida.name in params_set: @@ -943,48 +944,67 @@ class TracksController( BaseUIController, UsesVisualization, UsesHistoryDatasetA elif run_on_region and hasattr( input_dataset.datatype, 'get_track_type' ): # Dataset is indexed and hence a subset can be extracted and used # as input. - track_type, data_sources = input_dataset.datatype.get_track_type() - data_source = data_sources[ 'data' ] - converted_dataset = input_dataset.get_converted_dataset( trans, data_source ) - deps = input_dataset.get_converted_dataset_deps( trans, data_source ) - - # Create new HDA for input dataset's subset. - new_dataset = trans.app.model.HistoryDatasetAssociation( extension=input_dataset.ext, \ - dbkey=input_dataset.dbkey, \ - create_dataset=True, \ - sa_session=trans.sa_session, - name="Subset [%s:%i-%i] of data %i" % \ - ( chrom, low, high, input_dataset.hid ), - visible=False ) - target_history.add_dataset( new_dataset ) - trans.sa_session.add( new_dataset ) - trans.app.security_agent.set_all_dataset_permissions( new_dataset.dataset, hda_permissions ) - - # Write subset of data to new dataset - data_provider_class = get_data_provider( original_dataset=input_dataset ) - data_provider = data_provider_class( original_dataset=input_dataset, - converted_dataset=converted_dataset, - dependencies=deps ) - trans.app.object_store.create( new_dataset.dataset ) - data_provider.write_data_to_file( chrom, low, high, new_dataset.file_name ) - - # TODO: (a) size not working; (b) need to set peek. - new_dataset.set_size() - new_dataset.info = "Data subset for trackster" - new_dataset.set_dataset_state( trans.app.model.Dataset.states.OK ) - # Set metadata. - if trans.app.config.set_metadata_externally: - trans.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( trans.app.datatypes_registry.set_external_metadata_tool, trans, incoming = { 'input1':new_dataset }, overwrite=False ) + # Look for subset. + subset_dataset_association = trans.sa_session.query( trans.app.model.HistoryDatasetAssociationSubset ) \ + .filter_by( hda=input_dataset, location=location ) \ + .first() + if subset_dataset_association: + # Data subset exists. + subset_dataset = subset_dataset_association.subset else: - message = 'Attributes updated' - new_dataset.set_meta() - new_dataset.datatype.after_setting_metadata( new_dataset ) + # Need to create subset. + track_type, data_sources = input_dataset.datatype.get_track_type() + data_source = data_sources[ 'data' ] + converted_dataset = input_dataset.get_converted_dataset( trans, data_source ) + deps = input_dataset.get_converted_dataset_deps( trans, data_source ) + + # Create new HDA for input dataset's subset. + new_dataset = trans.app.model.HistoryDatasetAssociation( extension=input_dataset.ext, \ + dbkey=input_dataset.dbkey, \ + create_dataset=True, \ + sa_session=trans.sa_session, + name="Subset [%s] of data %i" % \ + ( location, input_dataset.hid ), + visible=False ) + target_history.add_dataset( new_dataset ) + trans.sa_session.add( new_dataset ) + trans.app.security_agent.set_all_dataset_permissions( new_dataset.dataset, hda_permissions ) + + # Write subset of data to new dataset + data_provider_class = get_data_provider( original_dataset=input_dataset ) + data_provider = data_provider_class( original_dataset=input_dataset, + converted_dataset=converted_dataset, + dependencies=deps ) + trans.app.object_store.create( new_dataset.dataset ) + data_provider.write_data_to_file( chrom, low, high, new_dataset.file_name ) + + # TODO: (a) size not working; (b) need to set peek. + new_dataset.set_size() + new_dataset.info = "Data subset for trackster" + new_dataset.set_dataset_state( trans.app.model.Dataset.states.OK ) + + # Set metadata. + # TODO: set meta internally if dataset is small enough? + if trans.app.config.set_metadata_externally: + trans.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( trans.app.datatypes_registry.set_external_metadata_tool, + trans, incoming = { 'input1':new_dataset }, + overwrite=False, job_params={ "source" : "trackster" } ) + else: + message = 'Attributes updated' + new_dataset.set_meta() + new_dataset.datatype.after_setting_metadata( new_dataset ) + + # Add HDA subset association. + subset_association = trans.app.model.HistoryDatasetAssociationSubset( hda=input_dataset, subset=new_dataset, location=location ) + trans.sa_session.add( subset_association ) + + subset_dataset = new_dataset trans.sa_session.flush() # Add dataset to tool's parameters. - if not set_param_value( tool_params, jida.name, new_dataset ): + if not set_param_value( tool_params, jida.name, subset_dataset ): return to_json_string( { "error" : True, "message" : "error setting parameter %s" % jida.name } ) # diff --git a/lib/galaxy/web/controllers/user.py b/lib/galaxy/web/controllers/user.py index 3e3a11fdde9..03e49ba90cf 100644 --- a/lib/galaxy/web/controllers/user.py +++ b/lib/galaxy/web/controllers/user.py @@ -10,8 +10,7 @@ from random import choice from galaxy.web.form_builder import * from galaxy.util.json import from_json_string, to_json_string from galaxy.web.framework.helpers import iff -from galaxy.security.validate_user_input import validate_email, validate_publicname, validate_password -from galaxy.openid.providers import OpenIDProvider +from galaxy.security.validate_user_input import validate_email, validate_publicname, validate_password, transform_publicname log = logging.getLogger( __name__ ) @@ -30,7 +29,7 @@ class UserOpenIDGrid( grids.Grid ): default_filter = { "openid" : "All" } default_sort_key = "-create_time" columns = [ - grids.TextColumn( "OpenID URL", key="openid" ), + grids.TextColumn( "OpenID URL", key="openid", link=( lambda x: dict( action='openid_auth', login_button="Login", openid_url=x.openid if not x.provider else '', openid_provider=x.provider, auto_associate=True ) ) ), grids.GridColumn( "Created", key="create_time", format=time_ago ), ] operations = [ @@ -48,32 +47,30 @@ class User( BaseUIController, UsesFormDefinitions ): return trans.fill_template( '/user/index.mako', cntrller=cntrller, webapp=webapp ) @web.expose def openid_auth( self, trans, webapp='galaxy', **kwd ): + '''Handles user request to access an OpenID provider''' if not trans.app.config.enable_openid: return trans.show_error_message( 'OpenID authentication is not enabled in this instance of Galaxy' ) message = 'Unspecified failure authenticating via OpenID' status = kwd.get( 'status', 'done' ) openid_url = kwd.get( 'openid_url', '' ) openid_provider = kwd.get( 'openid_provider', '' ) - referer = kwd.get( 'referer', trans.request.referer ) + if not openid_provider or openid_url: + openid_provider = trans.app.openid_providers.NO_PROVIDER_ID #empty fields cause validation errors + redirect = kwd.get( 'redirect', '' ).strip() auto_associate = util.string_as_bool( kwd.get( 'auto_associate', False ) ) use_panels = util.string_as_bool( kwd.get( 'use_panels', False ) ) action = 'login' - if auto_associate: - action = 'openid_manage' - if not referer: - referer = url_for( '/' ) consumer = trans.app.openid_manager.get_consumer( trans ) - openid_provider_obj = None - if not openid_url and openid_provider and trans.app.openid_providers.get( openid_provider ): - openid_provider_obj = trans.app.openid_providers.get( openid_provider ) - elif openid_url: - openid_provider_obj = OpenIDProvider( openid_url, openid_url, openid_url ) #for manually entered links use the link for id, name and url - elif openid_provider: - message = 'Invalid OpenID provider specified: %s' % ( openid_provider ) + if openid_url: + openid_provider_obj = trans.app.openid_providers.new_provider_from_identifier( openid_url ) else: + openid_provider_obj = trans.app.openid_providers.get( openid_provider ) + if not openid_url and openid_provider == trans.app.openid_providers.NO_PROVIDER_ID: message = 'An OpenID provider was not specified' - process_url = trans.request.base.rstrip( '/' ) + url_for( controller='user', action='openid_process', referer=referer, auto_associate=auto_associate, openid_provider=openid_provider ) - if openid_provider_obj is not None: + elif openid_provider_obj: + if not redirect: + redirect = ' ' + process_url = trans.request.base.rstrip( '/' ) + url_for( controller='user', action='openid_process', redirect=redirect, openid_provider=openid_provider, auto_associate=auto_associate ) #None of these values can be empty, or else a verification error will occur request = None try: request = consumer.begin( openid_provider_obj.op_endpoint_url ) @@ -87,84 +84,100 @@ class User( BaseUIController, UsesFormDefinitions ): redirect_url = request.redirectURL( trans.request.base, process_url ) trans.app.openid_manager.persist_session( trans, consumer ) - trans.response.send_redirect( redirect_url ) - return + return trans.response.send_redirect( redirect_url ) else: form = request.htmlMarkup( trans.request.base, process_url, form_tag_attrs={'id':'openid_message','target':'_top'} ) trans.app.openid_manager.persist_session( trans, consumer ) return form return trans.response.send_redirect( url_for( controller='user', action=action, + redirect=redirect, use_panels=use_panels, message=message, status='error' ) ) @web.expose def openid_process( self, trans, webapp='galaxy', **kwd ): + '''Handle's response from OpenID Providers''' if not trans.app.config.enable_openid: return trans.show_error_message( 'OpenID authentication is not enabled in this instance of Galaxy' ) auto_associate = util.string_as_bool( kwd.get( 'auto_associate', False ) ) action = 'login' - if auto_associate: + if trans.user: action = 'openid_manage' if trans.app.config.support_url is not None: contact = 'support' % trans.app.config.support_url else: contact = 'support' - message = 'Verification failed for an unknown reason. Please contact support for assistance.' + message = 'Verification failed for an unknown reason. Please contact %s for assistance.' % ( contact ) status = 'error' consumer = trans.app.openid_manager.get_consumer( trans ) info = consumer.complete( kwd, trans.request.url ) display_identifier = info.getDisplayIdentifier() - redirect_url = kwd.get( 'referer', url_for( '/' ) ) - openid_provider = kwd.get( 'openid_provider', '' ) + redirect = kwd.get( 'redirect', '' ).strip() + openid_provider = kwd.get( 'openid_provider', None ) if info.status == trans.app.openid_manager.FAILURE and display_identifier: message = "Login via OpenID failed. The technical reason for this follows, please include this message in your email if you need to %s to resolve this problem: %s" % ( contact, info.message ) return trans.response.send_redirect( url_for( controller='user', action=action, use_panels=True, + redirect=redirect, message=message, status='error' ) ) elif info.status == trans.app.openid_manager.SUCCESS: if info.endpoint.canonicalID: display_identifier = info.endpoint.canonicalID - user_openid = trans.sa_session.query( trans.app.model.UserOpenID ).filter( trans.app.model.UserOpenID.table.c.openid == display_identifier ).first() openid_provider_obj = trans.app.openid_providers.get( openid_provider ) + user_openid = trans.sa_session.query( trans.app.model.UserOpenID ).filter( trans.app.model.UserOpenID.table.c.openid == display_identifier ).first() + if not openid_provider_obj and user_openid and user_openid.provider: + openid_provider_obj = trans.app.openid_providers.get( user_openid.provider ) if not openid_provider_obj: - openid_provider_obj = OpenIDProvider( display_identifier, display_identifier, display_identifier ) + openid_provider_obj = trans.app.openid_providers.new_provider_from_identifier( display_identifier ) if not user_openid: user_openid = trans.app.model.UserOpenID( session=trans.galaxy_session, openid=display_identifier ) - elif not user_openid.user and user_openid.session.id != trans.galaxy_session.id: + if not user_openid.user: user_openid.session = trans.galaxy_session - elif user_openid.user and not auto_associate: - trans.handle_user_login( user_openid.user, webapp ) - trans.log_event( "User logged in via OpenID: %s" % display_identifier ) - openid_provider_obj.post_authentication( trans, trans.app.openid_manager, info ) - trans.response.send_redirect( redirect_url ) - return - if auto_associate and trans.user: - # The user is already logged in and requested association from - # the user prefs as opposed to using the OpenID form on the - # login page. + if not user_openid.provider and openid_provider: + user_openid.provider = openid_provider + if trans.user: if user_openid.user and user_openid.user.id != trans.user.id: message = "The OpenID %s is already associated with another Galaxy account, %s. Please disassociate it from that account before attempting to associate it with a new account." % ( display_identifier, user_openid.user.email ) status = "error" - elif user_openid.user and user_openid.user.id == trans.user.id: - message = "The OpenID %s is already associated with your Galaxy account, %s." % ( display_identifier, trans.user.email ) - status = "warning" - else: - user_openid.user_id = trans.user.id - trans.sa_session.add( user_openid ) - trans.sa_session.flush() - trans.log_event( "User associated OpenID: %s" % display_identifier ) - message = "The OpenID %s has been associated with your Galaxy account, %s." % ( display_identifier, trans.user.email ) - status = "done" + elif not user_openid.user or user_openid.user == trans.user: + if openid_provider_obj.id: + user_openid.provider = openid_provider_obj.id + user_openid.session = trans.galaxy_session + if not openid_provider_obj.never_associate_with_user: + if not auto_associate and ( user_openid.user and user_openid.user.id == trans.user.id ): + message = "The OpenID %s is already associated with your Galaxy account, %s." % ( display_identifier, trans.user.email ) + status = "warning" + else: + message = "The OpenID %s has been associated with your Galaxy account, %s." % ( display_identifier, trans.user.email ) + status = "done" + user_openid.user = trans.user + trans.sa_session.add( user_openid ) + trans.sa_session.flush() + trans.log_event( "User associated OpenID: %s" % display_identifier ) + else: + message = "The OpenID %s cannot be used to log into your Galaxy account, but any post authentication actions have been performed." % ( openid_provider_obj.name ) + status ="info" openid_provider_obj.post_authentication( trans, trans.app.openid_manager, info ) - trans.response.send_redirect( url_for( controller='user', + if redirect: + message = '%s
Click here to return to the page you were previously viewing.' % ( message, redirect ) + if redirect and status != "error": + return trans.response.send_redirect( redirect ) + return trans.response.send_redirect( url_for( controller='user', action='openid_manage', use_panels=True, + redirect=redirect, message=message, status=status ) ) - return + elif user_openid.user: + trans.handle_user_login( user_openid.user, webapp ) + trans.log_event( "User logged in via OpenID: %s" % display_identifier ) + openid_provider_obj.post_authentication( trans, trans.app.openid_manager, info ) + if not redirect: + redirect = url_for( '/' ) + return trans.response.send_redirect( redirect ) trans.sa_session.add( user_openid ) trans.sa_session.flush() message = "OpenID authentication was successful, but you need to associate your OpenID with a Galaxy account." @@ -179,10 +192,11 @@ class User( BaseUIController, UsesFormDefinitions ): email = sreg_resp.get( sreg_email_name, '' ) except AttributeError: email = '' - trans.response.send_redirect( url_for( controller='user', + #OpenID success, but user not logged in, and not previously associated + return trans.response.send_redirect( url_for( controller='user', action='openid_associate', - openid_provider=openid_provider, use_panels=True, + redirect=redirect, username=username, email=email, message=message, @@ -198,10 +212,12 @@ class User( BaseUIController, UsesFormDefinitions ): return trans.response.send_redirect( url_for( controller='user', action=action, use_panels=True, + redirect=redirect, message=message, status=status ) ) @web.expose def openid_associate( self, trans, cntrller='user', webapp='galaxy', **kwd ): + '''Associates a user with an OpenID log in''' if not trans.app.config.enable_openid: return trans.show_error_message( 'OpenID authentication is not enabled in this instance of Galaxy' ) use_panels = util.string_as_bool( kwd.get( 'use_panels', False ) ) @@ -209,9 +225,7 @@ class User( BaseUIController, UsesFormDefinitions ): status = kwd.get( 'status', 'done' ) email = kwd.get( 'email', '' ) username = kwd.get( 'username', '' ) - referer = kwd.get( 'referer', trans.request.referer ) - openid_provider = kwd.get( 'openid_provider', '' ) - openid_provider_obj = trans.app.openid_providers.get( openid_provider ) + redirect = kwd.get( 'redirect', '' ).strip() params = util.Params( kwd ) is_admin = cntrller == 'admin' and trans.user_is_admin() openids = trans.galaxy_session.openids @@ -223,18 +237,38 @@ class User( BaseUIController, UsesFormDefinitions ): if kwd.get( 'login_button', False ): message, status, user, success = self.__validate_login( trans, webapp, **kwd ) if success: + openid_objs = [] for openid in openids: - openid.user = user - trans.sa_session.add( openid ) + openid_provider_obj = trans.app.openid_providers.get( openid.provider ) + if not openid_provider_obj or not openid_provider_obj.never_associate_with_user: + openid.user = user + trans.sa_session.add( openid ) + trans.log_event( "User associated OpenID: %s" % openid.openid ) + if openid_provider_obj and openid_provider_obj.has_post_authentication_actions(): + openid_objs.append( openid_provider_obj ) trans.sa_session.flush() - for openid in openids: - trans.log_event( "User associated OpenID: %s" % openid.openid ) - redirect_url = referer - if not redirect_url: - redirect_url = url_for( '/' ) - if openid_provider_obj: - return trans.response.send_redirect( url_for( controller='user', action='openid_auth', openid_provider=openid_provider, referer=redirect_url ) ) - return trans.response.send_redirect( redirect_url ) + if len( openid_objs ) == 1: + return trans.response.send_redirect( url_for( controller='user', action='openid_auth', openid_provider=openid_objs[0].id, redirect=redirect, auto_associate=True ) ) + elif openid_objs: + message = 'You have authenticated with several OpenID providers, please click the following links to execute the post authentication actions. ' + message = "%s
    " % ( message ) + for openid in openid_objs: + message = '%s
  • %s
    • ' % ( message, url_for( controller='user', action='openid_auth', openid_provider=openid.id, redirect=redirect, auto_associate=True ), openid.name ) + message = "%s
" % ( message ) + return trans.response.send_redirect( url_for( controller='user', + action='openid_manage', + use_panels=use_panels, + redirect=redirect, + message=message, + status='info' ) ) + if redirect: + return trans.response.send_redirect( redirect ) + return trans.response.send_redirect( url_for( controller='user', + action='openid_manage', + use_panels=use_panels, + redirect=redirect, + message=message, + status='info' ) ) if kwd.get( 'create_user_button', False ): password = kwd.get( 'password', '' ) confirm = kwd.get( 'confirm', '' ) @@ -253,21 +287,40 @@ class User( BaseUIController, UsesFormDefinitions ): subscribe_checked, **kwd ) if success: - trans.handle_user_login( user, webapp ) - trans.log_event( "User created a new account" ) - trans.log_event( "User logged in" ) + openid_objs = [] for openid in openids: - openid.user = user - trans.sa_session.add( openid ) + openid_provider_obj = trans.app.openid_providers.get( openid.provider ) + if not openid_provider_obj: + openid_provider_obj = trans.app.openid_providers.new_provider_from_identifier( openid.identifier ) + if not openid_provider_obj.never_associate_with_user: + openid.user = user + trans.sa_session.add( openid ) + trans.log_event( "User associated OpenID: %s" % openid.openid ) + if openid_provider_obj.has_post_authentication_actions(): + openid_objs.append( openid_provider_obj ) trans.sa_session.flush() - for openid in openids: - trans.log_event( "User associated OpenID: %s" % openid.openid ) - redirect_url = referer - if not redirect_url: - redirect_url = url_for( '/' ) - if openid_provider_obj: - return trans.response.send_redirect( url_for( controller='user', action='openid_auth', openid_provider=openid_provider, referer=redirect_url ) ) - return trans.response.send_redirect( redirect_url ) + if len( openid_objs ) == 1: + return trans.response.send_redirect( url_for( controller='user', action='openid_auth', openid_provider=openid_objs[0].id, redirect=redirect, auto_associate=True ) ) + elif openid_objs: + message = 'You have authenticated with several OpenID providers, please click the following links to execute the post authentication actions. ' + message = "%s
    " % ( message ) + for openid in openid_objs: + message = '%s
  • %s
    • ' % ( message, url_for( controller='user', action='openid_auth', openid_provider=openid.id, redirect=redirect, auto_associate=True ), openid.name ) + message = "%s
" % ( message ) + return trans.response.send_redirect( url_for( controller='user', + action='openid_manage', + use_panels=True, + redirect=redirect, + message=message, + status='info' ) ) + if redirect: + return trans.response.send_redirect( redirect ) + return trans.response.send_redirect( url_for( controller='user', + action='openid_manage', + use_panels=use_panels, + redirect=redirect, + message=message, + status='info' ) ) else: message = error status = 'error' @@ -288,11 +341,10 @@ class User( BaseUIController, UsesFormDefinitions ): email=email, password='', confirm='', - username=username, + username=transform_publicname( trans, username ), header='', use_panels=use_panels, - redirect_url='', - referer='', + redirect=redirect, refresh_frames=[], message=message, status=status, @@ -301,11 +353,11 @@ class User( BaseUIController, UsesFormDefinitions ): user_type_fd_id_select_field=user_type_fd_id_select_field, user_type_form_definition=user_type_form_definition, widgets=widgets, - openids=openids, - openid_provider=openid_provider ) + openids=openids ) @web.expose @web.require_login( 'manage OpenIDs' ) def openid_disassociate( self, trans, webapp='galaxy', **kwd ): + '''Disassociates a user with an OpenID''' if not trans.app.config.enable_openid: return trans.show_error_message( 'OpenID authentication is not enabled in this instance of Galaxy' ) params = util.Params( kwd ) @@ -338,7 +390,7 @@ class User( BaseUIController, UsesFormDefinitions ): trans.log_event( "User disassociated OpenID: %s" % deleted_url ) message = '%s OpenIDs were disassociated from your Galaxy account.' % len( ids ) status = 'done' - trans.response.send_redirect( url_for( controller='user', + return trans.response.send_redirect( url_for( controller='user', action='openid_manage', use_panels=use_panels, message=message, @@ -346,29 +398,31 @@ class User( BaseUIController, UsesFormDefinitions ): @web.expose @web.require_login( 'manage OpenIDs' ) def openid_manage( self, trans, webapp='galaxy', **kwd ): + '''Manage OpenIDs for user''' if not trans.app.config.enable_openid: return trans.show_error_message( 'OpenID authentication is not enabled in this instance of Galaxy' ) use_panels = kwd.get( 'use_panels', False ) if 'operation' in kwd: operation = kwd['operation'].lower() if operation == "delete": - trans.response.send_redirect( url_for( controller='user', + return trans.response.send_redirect( url_for( controller='user', action='openid_disassociate', use_panels=use_panels, id=kwd['id'] ) ) - kwd['referer'] = url_for( controller='user', action='openid_manage', use_panels=True ) + + kwd['redirect'] = kwd.get( 'redirect', url_for( controller='user', action='openid_manage', use_panels=True ) ).strip() kwd['openid_providers'] = trans.app.openid_providers return self.user_openid_grid( trans, **kwd ) @web.expose def login( self, trans, webapp='galaxy', redirect_url='', refresh_frames=[], **kwd ): - referer = kwd.get( 'referer', trans.request.referer ) + '''Handle Galaxy Log in''' + redirect = kwd.get( 'redirect', trans.request.referer ).strip() use_panels = util.string_as_bool( kwd.get( 'use_panels', False ) ) message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) header = '' user = None email = kwd.get( 'email', '' ) - openid_provider = kwd.get( 'openid_provider', '' ) if kwd.get( 'login_button', False ): if webapp == 'galaxy' and not refresh_frames: if trans.app.config.require_login: @@ -376,8 +430,8 @@ class User( BaseUIController, UsesFormDefinitions ): else: refresh_frames = [ 'masthead', 'history' ] message, status, user, success = self.__validate_login( trans, webapp, **kwd ) - if success and referer and not referer.startswith( trans.request.base + url_for( controller='user', action='logout' ) ): - redirect_url = referer + if success and redirect and not redirect.startswith( trans.request.base + url_for( controller='user', action='logout' ) ): + redirect_url = redirect elif success: redirect_url = url_for( '/' ) if not user and trans.app.config.require_login: @@ -399,7 +453,7 @@ class User( BaseUIController, UsesFormDefinitions ): header=header, use_panels=use_panels, redirect_url=redirect_url, - referer=referer, + redirect=redirect, refresh_frames=refresh_frames, message=message, status=status, @@ -410,7 +464,7 @@ class User( BaseUIController, UsesFormDefinitions ): status = kwd.get( 'status', 'done' ) email = kwd.get( 'email', '' ) password = kwd.get( 'password', '' ) - referer = kwd.get( 'referer', trans.request.referer ) + redirect = kwd.get( 'redirect', trans.request.referer ).strip() success = False user = trans.sa_session.query( trans.app.model.User ).filter( trans.app.model.User.table.c.email==email ).first() if not user: @@ -430,7 +484,7 @@ class User( BaseUIController, UsesFormDefinitions ): if webapp == 'galaxy': trans.log_event( "User logged in" ) message = 'You are now logged in as %s.
You can go back to the page you were visiting or go to the home page.' % \ - ( user.email, referer, url_for( '/' ) ) + ( user.email, redirect, url_for( '/' ) ) if trans.app.config.require_login: message += ' Click here to continue to the home page.' % web.url_for( '/static/welcome.html' ) success = True @@ -470,7 +524,7 @@ class User( BaseUIController, UsesFormDefinitions ): username = util.restore_text( params.get( 'username', '' ) ) subscribe = params.get( 'subscribe', '' ) subscribe_checked = CheckboxField.is_checked( subscribe ) - referer = kwd.get( 'referer', trans.request.referer ) + redirect = kwd.get( 'redirect', trans.request.referer ).strip() is_admin = cntrller == 'admin' and trans.user_is_admin if not trans.app.config.allow_user_creation and not trans.user_is_admin(): message = 'User registration is disabled. Please contact your Galaxy administrator for an account.' @@ -527,14 +581,14 @@ class User( BaseUIController, UsesFormDefinitions ): email=email, password=password, confirm=confirm, - username=username, + username=transform_publicname( trans, username ), subscribe_checked=subscribe_checked, user_type_fd_id_select_field=user_type_fd_id_select_field, user_type_form_definition=user_type_form_definition, widgets=widgets, webapp=webapp, use_panels=use_panels, - referer=referer, + redirect=redirect, redirect_url=redirect_url, refresh_frames=refresh_frames, message=message, diff --git a/lib/galaxy/web/controllers/workflow.py b/lib/galaxy/web/controllers/workflow.py index ba1ce14f58a..4f238c37004 100644 --- a/lib/galaxy/web/controllers/workflow.py +++ b/lib/galaxy/web/controllers/workflow.py @@ -1119,7 +1119,7 @@ class WorkflowController( BaseUIController, Sharable, UsesStoredWorkflow, UsesAn valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' sname = stored.name sname = ''.join(c in valid_chars and c or '_' for c in sname)[0:150] - trans.response.headers["Content-Disposition"] = "attachment; filename=Galaxy-Workflow-%s.ga" % ( sname ) + trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy-Workflow-%s.ga"' % ( sname ) trans.response.set_content_type( 'application/galaxy-archive' ) return stored_dict @web.expose @@ -1160,7 +1160,7 @@ class WorkflowController( BaseUIController, Sharable, UsesStoredWorkflow, UsesAn import_button = True if tool_shed_url and not import_button: # Use urllib (send another request to the tool shed) to retrieve the workflow. - workflow_url = '%s/workflow/import_workflow?repository_metadata_id=%s&workflow_name=%s&webapp=%s&open_for_url=true' % \ + workflow_url = '%s/workflow/import_workflow?repository_metadata_id=%s&workflow_name=%s&webapp=%s&open_for_url=true&no_reset=true' % \ ( tool_shed_url, repository_metadata_id, tool_shed_encode( workflow_name ), webapp ) response = urllib2.urlopen( workflow_url ) workflow_text = response.read() @@ -1354,11 +1354,6 @@ class WorkflowController( BaseUIController, Sharable, UsesStoredWorkflow, UsesAn tool = trans.app.toolbox.get_tool( job.tool_id ) param_values = job.get_param_values( trans.app ) associations = cleanup_param_values( tool.inputs, param_values ) - # Doing it this way breaks dynamic parameters, backed out temporarily. - # def extract_callback( input, value, prefixed_name, prefixed_label ): - # if isinstance( value, UnvalidatedValue ): - # return str( value ) - # visit_input_values( tool.inputs, param_values, extract_callback ) step = model.WorkflowStep() step.type = 'tool' step.tool_id = job.tool_id @@ -2059,13 +2054,11 @@ def cleanup_param_values( inputs, values ): group_values = values[key] for i, rep_values in enumerate( group_values ): rep_index = rep_values['__index__'] - prefix = "%s_%d|" % ( key, rep_index ) - cleanup( prefix, input.inputs, group_values[i] ) + cleanup( "%s%s_%d|" % (prefix, key, rep_index ), input.inputs, group_values[i] ) elif isinstance( input, Conditional ): group_values = values[input.name] current_case = group_values['__current_case__'] - prefix = "%s|" % ( key ) - cleanup( prefix, input.cases[current_case].inputs, group_values ) + cleanup( "%s%s|" % ( prefix, key ), input.cases[current_case].inputs, group_values ) cleanup( "", inputs, values ) return associations diff --git a/lib/galaxy/web/form_builder.py b/lib/galaxy/web/form_builder.py index 248285fbc8e..15091b15929 100644 --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -6,6 +6,7 @@ import logging, sys, os, time from cgi import escape from galaxy.util import restore_text, relpath, nice_size from galaxy.web import url_for +from binascii import hexlify log = logging.getLogger(__name__) @@ -143,7 +144,7 @@ class FileField(BaseField): def get_html( self, prefix="" ): value_text = "" if self.value: - value_text = ' value="%s"' % self.value + value_text = ' value="%s"' % escape( str( self.value ), quote=True ) ajax_text = "" if self.ajax: ajax_text = ' galaxy-ajax-upload="true"' @@ -278,7 +279,7 @@ class SelectField(BaseField): if self.refresh_on_change: self.refresh_on_change_text = ' refresh_on_change="true"' if self.refresh_on_change_values: - self.refresh_on_change_text = '%s refresh_on_change_values="%s"' % ( self.refresh_on_change_text, ",".join( self.refresh_on_change_values ) ) + self.refresh_on_change_text = '%s refresh_on_change_values="%s"' % ( self.refresh_on_change_text, escape( ",".join( self.refresh_on_change_values ), quote=True ) ) else: self.refresh_on_change_text = '' def add_option( self, text, value, selected = False ): @@ -305,7 +306,7 @@ class SelectField(BaseField): if selected: selected_text = " checked='checked'" rval.append( '' % \ - ( style, prefix, self.name, escaped_value, uniq_id, selected_text, self.get_disabled_str( disabled ), uniq_id, text ) ) + ( style, prefix, self.name, escaped_value, uniq_id, selected_text, self.get_disabled_str( disabled ), uniq_id, escape( str( text ), quote=True ) ) ) ctr += 1 return "\n".join( rval ) def get_html_radio( self, prefix="", disabled=False ): @@ -350,7 +351,7 @@ class SelectField(BaseField): last_selected_value = value else: selected_text = "" - rval.append( '' % ( escape( str( value ), quote=True ), selected_text, text ) ) + rval.append( '' % ( escape( str( value ), quote=True ), selected_text, escape( str( text ), quote=True ) ) ) if last_selected_value: last_selected_value = ' last_selected_value="%s"' % escape( str( last_selected_value ), quote=True ) rval.insert( 0, 'Heading 1 -
    -
  • Option 1 -
    • -
  • Option 2 -
    • -
  • [+]Heading 1 -
      -
    • Option 3 -
      • -
    • Option 4 -
      • -
    -
    • -
- -
  • Option 5 -
    • - +
    +
    + + Heading 1 +
    +
    + Option 1 +
    +
    + Option 2 +
    +
    + + Heading 1 +
    +
    + Option 3 +
    +
    + Option 4 +
    +
    +
    +
    +
    +
    + Option 5 +
    +
    >>> t = DrillDownField( "foo", multiple=False, display="radio", options=[{'name': 'Heading 1', 'value': 'heading1', 'options': [{'name': 'Option 1', 'value': 'option1', 'options': []}, {'name': 'Option 2', 'value': 'option2', 'options': []}, {'name': 'Heading 1', 'value': 'heading1', 'options': [{'name': 'Option 3', 'value': 'option3', 'options': []}, {'name': 'Option 4', 'value': 'option4', 'options': []}]}]}, {'name': 'Option 5', 'value': 'option5', 'options': []}] ) >>> print t.get_html() -
      -
    • [+]Heading 1 -
        -
      • Option 1 -
        • -
      • Option 2 -
        • -
      • [+]Heading 1 -
          -
        • Option 3 -
          • -
        • Option 4 -
          • -
        -
        • -
      -
      • -
    • Option 5 -
      • -
    +
    +
    + + Heading 1 +
    +
    + Option 1 +
    +
    + Option 2 +
    +
    + + Heading 1 +
    +
    + Option 3 +
    +
    + Option 4 +
    +
    +
    +
    +
    +
    + Option 5 +
    +
    """ def __init__( self, name, multiple=None, display=None, refresh_on_change=False, options = [], value = [], refresh_on_change_values = [] ): self.name = name @@ -440,7 +459,7 @@ class DrillDownField( BaseField ): self.options = options if value and not isinstance( value, list ): value = [ value ] - else: + elif not value: value = [] self.value = value if display == "checkbox": @@ -466,30 +485,34 @@ class DrillDownField( BaseField ): if option['options']: new_parents = list( parent_options ) + [ option['value'] ] find_expanded_options( expanded_options, option['options'], new_parents ) - def recurse_options( html, options, expanded_options = [] ): + def recurse_options( html, options, base_id, expanded_options = [] ): for option in options: + escaped_option_value = escape( str( option['value'] ), quote=True ) selected = ( option['value'] in self.value ) - if selected: selected = ' checked' - else: selected = '' - if option['options']: - default_state = 'collapsed' - default_icon = '[+]' - if option['value'] in expanded_options: - default_state = 'expanded' - default_icon = '[-]' - html.append( '
  • %s%s' % ( default_icon, self.display, prefix, self.name, escape(str(option['value']), quote=True), selected, option['name']) ) - html.append( '
      ' % default_state ) - recurse_options( html, option['options'], expanded_options ) - html.append( '
    ') + if selected: + selected = ' checked' else: - html.append( '
  • %s' % ( self.display, prefix, self.name, escape(str(option['value']), quote=True), selected, option['name']) ) - html.append( '
    • ' ) + selected = '' + span_class = 'form-toggle icon-button toggle' + if option['value'] not in expanded_options: + span_class = "%s-expand" % ( span_class ) + html.append( '
    ') + drilldown_group_id = "%s-%s" % ( base_id, hexlify( option['value'] ) ) + if option['options']: + html.append( '' % ( span_class, drilldown_group_id ) ) + html.append( '%s' % ( self.display, prefix, self.name, escaped_option_value, selected, option['name']) ) + if option['options']: + html.append( '
    ' % ( drilldown_group_id ) ) + recurse_options( html, option['options'], drilldown_group_id, expanded_options ) + html.append( '
    ') + html.append( '
    ') + drilldown_id = "drilldown-%s-%s" % ( hexlify( prefix ), hexlify( self.name ) ) rval = [] - rval.append( '
      ' ) + rval.append( '
      ' % ( drilldown_id ) ) expanded_options = [] find_expanded_options( expanded_options, self.options ) - recurse_options( rval, self.options, expanded_options ) - rval.append( '
    ' ) + recurse_options( rval, self.options, drilldown_id, expanded_options ) + rval.append( '' ) return '\n'.join( rval ) class AddressField(BaseField): diff --git a/lib/galaxy/web/framework/__init__.py b/lib/galaxy/web/framework/__init__.py index f018b856a34..ef4fdbaf712 100644 --- a/lib/galaxy/web/framework/__init__.py +++ b/lib/galaxy/web/framework/__init__.py @@ -9,6 +9,7 @@ pkg_resources.require( "Cheetah" ) from Cheetah.Template import Template import base import pickle +from functools import wraps from galaxy import util from galaxy.exceptions import MessageException from galaxy.util.json import to_json_string, from_json_string @@ -62,6 +63,7 @@ def expose( func ): return func def json( func ): + @wraps(func) def decorator( self, trans, *args, **kwargs ): trans.response.set_content_type( "text/javascript" ) return simplejson.dumps( func( self, trans, *args, **kwargs ) ) @@ -71,6 +73,7 @@ def json( func ): return decorator def json_pretty( func ): + @wraps(func) def decorator( self, trans, *args, **kwargs ): trans.response.set_content_type( "text/javascript" ) return simplejson.dumps( func( self, trans, *args, **kwargs ), indent=4, sort_keys=True ) @@ -81,37 +84,46 @@ def json_pretty( func ): def require_login( verb="perform this action", use_panels=False, webapp='galaxy' ): def argcatcher( func ): + @wraps(func) def decorator( self, trans, *args, **kwargs ): if trans.get_user(): return func( self, trans, *args, **kwargs ) else: return trans.show_error_message( 'You must be logged in to %s.' - % ( url_for( controller='user', action='login', webapp=webapp ), verb ), use_panels=use_panels ) + % ( url_for( controller='user', action='login', webapp=webapp ), verb ), use_panels=use_panels ) return decorator return argcatcher - + def expose_api( func ): + @wraps(func) def decorator( self, trans, *args, **kwargs ): def error( environ, start_response ): start_response( error_status, [('Content-type', 'text/plain')] ) return error_message error_status = '403 Forbidden' - if 'key' not in kwargs: - error_message = 'No API key provided with request, please consult the API documentation.' - return error - try: - provided_key = trans.sa_session.query( trans.app.model.APIKeys ).filter( trans.app.model.APIKeys.table.c.key == kwargs['key'] ).one() - except NoResultFound: - error_message = 'Provided API key is not valid.' - return error - if provided_key.user.deleted: - error_message = 'User account is deactivated, please contact an administrator.' - return error - newest_key = provided_key.user.api_keys[0] - if newest_key.key != provided_key.key: - error_message = 'Provided API key has expired.' - return error + ## If there is a user, we've authenticated a session. + if not trans.user and isinstance(trans.galaxy_session, Bunch): + # If trans.user is already set, don't check for a key. + # This happens when we're authenticating using session instead of an API key. + # The Bunch clause is used to prevent the case where there's no user, but there is a real session. + # DBTODO: This needs to be fixed when merging transaction types. + if 'key' not in kwargs: + error_message = 'No API key provided with request, please consult the API documentation.' + return error + try: + provided_key = trans.sa_session.query( trans.app.model.APIKeys ).filter( trans.app.model.APIKeys.table.c.key == kwargs['key'] ).one() + except NoResultFound: + error_message = 'Provided API key is not valid.' + return error + if provided_key.user.deleted: + error_message = 'User account is deactivated, please contact an administrator.' + return error + newest_key = provided_key.user.api_keys[0] + if newest_key.key != provided_key.key: + error_message = 'Provided API key has expired.' + return error + trans.set_user( provided_key.user ) if trans.request.body: try: payload = util.recursively_stringify_dictionary_keys( simplejson.loads( trans.request.body ) ) @@ -121,7 +133,6 @@ def expose_api( func ): error_message = 'Your request did not appear to be valid JSON, please consult the API documentation' return error trans.response.set_content_type( "application/json" ) - trans.set_user( provided_key.user ) # Perform api_run_as processing, possibly changing identity if 'run_as' in kwargs: if not trans.user_can_do_run_as(): @@ -139,7 +150,6 @@ def expose_api( func ): except: trans.response.status = 400 return "That user does not exist." - try: if trans.debug: return simplejson.dumps( func( self, trans, *args, **kwargs ), indent=4, sort_keys=True ) @@ -156,6 +166,7 @@ def expose_api( func ): return decorator def require_admin( func ): + @wraps(func) def decorator( self, trans, *args, **kwargs ): if not trans.user_is_admin(): msg = "You must be an administrator to access this feature." @@ -202,10 +213,10 @@ class WebApplication( base.WebApplication ): return base.WebApplication.make_body_iterable( self, trans, body ) def transaction_chooser( self, environ, galaxy_app, session_cookie ): if 'is_api_request' in environ: - return GalaxyWebAPITransaction( environ, galaxy_app, self ) + return GalaxyWebAPITransaction( environ, galaxy_app, self, session_cookie ) else: return GalaxyWebUITransaction( environ, galaxy_app, self, session_cookie ) - + class GalaxyWebTransaction( base.DefaultWebTransaction ): """ Encapsulates web transaction specific state for the Galaxy application @@ -262,7 +273,7 @@ class GalaxyWebTransaction( base.DefaultWebTransaction ): except: action.user = None try: - action.session_id = self.galaxy_session.id + action.session_id = self.galaxy_session.id except: action.session_id = None self.sa_session.add( action ) @@ -317,7 +328,7 @@ class GalaxyWebTransaction( base.DefaultWebTransaction ): tstamp = time.localtime ( time.time() + 3600 * 24 * age ) self.response.cookies[name]['expires'] = time.strftime( '%a, %d-%b-%Y %H:%M:%S GMT', tstamp ) self.response.cookies[name]['version'] = version - def _ensure_valid_session( self, session_cookie ): + def _ensure_valid_session( self, session_cookie, create=True): """ Ensure that a valid Galaxy session exists and is available as trans.session (part of initialization) @@ -822,7 +833,7 @@ class FormBuilder( object ): def add_select( self, name, label, value=None, options=[], error=None, help=None, use_label=True ): self.inputs.append( SelectInput( name, label, value=value, options=options, error=error, help=help, use_label=use_label ) ) return self - + class FormInput( object ): """ Simple class describing a form input element @@ -837,15 +848,89 @@ class FormInput( object ): self.use_label = use_label class GalaxyWebAPITransaction( GalaxyWebTransaction ): - def __init__( self, environ, app, webapp ): + """ + TODO: Unify this with WebUITransaction, since we allow session auth now. + Enable functionality of 'create' parameter in parent _ensure_valid_session + """ + def __init__( self, environ, app, webapp, session_cookie): GalaxyWebTransaction.__init__( self, environ, app, webapp ) self.__user = None - self._ensure_valid_session( None ) + self._ensure_valid_session( session_cookie ) def _ensure_valid_session( self, session_cookie ): - self.galaxy_session = Bunch() - self.galaxy_session.history = self.galaxy_session.current_history = Bunch() - self.galaxy_session.history.genome_build = None - self.galaxy_session.is_api = True + #Check to see if there is an existing session. Never create a new one. + # Try to load an existing session + secure_id = self.get_cookie( name=session_cookie ) + galaxy_session = None + prev_galaxy_session = None + user_for_new_session = None + invalidate_existing_session = False + # Track whether the session has changed so we can avoid calling flush + # in the most common case (session exists and is valid). + galaxy_session_requires_flush = False + if secure_id: + # Decode the cookie value to get the session_key + session_key = self.security.decode_guid( secure_id ) + try: + # Make sure we have a valid UTF-8 string + session_key = session_key.encode( 'utf8' ) + except UnicodeDecodeError: + # We'll end up creating a new galaxy_session + session_key = None + if session_key: + # Retrieve the galaxy_session id via the unique session_key + galaxy_session = self.sa_session.query( self.app.model.GalaxySession ) \ + .filter( and_( self.app.model.GalaxySession.table.c.session_key==session_key, + self.app.model.GalaxySession.table.c.is_valid==True ) ) \ + .first() + if galaxy_session: + # If remote user is in use it can invalidate the session, so we need to to check some things now. + if self.app.config.use_remote_user: + assert "HTTP_REMOTE_USER" in self.environ, \ + "use_remote_user is set but no HTTP_REMOTE_USER variable" + remote_user_email = self.environ[ 'HTTP_REMOTE_USER' ] + # An existing session, make sure correct association exists + if galaxy_session.user is None: + # No user, associate + galaxy_session.user = self.get_or_create_remote_user( remote_user_email ) + galaxy_session_requires_flush = True + elif galaxy_session.user.email != remote_user_email: + # Session exists but is not associated with the correct remote user + log.warning( "User logged in as '%s' externally, but has a cookie as '%s' invalidating session", + remote_user_email, galaxy_session.user.email ) + galaxy_session = None + else: + if galaxy_session.user and galaxy_session.user.external: + # Remote user support is not enabled, but there is an existing + # session with an external user, invalidate + invalidate_existing_session = True + log.warning( "User '%s' is an external user with an existing session, invalidating session since external auth is disabled", + galaxy_session.user.email ) + galaxy_session = None + elif galaxy_session.user is not None and galaxy_session.user.deleted: + invalidate_existing_session = True + log.warning( "User '%s' is marked deleted, invalidating session" % galaxy_session.user.email ) + galaxy_session = None + # No relevant cookies, or couldn't find, or invalid, so create a new session + if galaxy_session: + self.galaxy_session = galaxy_session + self.user = galaxy_session.user + # Do we need to flush the session? + if galaxy_session_requires_flush: + self.sa_session.add( galaxy_session ) + # FIXME: If prev_session is a proper relation this would not + # be needed. + if prev_galaxy_session: + self.sa_session.add( prev_galaxy_session ) + self.sa_session.flush() + # If the old session was invalid, get a new history with our new session + if not galaxy_session: + #Failed to find a session. Set up fake stuff for API transaction + self.user = None + self.galaxy_session = Bunch() + self.galaxy_session.history = self.galaxy_session.current_history = Bunch() + self.galaxy_session.history.genome_build = None + self.galaxy_session.is_api = True + def get_user( self ): """Return the current user (the expose_api decorator ensures that it is set).""" return self.__user diff --git a/lib/galaxy/web/framework/helpers/__init__.py b/lib/galaxy/web/framework/helpers/__init__.py index 02441e0cbfe..77f6dbbea08 100644 --- a/lib/galaxy/web/framework/helpers/__init__.py +++ b/lib/galaxy/web/framework/helpers/__init__.py @@ -45,14 +45,28 @@ def css( *args ): """ return "\n".join( [ stylesheet_link_tag( "/static/style/" + name + ".css?v=%s" % server_starttime ) for name in args ] ) -def js( *args ): +def js_helper( prefix, *args ): """ - Take a list of javascript names (no extension) and return appropriate + Take a prefix and list of javascript names and return appropriate string of script tags. Cache-bust with time that server started running on """ - return "\n".join( [ javascript_include_tag( "/static/scripts/" + name + ".js?v=%s" % server_starttime ) for name in args ] ) + return "\n".join( [ javascript_include_tag( prefix + name + ".js?v=%s" % server_starttime ) for name in args ] ) + +def js( *args ): + """ + Take a prefix and list of javascript names and return appropriate + string of script tags. + """ + return js_helper( '/static/scripts/', *args ) + +def templates( *args ): + """ + Take a list of template names (no extension) and return appropriate + string of script tags. + """ + return js_helper( '/static/scripts/templates/compiled/', *args ) # Hashes diff --git a/lib/galaxy/webapps/community/app.py b/lib/galaxy/webapps/community/app.py index 3651f67485c..f564ef31340 100644 --- a/lib/galaxy/webapps/community/app.py +++ b/lib/galaxy/webapps/community/app.py @@ -43,6 +43,8 @@ class UniverseApplication( object ): # Load security policy self.security_agent = self.model.security_agent self.quota_agent = galaxy.quota.NoQuotaAgent( self.model ) - self.openid_providers = OpenIDProviders() #TODO: Add OpenID support + # TODO: Add OpenID support + self.openid_providers = OpenIDProviders() + self.shed_counter = self.model.shed_counter def shutdown( self ): pass diff --git a/lib/galaxy/webapps/community/config.py b/lib/galaxy/webapps/community/config.py index 2e793b63e65..bfa31b51fc1 100644 --- a/lib/galaxy/webapps/community/config.py +++ b/lib/galaxy/webapps/community/config.py @@ -39,8 +39,6 @@ class Configuration( object ): self.file_path = resolve_path( kwargs.get( "file_path", "database/files" ), self.root ) self.new_file_path = resolve_path( kwargs.get( "new_file_path", "database/tmp" ), self.root ) self.cookie_path = kwargs.get( "cookie_path", "/" ) - # web API - self.enable_api = string_as_bool( kwargs.get( 'enable_api', False ) ) self.enable_quotas = string_as_bool( kwargs.get( 'enable_quotas', False ) ) self.datatypes_config = kwargs.get( 'datatypes_config_file', 'datatypes_conf.xml' ) self.test_conf = resolve_path( kwargs.get( "test_conf", "" ), self.root ) @@ -84,6 +82,12 @@ class Configuration( object ): self.screencasts_url = kwargs.get( 'screencasts_url', None ) self.log_events = False self.cloud_controller_instance = False + self.server_name = '' + self.job_manager = '' + self.default_job_handlers = [] + self.job_handlers = [] + self.tool_handlers = [] + self.tool_runners = [] # Proxy features self.apache_xsendfile = kwargs.get( 'apache_xsendfile', False ) self.nginx_x_accel_redirect_base = kwargs.get( 'nginx_x_accel_redirect_base', False ) @@ -94,6 +98,7 @@ class Configuration( object ): self.global_conf_parser = global_conf_parser if global_conf and "__file__" in global_conf: global_conf_parser.read(global_conf['__file__']) + self.running_functional_tests = string_as_bool( kwargs.get( 'running_functional_tests', False ) ) def get( self, key, default ): return self.config_dict.get( key, default ) def get_bool( self, key, default ): diff --git a/lib/galaxy/webapps/community/controllers/admin.py b/lib/galaxy/webapps/community/controllers/admin.py index 230600e6f7f..d0a513a1485 100644 --- a/lib/galaxy/webapps/community/controllers/admin.py +++ b/lib/galaxy/webapps/community/controllers/admin.py @@ -3,9 +3,14 @@ from galaxy.webapps.community import model from galaxy.model.orm import * from galaxy.web.framework.helpers import time_ago, iff, grids from galaxy.util import inflector +from galaxy.util.shed_util import get_configured_ui from common import * from repository import RepositoryListGrid, CategoryListGrid + +from galaxy import eggs +eggs.require('mercurial') from mercurial import hg + import logging log = logging.getLogger( __name__ ) @@ -321,7 +326,7 @@ class RepositoryMetadataListGrid( grids.Grid ): def get_value( self, trans, grid, repository_metadata ): repository = repository_metadata.repository repo = hg.repository( get_configured_ui(), repository.repo_path ) - ctx = get_changectx_for_changeset( trans, repo, repository_metadata.changeset_revision ) + ctx = get_changectx_for_changeset( repo, repository_metadata.changeset_revision ) return "%s:%s" % ( str( ctx.rev() ), repository_metadata.changeset_revision ) class ToolsColumn( grids.TextColumn ): def get_value( self, trans, grid, repository_metadata ): @@ -457,6 +462,31 @@ class AdminController( BaseUIController, Admin ): status=status ) ) @web.expose @web.require_admin + def reset_all_repository_metadata( self, trans, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + if 'reset_all_repository_metadata_button' in kwd: + count = 0 + for repository in trans.sa_session.query( trans.model.Repository ) \ + .filter( trans.model.Repository.table.c.deleted == False ): + try: + reset_all_repository_metadata( trans, trans.security.encode_id( repository.id ) ) + log.debug( "Reset metadata on repository %s" % repository.name ) + count += 1 + except Exception, e: + log.debug( "Error attempting to reset metadata on repository %s: %s" % ( repository.name, str( e ) ) ) + message = "Reset metadata on %d repositories" % count + trans.response.send_redirect( web.url_for( controller='admin', + action='browse_repository_metadata', + webapp='community', + message=util.sanitize_text( message ), + status=status ) ) + return trans.fill_template( '/webapps/community/admin/reset_all_repository_metadata.mako', + message=message, + status=status ) + @web.expose + @web.require_admin def browse_repositories( self, trans, **kwd ): # We add params to the keyword dict in this method in order to rename the param # with an "f-" prefix, simulating filtering by clicking a search link. We have @@ -527,6 +557,18 @@ class AdminController( BaseUIController, Admin ): return self.repository_list_grid( trans, **kwd ) @web.expose @web.require_admin + def regenerate_statistics( self, trans, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + if 'regenerate_statistics_button' in kwd: + trans.app.shed_counter.generate_statistics() + message = "Successfully regenerated statistics" + return trans.fill_template( '/webapps/community/admin/statistics.mako', + message=message, + status=status ) + @web.expose + @web.require_admin def delete_repository( self, trans, **kwd ): params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) diff --git a/lib/galaxy/webapps/community/controllers/common.py b/lib/galaxy/webapps/community/controllers/common.py index 813f0b9d91d..8f932fa95d1 100644 --- a/lib/galaxy/webapps/community/controllers/common.py +++ b/lib/galaxy/webapps/community/controllers/common.py @@ -5,12 +5,15 @@ from galaxy.datatypes.checkers import * from galaxy.tools import * from galaxy.util.json import from_json_string, to_json_string from galaxy.util.hash_util import * -from galaxy.util.shed_util import copy_sample_loc_file, generate_datatypes_metadata, generate_tool_metadata, generate_workflow_metadata -from galaxy.util.shed_util import handle_sample_tool_data_table_conf_file +from galaxy.util.shed_util import copy_sample_file, get_configured_ui, generate_datatypes_metadata, generate_tool_metadata, generate_workflow_metadata +from galaxy.util.shed_util import handle_sample_tool_data_table_conf_file, to_html_escaped, to_html_str, update_repository from galaxy.web.base.controller import * from galaxy.webapps.community import model from galaxy.model.orm import * from galaxy.model.item_attrs import UsesItemRatings + +from galaxy import eggs +eggs.require('mercurial') from mercurial import hg, ui, commands log = logging.getLogger( __name__ ) @@ -140,7 +143,7 @@ def get_revision_label( trans, repository, changeset_revision ): changeset rev and the changeset revision string. """ repo = hg.repository( get_configured_ui(), repository.repo_path ) - ctx = get_changectx_for_changeset( trans, repo, changeset_revision ) + ctx = get_changectx_for_changeset( repo, changeset_revision ) if ctx: return "%s:%s" % ( str( ctx.rev() ), changeset_revision ) else: @@ -201,18 +204,18 @@ def check_tool_input_params( trans, name, tool, sample_files, invalid_files ): can_set_metadata = False correction_msg = "This file requires an entry in the tool_data_table_conf.xml file. " correction_msg += "Upload a file named tool_data_table_conf.xml.sample to the repository " - correction_msg += "that includes the required entry to resolve this issue.
    " + correction_msg += "that includes the required entry to correct this error.
    " invalid_files.append( ( name, correction_msg ) ) if options.index_file or options.missing_index_file: # Make sure the repository contains the required xxx.loc.sample file. index_file = options.index_file or options.missing_index_file - index_head, index_tail = os.path.split( index_file ) + index_file_path, index_file_name = os.path.split( index_file ) sample_found = False for sample_file in sample_files: - sample_head, sample_tail = os.path.split( sample_file ) - if sample_tail == '%s.sample' % index_tail: - copy_sample_loc_file( trans.app, sample_file ) - options.index_file = index_tail + sample_file_path, sample_file_name = os.path.split( sample_file ) + if sample_file_name == '%s.sample' % index_file_name: + copy_sample_file( trans.app, sample_file ) + options.index_file = index_file_name options.missing_index_file = None if options.tool_data_table: options.tool_data_table.missing_index_file = None @@ -221,7 +224,7 @@ def check_tool_input_params( trans, name, tool, sample_files, invalid_files ): if not sample_found: can_set_metadata = False correction_msg = "This file refers to a file named %s. " % str( index_file ) - correction_msg += "Upload a file named %s.sample to the repository to correct this error." % str( index_tail ) + correction_msg += "Upload a file named %s.sample to the repository to correct this error." % str( index_file_name ) invalid_files.append( ( name, correction_msg ) ) return can_set_metadata, invalid_files def new_tool_metadata_required( trans, id, metadata_dict ): @@ -280,16 +283,20 @@ def new_workflow_metadata_required( trans, id, metadata_dict ): else: # There is no saved repository metadata, so we need to create a new repository_metadata table record. return True - # The received metadata_dict includes no metadata for workflows, so a new repository_metadata table - # record is not needed. + # The received metadata_dict includes no metadata for workflows, so a new repository_metadata table record is not needed. return False -def generate_metadata_for_repository_tip( trans, id, ctx, changeset_revision, repo_dir ): - # Browse the repository tip files on disk to generate metadata. This is faster than - # the generate_metadata_for_changeset_revision() method below because fctx.data() does - # not have to be written to disk to load tools. also, since changeset_revision is the - # repository tip, we handle things like .loc.sample files here. +def generate_metadata_for_repository_tip( trans, id, ctx, changeset_revision, repo, repo_dir ): + """ + Browse the repository tip files on disk to generate metadata. This is faster than the + generate_metadata_for_changeset_revision() method below because fctx.data() does not have + to be written to disk to load tools. We also handle things like .loc.sample files and + invalid_tool_configs here, while they are ignored in older revisions. + """ + # If a push from the command line is occurring, update the repository files on disk before setting metadata. + update_repository( repo, str( ctx.rev() ) ) metadata_dict = {} invalid_files = [] + invalid_tool_configs = [] sample_files = [] datatypes_config = None # Find datatypes_conf.xml if it exists. @@ -319,18 +326,30 @@ def generate_metadata_for_repository_tip( trans, id, ctx, changeset_revision, re if not ( check_binary( full_path ) or check_image( full_path ) or check_gzip( full_path )[ 0 ] or check_bz2( full_path )[ 0 ] or check_zip( full_path ) ): try: - tool = load_tool( trans, full_path ) - valid = True + # Make sure we're looking at a tool config and not a display application config or something else. + element_tree = util.parse_xml( full_path ) + element_tree_root = element_tree.getroot() + is_tool = element_tree_root.tag == 'tool' except Exception, e: - valid = False - invalid_files.append( ( name, str( e ) ) ) - if valid and tool is not None: - can_set_metadata, invalid_files = check_tool_input_params( trans, name, tool, sample_files, invalid_files ) - if can_set_metadata: - # Update the list of metadata dictionaries for tools in metadata_dict. - tool_config = os.path.join( root, name ) - repository_clone_url = generate_clone_url( trans, id ) - metadata_dict = generate_tool_metadata( tool_config, tool, repository_clone_url, metadata_dict ) + log.debug( "Error parsing %s, exception: %s" % ( full_path, str( e ) ) ) + is_tool = False + if is_tool: + try: + tool = load_tool( trans, full_path ) + valid = True + except Exception, e: + valid = False + invalid_files.append( ( name, str( e ) ) ) + invalid_tool_configs.append( name ) + if valid and tool is not None: + can_set_metadata, invalid_files = check_tool_input_params( trans, name, tool, sample_files, invalid_files ) + if can_set_metadata: + # Update the list of metadata dictionaries for tools in metadata_dict. + tool_config = os.path.join( root, name ) + repository_clone_url = generate_clone_url( trans, id ) + metadata_dict = generate_tool_metadata( tool_config, tool, repository_clone_url, metadata_dict ) + else: + invalid_tool_configs.append( name ) # Find all exported workflows elif name.endswith( '.ga' ): try: @@ -344,6 +363,8 @@ def generate_metadata_for_repository_tip( trans, id, ctx, changeset_revision, re metadata_dict = generate_workflow_metadata( relative_path, exported_workflow_dict, metadata_dict ) except Exception, e: invalid_files.append( ( name, str( e ) ) ) + if invalid_tool_configs: + metadata_dict[ 'invalid_tools' ] = invalid_tool_configs return metadata_dict, invalid_files def generate_metadata_for_changeset_revision( trans, id, ctx, changeset_revision, repo_dir ): # Browse repository files within a change set to generate metadata. @@ -388,21 +409,29 @@ def generate_metadata_for_changeset_revision( trans, id, ctx, changeset_revision if not ( check_binary( tmp_filename ) or check_image( tmp_filename ) or check_gzip( tmp_filename )[ 0 ] or check_bz2( tmp_filename )[ 0 ] or check_zip( tmp_filename ) ): try: - tool = load_tool( trans, tmp_filename ) - valid = True - except Exception, e: - invalid_files.append( ( filename, str( e ) ) ) - valid = False - if valid and tool is not None: - # Update the list of metadata dictionaries for tools in metadata_dict. Note that filename - # here is the relative path to the config file within the change set context, something - # like filtering.xml, but when the change set was the repository tip, the value was - # something like database/community_files/000/repo_1/filtering.xml. This shouldn't break - # anything, but may result in a bit of confusion when maintaining the code / data over time. - # IMPORTANT NOTE: Here we are assuming that since the current change set is not the repository - # tip, we do not have to handle any .loc.sample files since they would have been handled previously. - repository_clone_url = generate_clone_url( trans, id ) - metadata_dict = generate_tool_metadata( filename, tool, repository_clone_url, metadata_dict ) + # Make sure we're looking at a tool config and not a display application config or something else. + element_tree = util.parse_xml( tmp_filename ) + element_tree_root = element_tree.getroot() + is_tool = element_tree_root.tag == 'tool' + except: + is_tool = False + if is_tool: + try: + tool = load_tool( trans, tmp_filename ) + valid = True + except Exception, e: + invalid_files.append( ( filename, str( e ) ) ) + valid = False + if valid and tool is not None: + # Update the list of metadata dictionaries for tools in metadata_dict. Note that filename + # here is the relative path to the config file within the change set context, something + # like filtering.xml, but when the change set was the repository tip, the value was + # something like database/community_files/000/repo_1/filtering.xml. This shouldn't break + # anything, but may result in a bit of confusion when maintaining the code / data over time. + # IMPORTANT NOTE: Here we are assuming that since the current change set is not the repository + # tip, we do not have to handle any .loc.sample files since they would have been handled previously. + repository_clone_url = generate_clone_url( trans, id ) + metadata_dict = generate_tool_metadata( filename, tool, repository_clone_url, metadata_dict ) try: os.unlink( tmp_filename ) except: @@ -425,12 +454,12 @@ def set_repository_metadata( trans, id, changeset_revision, content_alert_str='' repository = get_repository( trans, id ) repo_dir = repository.repo_path repo = hg.repository( get_configured_ui(), repo_dir ) - ctx = get_changectx_for_changeset( trans, repo, changeset_revision ) + ctx = get_changectx_for_changeset( repo, changeset_revision ) metadata_dict = {} invalid_files = [] if ctx is not None: if changeset_revision == repository.tip: - metadata_dict, invalid_files = generate_metadata_for_repository_tip( trans, id, ctx, changeset_revision, repo_dir ) + metadata_dict, invalid_files = generate_metadata_for_repository_tip( trans, id, ctx, changeset_revision, repo, repo_dir ) else: metadata_dict, invalid_files = generate_metadata_for_changeset_revision( trans, id, ctx, changeset_revision, repo_dir ) if metadata_dict: @@ -439,10 +468,14 @@ def set_repository_metadata( trans, id, changeset_revision, content_alert_str='' # Create a new repository_metadata table row. repository_metadata = trans.model.RepositoryMetadata( repository.id, changeset_revision, metadata_dict ) trans.sa_session.add( repository_metadata ) - trans.sa_session.flush() - # If this is the first record stored for this repository, see if we need to send any email alerts. - if len( repository.downloadable_revisions ) == 1: - handle_email_alerts( trans, repository, content_alert_str='', new_repo_alert=True, admin_only=False ) + try: + trans.sa_session.flush() + # If this is the first record stored for this repository, see if we need to send any email alerts. + if len( repository.downloadable_revisions ) == 1: + handle_email_alerts( trans, repository, content_alert_str='', new_repo_alert=True, admin_only=False ) + except TypeError, e: + message = "Unable to save metadata for this repository probably due to a tool config file that doesn't conform to the Cheetah template syntax." + status = 'error' else: repository_metadata = get_latest_repository_metadata( trans, id ) if repository_metadata: @@ -450,9 +483,13 @@ def set_repository_metadata( trans, id, changeset_revision, content_alert_str='' repository_metadata.changeset_revision = changeset_revision repository_metadata.metadata = metadata_dict trans.sa_session.add( repository_metadata ) - trans.sa_session.flush() + try: + trans.sa_session.flush() + except TypeError, e: + message = "Unable to save metadata for this repository probably due to a tool config file that doesn't conform to the Cheetah template syntax." + status = 'error' else: - # There are no tools in the repository, and we're setting metadat on the repository tip. + # There are no tools in the repository, and we're setting metadata on the repository tip. repository_metadata = trans.model.RepositoryMetadata( repository.id, changeset_revision, metadata_dict ) trans.sa_session.add( repository_metadata ) trans.sa_session.flush() @@ -514,9 +551,9 @@ def reset_all_repository_metadata( trans, id, **kwd ): ancestor_metadata_dict = None for changeset in repo.changelog: current_changeset_revision = str( repo.changectx( changeset ) ) - ctx = get_changectx_for_changeset( trans, repo, current_changeset_revision ) + ctx = get_changectx_for_changeset( repo, current_changeset_revision ) if current_changeset_revision == repository.tip: - current_metadata_dict, invalid_files = generate_metadata_for_repository_tip( trans, id, ctx, current_changeset_revision, repo_dir ) + current_metadata_dict, invalid_files = generate_metadata_for_repository_tip( trans, id, ctx, current_changeset_revision, repo, repo_dir ) else: current_metadata_dict, invalid_files = generate_metadata_for_changeset_revision( trans, id, ctx, current_changeset_revision, repo_dir ) if current_metadata_dict: @@ -538,8 +575,8 @@ def reset_all_repository_metadata( trans, id, **kwd ): create_or_update_repository_metadata( trans, id, repository, ancestor_changeset_revision, ancestor_metadata_dict ) # Keep track of the changeset_revisions that we've persisted. changeset_revisions.append( ancestor_changeset_revision ) - ancestor_changeset_revision = None - ancestor_metadata_dict = None + ancestor_changeset_revision = current_changeset_revision + ancestor_metadata_dict = current_metadata_dict else: # We're either at the first change set in the change log or we have just created or updated # a repository_metadata record. At this point we set the ancestor changeset to the current @@ -722,12 +759,12 @@ def compare_datatypes( ancestor_datatypes, current_datatypes ): # Currently the only way to differentiate datatypes is by name. ancestor_datatype_dtype = ancestor_datatype[ 'dtype' ] ancestor_datatype_extension = ancestor_datatype[ 'extension' ] - ancestor_datatype_mimetype = ancestor_datatype[ 'mimetype' ] + ancestor_datatype_mimetype = ancestor_datatype.get( 'mimetype', None ) found_in_current = False for current_datatype in current_datatypes: if current_datatype[ 'dtype' ] == ancestor_datatype_dtype and \ current_datatype[ 'extension' ] == ancestor_datatype_extension and \ - current_datatype[ 'mimetype' ] == ancestor_datatype_mimetype: + current_datatype.get( 'mimetype', None ) == ancestor_datatype_mimetype: found_in_current = True break if not found_in_current: @@ -740,7 +777,7 @@ def compare_datatypes( ancestor_datatypes, current_datatypes ): def get_repository_by_name( trans, name ): """Get a repository from the database via name""" return trans.sa_session.query( trans.model.Repository ).filter_by( name=name ).one() -def get_changectx_for_changeset( trans, repo, changeset_revision, **kwd ): +def get_changectx_for_changeset( repo, changeset_revision, **kwd ): """Retrieve a specified changectx from a repository""" for changeset in repo.changelog: ctx = repo.changectx( changeset ) @@ -753,15 +790,6 @@ def change_set_is_malicious( trans, id, changeset_revision, **kwd ): if repository_metadata: return repository_metadata.malicious return False -def get_configured_ui(): - # Configure any desired ui settings. - _ui = ui.ui() - # The following will suppress all messages. This is - # the same as adding the following setting to the repo - # hgrc file' [ui] section: - # quiet = True - _ui.setconfig( 'ui', 'quiet', True ) - return _ui def get_user( trans, id ): """Get a user from the database by id""" return trans.sa_session.query( trans.model.User ).get( trans.security.decode_id( id ) ) @@ -864,9 +892,10 @@ def check_file_contents( trans ): return True return False def update_for_browsing( trans, repository, current_working_dir, commit_message='' ): - # Make a copy of a repository's files for browsing, remove from disk all files that - # are not tracked, and commit all added, modified or removed files that have not yet - # been committed. + # This method id deprecated, but we'll keep it around for a while in case we need it. The problem is that hg purge + # is not supported by the mercurial API. + # Make a copy of a repository's files for browsing, remove from disk all files that are not tracked, and commit all + # added, modified or removed files that have not yet been committed. repo_dir = repository.repo_path repo = hg.repository( get_configured_ui(), repo_dir ) # The following will delete the disk copy of only the files in the repository. @@ -963,7 +992,7 @@ def load_tool_from_changeset_revision( trans, repository_id, changeset_revision, # Get the tool config file name from the hgweb url, something like: # /repos/test/convert_chars1/file/e58dcf0026c7/convert_characters.xml old_tool_config_file_name = tool_config.split( '/' )[ -1 ] - ctx = get_changectx_for_changeset( trans, repo, changeset_revision ) + ctx = get_changectx_for_changeset( repo, changeset_revision ) fctx = None for filename in ctx: filename_head, filename_tail = os.path.split( filename ) @@ -997,12 +1026,24 @@ def build_changeset_revision_select_field( trans, repository, selected_value=Non """ repo = hg.repository( get_configured_ui(), repository.repo_path ) options = [] + changeset_tups = [] refresh_on_change_values = [] for repository_metadata in repository.downloadable_revisions: changeset_revision = repository_metadata.changeset_revision - revision_label = get_revision_label( trans, repository, changeset_revision ) - options.append( ( revision_label, changeset_revision ) ) + ctx = get_changectx_for_changeset( repo, changeset_revision ) + if ctx: + rev = '%04d' % ctx.rev() + label = "%s:%s" % ( str( ctx.rev() ), changeset_revision ) + else: + rev = '-1' + label = "-1:%s" % changeset_revision + changeset_tups.append( ( rev, label, changeset_revision ) ) refresh_on_change_values.append( changeset_revision ) + # Sort options by the revision label. Even though the downloadable_revisions query sorts by update_time, + # the changeset revisions may not be sorted correctly because setting metadata over time will reset update_time. + for changeset_tup in sorted( changeset_tups ): + # Display the latest revision first. + options.insert( 0, ( changeset_tup[1], changeset_tup[2] ) ) if add_id_to_name: name = 'changeset_revision_%d' % repository.id else: diff --git a/lib/galaxy/webapps/community/controllers/hg.py b/lib/galaxy/webapps/community/controllers/hg.py index fe37a275125..e1a9960c256 100644 --- a/lib/galaxy/webapps/community/controllers/hg.py +++ b/lib/galaxy/webapps/community/controllers/hg.py @@ -1,5 +1,9 @@ import os, logging from galaxy.web.base.controller import * +from galaxy.webapps.community.controllers.common import * + +from galaxy import eggs +eggs.require('mercurial') from mercurial.hgweb.hgwebdir_mod import hgwebdir from mercurial.hgweb.request import wsgiapplication @@ -10,7 +14,20 @@ class HgController( BaseUIController ): def handle_request( self, trans, **kwd ): # The os command that results in this method being called will look something like # hg clone http://test@127.0.0.1:9009/repos/test/convert_characters1 - return wsgiapplication( make_web_app ) + cmd = kwd.get( 'cmd', None ) + wsgi_app = wsgiapplication( make_web_app ) + # Hack: Add a parameter to requests for which we do not want all repository metadata reset. + reset_metadata = not ( kwd.get( 'no_reset', False ) ) + if cmd == 'listkeys' and reset_metadata: + # This possibly results from an "hg push" from the command line. When doing this, the following 7 commands, in order, + # will be retrieved from environ: between -> capabilities -> heads -> branchmap -> unbundle -> unbundle -> listkeys + path_info = kwd.get( 'path_info', None ) + if path_info: + owner, name = path_info.split( '/' ) + repository = get_repository_by_name_and_owner( trans, name, owner ) + if repository: + reset_all_repository_metadata( trans, trans.security.encode_id( repository.id ) ) + return wsgi_app def make_web_app(): hgweb_config = "%s/hgweb.config" % os.getcwd() diff --git a/lib/galaxy/webapps/community/controllers/repository.py b/lib/galaxy/webapps/community/controllers/repository.py index ebe3a52d6cd..e42bd5bc466 100644 --- a/lib/galaxy/webapps/community/controllers/repository.py +++ b/lib/galaxy/webapps/community/controllers/repository.py @@ -10,19 +10,16 @@ from galaxy.webapps.community.model import directory_hash_id from galaxy.web.framework.helpers import time_ago, iff, grids from galaxy.util.json import from_json_string, to_json_string from galaxy.model.orm import * +from galaxy.util.shed_util import get_configured_ui from common import * + +from galaxy import eggs +eggs.require('mercurial') from mercurial import hg, ui, patch, commands log = logging.getLogger( __name__ ) -# Characters that must be html escaped -MAPPED_CHARS = { '>' :'>', - '<' :'<', - '"' : '"', - '&' : '&', - '\'' : ''' } MAX_CONTENT_SIZE = 32768 -VALID_CHARS = set( string.letters + string.digits + "'\"-=_.()/+*^,:?!#[]%\\$@;{}" ) VALID_REPOSITORYNAME_RE = re.compile( "^[a-z0-9\_]+$" ) class CategoryListGrid( grids.Grid ): @@ -407,6 +404,40 @@ class RepositoryController( BaseUIController, ItemRatings ): # Render the list view return self.valid_repository_list_grid( trans, **kwd ) @web.expose + def browse_invalid_tools( self, trans, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + webapp = params.get( 'webapp', 'community' ) + cntrller = params.get( 'cntrller', 'repository' ) + is_admin = trans.user_is_admin() + invalid_tools_dict = odict() + if is_admin and cntrller == 'admin': + for repository in trans.sa_session.query( trans.model.Repository ) \ + .filter( trans.model.Repository.table.c.deleted == False ) \ + .order_by( trans.model.Repository.table.c.name ): + for downloadable_revision in repository.downloadable_revisions: + metadata = downloadable_revision.metadata + invalid_tools = metadata.get( 'invalid_tools', [] ) + for invalid_tool_config in invalid_tools: + invalid_tools_dict[ invalid_tool_config ] = ( repository.id, repository.name, downloadable_revision.changeset_revision ) + else: + for repository in trans.sa_session.query( trans.model.Repository ) \ + .filter( and_( trans.model.Repository.table.c.deleted == False, + trans.model.Repository.table.c.user_id == trans.user.id ) ) \ + .order_by( trans.model.Repository.table.c.name ): + for downloadable_revision in repository.downloadable_revisions: + metadata = downloadable_revision.metadata + invalid_tools = metadata.get( 'invalid_tools', [] ) + for invalid_tool_config in invalid_tools: + invalid_tools_dict[ invalid_tool_config ] = ( repository.id, repository.name, downloadable_revision.changeset_revision ) + return trans.fill_template( '/webapps/community/repository/browse_invalid_tools.mako', + cntrller=cntrller, + invalid_tools_dict=invalid_tools_dict, + webapp=webapp, + message=message, + status=status ) + @web.expose def find_workflows( self, trans, **kwd ): params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) @@ -444,13 +475,16 @@ class RepositoryController( BaseUIController, ItemRatings ): # This can only occur when there is a multi-select grid with check boxes and an operation, # and the user clicked the operation button without checking any of the check boxes. return trans.show_error_message( "No items were selected." ) - workflow_names = [ item.lower() for item in util.listify( kwd.get( 'workflow_name', '' ) ) ] - exact_matches = params.get( 'exact_matches', '' ) - exact_matches_checked = CheckboxField.is_checked( exact_matches ) - match_tuples = [] - ok = True - if workflow_names: - ok, match_tuples = self.__search_repository_metadata( trans, exact_matches_checked, workflow_names=workflow_names ) + if 'find_workflows_button' in kwd: + workflow_names = [ item.lower() for item in util.listify( kwd.get( 'workflow_name', '' ) ) ] + exact_matches = params.get( 'exact_matches', '' ) + exact_matches_checked = CheckboxField.is_checked( exact_matches ) + match_tuples = [] + ok = True + if workflow_names: + ok, match_tuples = self.__search_repository_metadata( trans, exact_matches_checked, workflow_names=workflow_names ) + else: + ok, match_tuples = self.__search_repository_metadata( trans, exact_matches_checked, workflow_names=[], all_workflows=True ) if ok: kwd[ 'match_tuples' ] = match_tuples # Render the list view @@ -475,6 +509,9 @@ class RepositoryController( BaseUIController, ItemRatings ): else: message = "No search performed - each field must contain the same number of comma-separated items." status = "error" + else: + exact_matches_checked = False + workflow_names = [] exact_matches_check_box = CheckboxField( 'exact_matches', checked=exact_matches_checked ) return trans.fill_template( '/webapps/community/repository/find_workflows.mako', webapp=webapp, @@ -562,7 +599,7 @@ class RepositoryController( BaseUIController, ItemRatings ): exact_matches_check_box=exact_matches_check_box, message=message, status=status ) - def __search_repository_metadata( self, trans, exact_matches_checked, tool_ids='', tool_names='', tool_versions='', workflow_names='' ): + def __search_repository_metadata( self, trans, exact_matches_checked, tool_ids='', tool_names='', tool_versions='', workflow_names='', all_workflows=False ): match_tuples = [] ok = True for repository_metadata in trans.sa_session.query( model.RepositoryMetadata ): @@ -630,6 +667,8 @@ class RepositoryController( BaseUIController, ItemRatings ): for workflow_name in workflow_names: if self.__in_workflow_dict( workflow_dict, exact_matches_checked, workflow_name ): match_tuples.append( ( repository_metadata.repository_id, repository_metadata.changeset_revision ) ) + elif all_workflows and 'workflows' in metadata: + match_tuples.append( ( repository_metadata.repository_id, repository_metadata.changeset_revision ) ) return ok, match_tuples def __in_workflow_dict( self, workflow_dict, exact_matches_checked, workflow_name ): workflow_dict_workflow_name = workflow_dict[ 'name' ].lower() @@ -716,10 +755,14 @@ class RepositoryController( BaseUIController, ItemRatings ): if not includes_tools and 'tools' in repository_metadata.metadata: includes_tools = True repository = get_repository( trans, trans.security.encode_id( repository_metadata.repository_id ) ) - repository_id = trans.security.encode_id( repository.id ) + # Get the changelog rev for this changeset_revision. + repo_dir = repository.repo_path + repo = hg.repository( get_configured_ui(), repo_dir ) changeset_revision = repository_metadata.changeset_revision + ctx = get_changectx_for_changeset( repo, changeset_revision ) + repository_id = trans.security.encode_id( repository.id ) repository_clone_url = generate_clone_url( trans, repository_id ) - repo_info_dict[ repository.name ] = ( repository.description, repository_clone_url, changeset_revision ) + repo_info_dict[ repository.name ] = ( repository.description, repository_clone_url, changeset_revision, str( ctx.rev() ) ) return encode( repo_info_dict ), includes_tools @web.expose def preview_tools_in_changeset( self, trans, repository_id, **kwd ): @@ -765,14 +808,47 @@ class RepositoryController( BaseUIController, ItemRatings ): # Tell the caller if the repository includes Galaxy tools so the page # enabling selection of the tool panel section can be displayed. includes_tools = 'tools' in repository_metadata.metadata + # Get the changelog rev for this changeset_revision. + repo_dir = repository.repo_path + repo = hg.repository( get_configured_ui(), repo_dir ) + ctx = get_changectx_for_changeset( repo, changeset_revision ) repo_info_dict = {} - repo_info_dict[ repository.name ] = ( repository.description, repository_clone_url, changeset_revision ) + repo_info_dict[ repository.name ] = ( repository.description, repository_clone_url, changeset_revision, str( ctx.rev() ) ) encoded_repo_info_dict = encode( repo_info_dict ) # Redirect back to local Galaxy to perform install. url = '%sadmin_toolshed/install_repository?tool_shed_url=%s&repo_info_dict=%s&includes_tools=%s' % \ ( galaxy_url, url_for( '/', qualified=True ), encoded_repo_info_dict, str( includes_tools ) ) return trans.response.send_redirect( url ) @web.expose + def get_ctx_rev( self, trans, **kwd ): + """Given a repository and changeset_revision, return the correct ctx.rev() value.""" + repository_name = kwd[ 'name' ] + repository_owner = kwd[ 'owner' ] + changeset_revision = kwd[ 'changeset_revision' ] + repository = get_repository_by_name_and_owner( trans, repository_name, repository_owner ) + repo_dir = repository.repo_path + repo = hg.repository( get_configured_ui(), repo_dir ) + ctx = get_changectx_for_changeset( repo, changeset_revision ) + if ctx: + return str( ctx.rev() ) + return '' + @web.expose + def get_readme( self, trans, **kwd ): + """If the received changeset_revision includes a file named readme (case ignored), return it's contents.""" + repository_name = kwd[ 'name' ] + repository_owner = kwd[ 'owner' ] + changeset_revision = kwd[ 'changeset_revision' ] + repository = get_repository_by_name_and_owner( trans, repository_name, repository_owner ) + repo_dir = repository.repo_path + for root, dirs, files in os.walk( repo_dir ): + for name in files: + if name.lower() in [ 'readme', 'readme.txt', 'read_me', 'read_me.txt', '%s.txt' % repository_name ]: + f = open( os.path.join( root, name ), 'r' ) + text = f.read() + f.close() + return str( text ) + return '' + @web.expose def get_tool_versions( self, trans, **kwd ): """ For each valid /downloadable change set (up to the received changeset_revision) in the @@ -793,7 +869,7 @@ class RepositoryController( BaseUIController, ItemRatings ): tool_version_dicts.append( repository_metadata.tool_versions ) if current_changeset_revision == changeset_revision: break - if repository_metadata.tool_versions: + if tool_version_dicts: return to_json_string( tool_version_dicts ) return '' @web.expose @@ -810,6 +886,9 @@ class RepositoryController( BaseUIController, ItemRatings ): changeset_revision = params.get( 'changeset_revision', None ) webapp = params.get( 'webapp', 'community' ) repository = get_repository_by_name_and_owner( trans, name, owner ) + repo_dir = repository.repo_path + repo = hg.repository( get_configured_ui(), repo_dir ) + latest_ctx = get_changectx_for_changeset( repo, changeset_revision ) from_update_manager = webapp == 'update_manager' if from_update_manager: update = 'true' @@ -820,10 +899,10 @@ class RepositoryController( BaseUIController, ItemRatings ): url += '&name=%s&owner=%s&changeset_revision=%s&latest_changeset_revision=' % \ ( repository.name, repository.user.username, changeset_revision ) if changeset_revision == repository.tip: - # If changeset_revision is the repository tip, then - # we know there are no additional updates for the tools. + # If changeset_revision is the repository tip, we know there are no additional updates for the tools. if from_update_manager: return no_update + # Return the same value for changeset_revision and latest_changeset_revision. url += repository.tip else: repository_metadata = get_repository_metadata_by_changeset_revision( trans, @@ -834,15 +913,15 @@ class RepositoryController( BaseUIController, ItemRatings ): # repository, then we know there are no additional updates for the tools. if from_update_manager: return no_update - url += changeset_revision + else: + # Return the same value for changeset_revision and latest_changeset_revision. + url += changeset_revision else: # The changeset_revision column in the repository_metadata table has been # updated with a new changeset_revision value since the repository was cloned. - repo_dir = repository.repo_path - repo = hg.repository( get_configured_ui(), repo_dir ) # Load each tool in the repository's changeset_revision to generate a list of # tool guids, since guids differentiate tools by id and version. - ctx = get_changectx_for_changeset( trans, repo, changeset_revision ) + ctx = get_changectx_for_changeset( repo, changeset_revision ) if ctx is not None: tool_guids = [] for filename in ctx: @@ -859,12 +938,21 @@ class RepositoryController( BaseUIController, ItemRatings ): if not ( check_binary( tmp_filename ) or check_image( tmp_filename ) or check_gzip( tmp_filename )[ 0 ] or check_bz2( tmp_filename )[ 0 ] or check_zip( tmp_filename ) ): try: - tool = load_tool( trans, tmp_filename ) - valid = True - except: - valid = False - if valid and tool is not None: - tool_guids.append( generate_tool_guid( trans, repository, tool ) ) + # Make sure we're looking at a tool config and not a display application config or something else. + element_tree = util.parse_xml( tmp_filename ) + element_tree_root = element_tree.getroot() + is_tool = element_tree_root.tag == 'tool' + except Exception, e: + log.debug( "Error parsing %s, exception: %s" % ( tmp_filename, str( e ) ) ) + is_tool = False + if is_tool: + try: + tool = load_tool( trans, tmp_filename ) + valid = True + except: + valid = False + if valid and tool is not None: + tool_guids.append( generate_tool_guid( trans, repository, tool ) ) try: os.unlink( tmp_filename ) except: @@ -882,11 +970,12 @@ class RepositoryController( BaseUIController, ItemRatings ): metadata_tool_guids.append( tool_dict[ 'guid' ] ) metadata_tool_guids.sort() if tool_guids == metadata_tool_guids: - # We've found the repository_metadata record whose changeset_revision - # value has been updated. + # We've found the repository_metadata record whose changeset_revision value has been updated. if from_update_manager: return update url += repository_metadata.changeset_revision + # Get the ctx_rev for the discovered changeset_revision. + latest_ctx = get_changectx_for_changeset( repo, repository_metadata.changeset_revision ) found = True break if not found: @@ -896,10 +985,11 @@ class RepositoryController( BaseUIController, ItemRatings ): return no_update url += changeset_revision else: - # There are not tools in the changeset_revision, so no tool updates are possible. + # There are no tools in the changeset_revision, so no tool updates are possible. if from_update_manager: return no_update url += changeset_revision + url += '&latest_ctx_rev=%s' % str( latest_ctx.rev() ) return trans.response.send_redirect( url ) @web.expose def browse_repositories( self, trans, **kwd ): @@ -1123,13 +1213,10 @@ class RepositoryController( BaseUIController, ItemRatings ): new_hgweb_config.flush() shutil.move( tmp_fname, os.path.abspath( hgweb_config ) ) def __create_hgrc_file( self, repository ): - # At this point, an entry for the repository is required to be in the hgweb.config - # file so we can call repository.repo_path. - # Since we support both http and https, we set push_ssl to False to override - # the default (which is True) in the mercurial api. - # The hg purge extension purges all files and directories not being tracked by - # mercurial in the current repository. It'll remove unknown files and empty - # directories. This is used in the update_for_browsing() method. + # At this point, an entry for the repository is required to be in the hgweb.config file so we can call repository.repo_path. + # Since we support both http and https, we set push_ssl to False to override the default (which is True) in the mercurial api. + # The hg purge extension purges all files and directories not being tracked by mercurial in the current repository. It'll + # remove unknown files and empty directories. This is not currently used because it is not supported in the mercurial API. repo = hg.repository( get_configured_ui(), path=repository.repo_path ) fp = repo.opener( 'hgrc', 'wb' ) fp.write( '[paths]\n' ) @@ -1153,7 +1240,7 @@ class RepositoryController( BaseUIController, ItemRatings ): repo = hg.repository( get_configured_ui(), repository.repo_path ) current_working_dir = os.getcwd() # Update repository files for browsing. - update_for_browsing( trans, repository, current_working_dir, commit_message=commit_message ) + update_repository( repo ) is_malicious = change_set_is_malicious( trans, id, repository.tip ) return trans.fill_template( '/webapps/community/repository/browse_repository.mako', repo=repo, @@ -1269,7 +1356,7 @@ class RepositoryController( BaseUIController, ItemRatings ): repo.commit( user=trans.user.username, text=commit_message ) handle_email_alerts( trans, repository ) # Update the repository files for browsing. - update_for_browsing( trans, repository, current_working_dir, commit_message=commit_message ) + update_repository( repo ) # Get the new repository tip. repo = hg.repository( get_configured_ui(), repo_dir ) if tip != repository.tip: @@ -1567,7 +1654,7 @@ class RepositoryController( BaseUIController, ItemRatings ): status = params.get( 'status', 'done' ) repository = get_repository( trans, id ) repo = hg.repository( get_configured_ui(), repository.repo_path ) - ctx = get_changectx_for_changeset( trans, repo, ctx_str ) + ctx = get_changectx_for_changeset( repo, ctx_str ) if ctx is None: message = "Repository does not include changeset revision '%s'." % str( ctx_str ) status = 'error' @@ -1581,7 +1668,7 @@ class RepositoryController( BaseUIController, ItemRatings ): anchors = modified + added + removed + deleted + unknown + ignored + clean diffs = [] for diff in patch.diff( repo, node1=ctx_parent.node(), node2=ctx.node() ): - diffs.append( self.to_html_escaped( diff ) ) + diffs.append( to_html_escaped( diff ) ) is_malicious = change_set_is_malicious( trans, id, repository.tip ) return trans.fill_template( '/webapps/community/repository/view_changeset.mako', repository=repository, @@ -1802,6 +1889,82 @@ class RepositoryController( BaseUIController, ItemRatings ): changeset_revision=changeset_revision, message=message, status='error' ) ) + @web.expose + def load_invalid_tool( self, trans, repository_id, tool_config, changeset_revision, **kwd ): + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'error' ) + webapp = params.get( 'webapp', 'community' ) + repository = get_repository( trans, repository_id ) + repo_dir = repository.repo_path + repo = hg.repository( get_configured_ui(), repo_dir ) + ctx = get_changectx_for_changeset( repo, changeset_revision ) + invalid_message = '' + if changeset_revision == repository.tip: + for root, dirs, files in os.walk( repo_dir ): + found = False + for name in files: + if name == tool_config: + tool_config_path = os.path.join( root, name ) + found = True + break + if found: + break + metadata_dict, invalid_files = generate_metadata_for_repository_tip( trans, repository_id, ctx, changeset_revision, repo, repo_dir ) + else: + for filename in ctx: + if filename == tool_config: + fctx = ctx[ filename ] + # Write the contents of datatypes_config.xml to a temporary file. + fh = tempfile.NamedTemporaryFile( 'w' ) + tool_config_path = fh.name + fh.close() + fh = open( tool_config_path, 'w' ) + fh.write( fctx.data() ) + fh.close() + break + metadata_dict, invalid_files = generate_metadata_for_changeset_revision( trans, repository_id, ctx, changeset_revision, repo_dir ) + for invalid_file_tup in invalid_files: + invalid_tool_config, invalid_msg = invalid_file_tup + if tool_config == invalid_tool_config: + invalid_message = invalid_msg + break + tool, message = load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_config_path ) + tool_state = self.__new_state( trans ) + is_malicious = change_set_is_malicious( trans, repository_id, repository.tip ) + if changeset_revision != repository.tip: + try: + os.unlink( tool_config_path ) + except: + pass + try: + if invalid_message: + message = invalid_message + return trans.fill_template( "/webapps/community/repository/tool_form.mako", + repository=repository, + changeset_revision=changeset_revision, + tool=tool, + tool_state=tool_state, + is_malicious=is_malicious, + webapp=webapp, + message=message, + status='error' ) + except Exception, e: + message = "This tool is invalid because: %s." % str( e ) + if webapp == 'galaxy': + return trans.response.send_redirect( web.url_for( controller='repository', + action='preview_tools_in_changeset', + repository_id=repository_id, + changeset_revision=changeset_revision, + message=message, + status='error' ) ) + return trans.response.send_redirect( web.url_for( controller='repository', + action='browse_repositories', + operation='view_or_manage_repository', + id=repository_id, + changeset_revision=changeset_revision, + message=message, + status='error' ) ) def __new_state( self, trans, all_pages=False ): """ Create a new `DefaultToolState` for this tool. It will not be initialized @@ -1919,20 +2082,20 @@ class RepositoryController( BaseUIController, ItemRatings ): trans.response.headers['Pragma'] = 'no-cache' trans.response.headers['Expires'] = '0' if is_gzip( file_path ): - to_html = self.to_html_str( '\ngzip compressed file\n' ) + to_html = to_html_str( '\ngzip compressed file\n' ) elif is_bz2( file_path ): - to_html = self.to_html_str( '\nbz2 compressed file\n' ) + to_html = to_html_str( '\nbz2 compressed file\n' ) elif check_zip( file_path ): - to_html = self.to_html_str( '\nzip compressed file\n' ) + to_html = to_html_str( '\nzip compressed file\n' ) elif check_binary( file_path ): - to_html = self.to_html_str( '\nBinary file\n' ) + to_html = to_html_str( '\nBinary file\n' ) else: to_html = '' for i, line in enumerate( open( file_path ) ): - to_html = '%s%s' % ( to_html, self.to_html_str( line ) ) + to_html = '%s%s' % ( to_html, to_html_str( line ) ) if len( to_html ) > MAX_CONTENT_SIZE: large_str = '\nFile contents truncated because file size is larger than maximum viewing size of %s\n' % util.nice_size( MAX_CONTENT_SIZE ) - to_html = '%s%s' % ( to_html, self.to_html_str( large_str ) ) + to_html = '%s%s' % ( to_html, to_html_str( large_str ) ) break return to_html @web.expose @@ -1941,34 +2104,6 @@ class RepositoryController( BaseUIController, ItemRatings ): message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) return trans.fill_template( '/webapps/community/repository/help.mako', message=message, status=status, **kwd ) - def to_html_escaped( self, text ): - """Translates the characters in text to html values""" - translated = [] - for c in text: - if c in [ '\r\n', '\n', ' ', '\t' ] or c in VALID_CHARS: - translated.append( c ) - elif c in MAPPED_CHARS: - translated.append( MAPPED_CHARS[ c ] ) - else: - translated.append( 'X' ) - return ''.join( translated ) - def to_html_str( self, text ): - """Translates the characters in text to sn html string""" - translated = [] - for c in text: - if c in VALID_CHARS: - translated.append( c ) - elif c in MAPPED_CHARS: - translated.append( MAPPED_CHARS[ c ] ) - elif c == ' ': - translated.append( ' ' ) - elif c == '\t': - translated.append( '    ' ) - elif c == '\n': - translated.append( '
    ' ) - elif c not in [ '\r' ]: - translated.append( 'X' ) - return ''.join( translated ) def __build_allow_push_select_field( self, trans, current_push_list, selected_value='none' ): options = [] for user in trans.sa_session.query( trans.model.User ): diff --git a/lib/galaxy/webapps/community/controllers/upload.py b/lib/galaxy/webapps/community/controllers/upload.py index 7dd14c981e0..6dba15d8071 100644 --- a/lib/galaxy/webapps/community/controllers/upload.py +++ b/lib/galaxy/webapps/community/controllers/upload.py @@ -3,7 +3,10 @@ from galaxy.web.base.controller import * from galaxy.model.orm import * from galaxy.datatypes.checkers import * from common import * -from galaxy.util.shed_util import handle_sample_tool_data_table_conf_file +from galaxy.util.shed_util import get_configured_ui, handle_sample_tool_data_table_conf_file + +from galaxy import eggs +eggs.require('mercurial') from mercurial import hg, ui, commands log = logging.getLogger( __name__ ) @@ -119,14 +122,9 @@ class UploadController( BaseUIController ): else: content_alert_str = '' commands.add( repo.ui, repo, full_path ) - try: - commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message ) - except Exception, e: - # I never have a problem with commands.commit on a Mac, but in the test/production - # tool shed environment, it occasionally throws a "TypeError: array item must be char" - # exception. If this happens, we'll try the following. - repo.dirstate.write() - repo.commit( user=trans.user.username, text=commit_message ) + # Convert from unicode to prevent "TypeError: array item must be char" + full_path = full_path.encode( 'ascii', 'replace' ) + commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message ) if full_path.endswith( 'tool_data_table_conf.xml.sample' ): # Handle the special case where a tool_data_table_conf.xml.sample # file is being uploaded by parsing the file and adding new entries @@ -136,15 +134,14 @@ class UploadController( BaseUIController ): if error: message = '%s
    %s' % ( message, error_message ) if full_path.endswith( '.loc.sample' ): - # Handle the special case where a xxx.loc.sample file is - # being uploaded by copying it to ~/tool-data/xxx.loc. - copy_sample_loc_file( trans.app, full_path ) + # Handle the special case where a xxx.loc.sample file is being uploaded by copying it to ~/tool-data/xxx.loc. + copy_sample_file( trans.app, full_path ) # See if the content of the change set was valid. admin_only = len( repository.downloadable_revisions ) != 1 handle_email_alerts( trans, repository, content_alert_str=content_alert_str, new_repo_alert=new_repo_alert, admin_only=admin_only ) if ok: # Update the repository files for browsing. - update_for_browsing( trans, repository, current_working_dir, commit_message=commit_message ) + update_repository( repo ) # Get the new repository tip. if tip != repository.tip: if ( isgzip or isbz2 ) and uncompress_file: @@ -201,18 +198,18 @@ class UploadController( BaseUIController ): repo = hg.repository( get_configured_ui(), repo_dir ) files_to_remove = [] content_alert_str = '' + undesirable_dirs_removed = 0 + undesirable_files_removed = 0 ok, message = self.__check_archive( tar ) if not ok: tar.close() uploaded_file.close() - return ok, message, files_to_remove, content_alert_str + return ok, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed else: if upload_point is not None: full_path = os.path.abspath( os.path.join( repo_dir, upload_point ) ) else: full_path = os.path.abspath( repo_dir ) - undesirable_dirs_removed = 0 - undesirable_files_removed = 0 filenames_in_archive = [] for tarinfo_obj in tar.getmembers(): ok = os.path.basename( tarinfo_obj.name ) not in undesirable_files @@ -249,14 +246,11 @@ class UploadController( BaseUIController ): if full_name not in filenames_in_archive: files_to_remove.append( full_name ) for repo_file in files_to_remove: - # Remove files in the repository (relative to the upload point) - # that are not in the uploaded archive. + # Remove files in the repository (relative to the upload point) that are not in the uploaded archive. try: commands.remove( repo.ui, repo, repo_file, force=True ) except Exception, e: - # I never have a problem with commands.remove on a Mac, but in the test/production - # tool shed environment, it throws an exception whenever I delete all files from a - # repository. If this happens, we'll try the following. + log.debug( "Error removing files using the mercurial API, so trying a different approach, the error was: %s" % str( e )) relative_selected_file = selected_file.split( 'repo_%d' % repository.id )[1].lstrip( '/' ) repo.dirstate.remove( relative_selected_file ) repo.dirstate.write() @@ -290,19 +284,11 @@ class UploadController( BaseUIController ): # appending them to the shed's tool_data_table_conf.xml file on disk. error, message = handle_sample_tool_data_table_conf_file( trans.app, filename_in_archive ) if error: - return False, message, files_to_remove, content_alert_str + return False, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed if filename_in_archive.endswith( '.loc.sample' ): - # Handle the special case where a xxx.loc.sample file is - # being uploaded by copying it to ~/tool-data/xxx.loc. - copy_sample_loc_file( trans.app, filename_in_archive ) - try: - commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message ) - except Exception, e: - # I never have a problem with commands.commit on a Mac, but in the test/production - # tool shed environment, it occasionally throws a "TypeError: array item must be char" - # exception. If this happens, we'll try the following. - repo.dirstate.write() - repo.commit( user=trans.user.username, text=commit_message ) + # Handle the special case where a xxx.loc.sample file is being uploaded by copying it to ~/tool-data/xxx.loc. + copy_sample_file( trans.app, filename_in_archive ) + commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message ) # See if the content of the change set was valid. admin_only = len( repository.downloadable_revisions ) != 1 handle_email_alerts( trans, repository, content_alert_str=content_alert_str, new_repo_alert=new_repo_alert, admin_only=admin_only ) diff --git a/lib/galaxy/webapps/community/framework/middleware/hg.py b/lib/galaxy/webapps/community/framework/middleware/hg.py index 176e534dd66..a9cbbc2748d 100644 --- a/lib/galaxy/webapps/community/framework/middleware/hg.py +++ b/lib/galaxy/webapps/community/framework/middleware/hg.py @@ -3,7 +3,6 @@ Middleware for handling hg authentication for users pushing change sets to local """ import os, logging from sqlalchemy import * -from mercurial import ui, hg from paste.auth.basic import AuthBasicAuthenticator from paste.httpheaders import REMOTE_USER, AUTH_TYPE diff --git a/lib/galaxy/webapps/community/model/__init__.py b/lib/galaxy/webapps/community/model/__init__.py index 1d8d755ffce..1065fe47314 100644 --- a/lib/galaxy/webapps/community/model/__init__.py +++ b/lib/galaxy/webapps/community/model/__init__.py @@ -9,7 +9,11 @@ from galaxy import util from galaxy.util.bunch import Bunch from galaxy.util.hash_util import * from galaxy.web.form_builder import * + +from galaxy import eggs +eggs.require('mercurial') from mercurial import hg, ui + log = logging.getLogger( __name__ ) class User( object ): diff --git a/lib/galaxy/webapps/community/model/mapping.py b/lib/galaxy/webapps/community/model/mapping.py index 7f476706410..43c0651151c 100644 --- a/lib/galaxy/webapps/community/model/mapping.py +++ b/lib/galaxy/webapps/community/model/mapping.py @@ -13,6 +13,7 @@ from galaxy.model.orm import * from galaxy.model.orm.ext.assignmapper import * from galaxy.model.custom_types import * from galaxy.util.bunch import Bunch +from galaxy.util.shed_util import ShedCounter from galaxy.webapps.community.security import CommunityRBACAgent metadata = MetaData() @@ -247,4 +248,5 @@ def init( file_path, url, engine_options={}, create_tables=False ): result.create_tables = create_tables #load local galaxy security policy result.security_agent = CommunityRBACAgent( result ) + result.shed_counter = ShedCounter( result ) return result diff --git a/lib/galaxy/webapps/community/security/__init__.py b/lib/galaxy/webapps/community/security/__init__.py index a265e547de7..b43c6530c61 100644 --- a/lib/galaxy/webapps/community/security/__init__.py +++ b/lib/galaxy/webapps/community/security/__init__.py @@ -6,7 +6,6 @@ from datetime import datetime, timedelta from galaxy.util.bunch import Bunch from galaxy.util import listify from galaxy.model.orm import * -from mercurial import hg, ui log = logging.getLogger(__name__) diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 2dbfd7b2ab0..7f915c18ed6 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -199,7 +199,7 @@ class ToolModule( WorkflowModule ): # TODO: If workflows are ever enhanced to use tool version # in addition to tool id, enhance the selection process here # to retrieve the correct version of the tool. - tool_version = self.__get_tool_version( trans, tool_id ) + tool_version = Class.__get_tool_version( trans, tool_id ) if tool_version: tool_version_ids = tool_version.get_version_ids( trans.app ) for tool_version_id in tool_version_ids: @@ -219,7 +219,8 @@ class ToolModule( WorkflowModule ): module.post_job_actions = pjadict return module return None - def __get_tool_version( self, trans, tool_id ): + @classmethod + def __get_tool_version( cls, trans, tool_id ): # Return a ToolVersion if one exists for tool_id. return trans.sa_session.query( trans.app.model.ToolVersion ) \ .filter( trans.app.model.ToolVersion.table.c.tool_id == tool_id ) \ diff --git a/run.sh b/run.sh old mode 100755 new mode 100644 index e52f2f6bfdb..ca14ece28a7 --- a/run.sh +++ b/run.sh @@ -52,4 +52,17 @@ if [ $FETCH_EGGS -eq 1 ]; then fi fi -python ./scripts/paster.py serve universe_wsgi.ini $@ +if [ -n "$GALAXY_RUN_ALL" ]; then + servers=`sed -n 's/^\[server:\(.*\)\]/\1/ p' universe_wsgi.ini | xargs echo` + daemon=`echo "$@" | grep -q daemon` + if [ $? -ne 0 ]; then + echo 'ERROR: $GALAXY_RUN_ALL cannot be used without the `--daemon` or `--stop-daemon` arguments to run.sh' + exit 1 + fi + for server in $servers; do + echo "Handling $server with log file $server.log..." + python ./scripts/paster.py serve universe_wsgi.ini --server-name=$server --pid-file=$server.pid --log-file=$server.log $@ + done +else + python ./scripts/paster.py serve universe_wsgi.ini $@ +fi diff --git a/run_functional_tests.sh b/run_functional_tests.sh index 4558ad093e9..b1fe35f0148 100755 --- a/run_functional_tests.sh +++ b/run_functional_tests.sh @@ -20,6 +20,24 @@ elif [ $1 = '-list' ]; then echo "===========================================================================================================================================" echo "'run_functional_tests.sh -id bbb' for testing one tool with id 'bbb' ('bbb' is the tool id)" echo "'run_functional_tests.sh -sid ccc' for testing one section with sid 'ccc' ('ccc' is the string after 'section::')" +elif [ $1 = '-migrated' ]; then + if [ ! $2 ]; then + python ./scripts/functional_tests.py -v functional.test_toolbox --with-nosehtml --html-report-file run_functional_tests.html -migrated + elif [ $2 = '-id' ]; then + # TODO: This option is not tested... + python ./scripts/functional_tests.py -v functional.test_toolbox:TestForTool_$3 --with-nosehtml --html-report-file run_functional_tests.html -migrated + else + python ./scripts/functional_tests.py -v functional.test_toolbox --with-nosehtml --html-report-file run_functional_tests.html -migrated + fi +elif [ $1 = '-installed' ]; then + if [ ! $2 ]; then + python ./scripts/functional_tests.py -v functional.test_toolbox --with-nosehtml --html-report-file run_functional_tests.html -installed + elif [ $2 = '-id' ]; then + # TODO: This option is not tested... + python ./scripts/functional_tests.py -v functional.test_toolbox:TestForTool_$3 --with-nosehtml --html-report-file run_functional_tests.html -installed + else + python ./scripts/functional_tests.py -v functional.test_toolbox --with-nosehtml --html-report-file run_functional_tests.html -installed + fi else python ./scripts/functional_tests.py -v --with-nosehtml --html-report-file run_functional_tests.html $1 fi diff --git a/scripts/api/README b/scripts/api/README index 3597c7c6241..d2ca618ac4b 100644 --- a/scripts/api/README +++ b/scripts/api/README @@ -3,7 +3,6 @@ until the documentation is written. Set these options in universe_wsgi.ini and start the server: -enable_api = True admin_users = you@example.org library_import_dir = /path/to/some/directory diff --git a/scripts/check_galaxy.py b/scripts/check_galaxy.py index f4f418516c0..e2ce1001aa3 100755 --- a/scripts/check_galaxy.py +++ b/scripts/check_galaxy.py @@ -249,7 +249,7 @@ class Browser: #return((p.id, p.status)) def diff(self): - self.get("/datasets/%s/display/index" % self.id) + self.get("/datasets/%s/display/display?to_ext=bed" % self.id) data = tc.browser.get_html() tmp = tempfile.mkstemp() dprint("tmp file: %s" % tmp[1]) diff --git a/scripts/check_python.py b/scripts/check_python.py index 15017c9d2c0..e6fdc00a7c0 100644 --- a/scripts/check_python.py +++ b/scripts/check_python.py @@ -1,14 +1,14 @@ import os, sys msg = """ERROR: Your Python version is: %s -Galaxy is currently supported on Python 2.4, 2.5 and 2.6. To run Galaxy, +Galaxy is currently supported on Python 2.5, 2.6 and 2.7. To run Galaxy, please download and install a supported version from python.org. If a supported version is installed but is not your default, getgalaxy.org contains instructions on how to force Galaxy to use a different version.""" % sys.version[:3] def check_python(): try: - assert sys.version_info[:2] >= ( 2, 4 ) and sys.version_info[:2] <= ( 2, 7 ) + assert sys.version_info[:2] >= ( 2, 5 ) and sys.version_info[:2] <= ( 2, 7 ) except AssertionError: print >>sys.stderr, msg raise diff --git a/scripts/cleanup_datasets/cleanup_datasets.py b/scripts/cleanup_datasets/cleanup_datasets.py index 24e0ed8400c..d9b72554f5f 100644 --- a/scripts/cleanup_datasets/cleanup_datasets.py +++ b/scripts/cleanup_datasets/cleanup_datasets.py @@ -505,7 +505,7 @@ def _purge_folder( folder, app, remove_from_disk, info_only = False ): class CleanupDatasetsApplication( object ): """Encapsulates the state of a Universe application""" def __init__( self, config ): - if config.database_connection is None: + if config.database_connection is False: config.database_connection = "sqlite:///%s?isolation_level=IMMEDIATE" % config.database self.object_store = build_object_store_from_config( config ) # Setup the database engine and ORM diff --git a/scripts/functional_tests.py b/scripts/functional_tests.py index 7022bd5c445..5dedaa76a4b 100644 --- a/scripts/functional_tests.py +++ b/scripts/functional_tests.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -import os, sys, shutil +import os, sys, shutil, tempfile, re # Assume we are run from the galaxy root directory, add lib to the python path cwd = os.getcwd() @@ -32,6 +32,13 @@ from galaxy.app import UniverseApplication from galaxy.web import buildapp from galaxy import tools from galaxy.util import bunch +from galaxy import util +from galaxy.util.json import to_json_string + +import nose.core +import nose.config +import nose.loader +import nose.plugins.manager log = logging.getLogger( "functional_tests.py" ) @@ -40,28 +47,131 @@ default_galaxy_test_port_min = 8000 default_galaxy_test_port_max = 9999 default_galaxy_locales = 'en' default_galaxy_test_file_dir = "test-data" +migrated_tool_panel_config = 'migrated_tools_conf.xml' +installed_tool_panel_configs = [ 'shed_tool_conf.xml' ] -def main(): - +def parse_tool_panel_config( config, shed_tools_dict ): + """ + Parse a shed-related tool panel config to generate the shed_tools_dict. This only happens when testing tools installed from the tool shed. + """ + last_galaxy_test_file_dir = None + last_tested_repository_name = None + last_tested_changeset_revision = None + tree = util.parse_xml( config ) + root = tree.getroot() + for elem in root: + if elem.tag == 'tool': + galaxy_test_file_dir, \ + last_tested_repository_name, \ + last_tested_changeset_revision = get_installed_repository_info( elem, + last_galaxy_test_file_dir, + last_tested_repository_name, + last_tested_changeset_revision ) + if galaxy_test_file_dir: + if galaxy_test_file_dir != last_galaxy_test_file_dir: + if not os.path.isabs( galaxy_test_file_dir ): + galaxy_test_file_dir = os.path.join( os.getcwd(), galaxy_test_file_dir ) + guid = elem.get( 'guid' ) + shed_tools_dict[ guid ] = galaxy_test_file_dir + last_galaxy_test_file_dir = galaxy_test_file_dir + elif elem.tag == 'section': + for section_elem in elem: + if section_elem.tag == 'tool': + galaxy_test_file_dir, \ + last_tested_repository_name, \ + last_tested_changeset_revision = get_installed_repository_info( section_elem, + last_galaxy_test_file_dir, + last_tested_repository_name, + last_tested_changeset_revision ) + if galaxy_test_file_dir: + if galaxy_test_file_dir != last_galaxy_test_file_dir: + if not os.path.isabs( galaxy_test_file_dir ): + galaxy_test_file_dir = os.path.join( os.getcwd(), galaxy_test_file_dir ) + guid = section_elem.get( 'guid' ) + shed_tools_dict[ guid ] = galaxy_test_file_dir + last_galaxy_test_file_dir = galaxy_test_file_dir + return shed_tools_dict + +def get_installed_repository_info( elem, last_galaxy_test_file_dir, last_tested_repository_name, last_tested_changeset_revision ): + """ + Return the GALAXY_TEST_FILE_DIR, the containing repository name and the change set revision for the tool elem. + This only happens when testing tools installed from the tool shed. + """ + tool_config_path = elem.get( 'file' ) + installed_tool_path_items = tool_config_path.split( '/repos/' ) + sans_shed = installed_tool_path_items[ 1 ] + path_items = sans_shed.split( '/' ) + repository_owner = path_items[ 0 ] + repository_name = path_items[ 1 ] + changeset_revision = path_items[ 2 ] + if repository_name != last_tested_repository_name or changeset_revision != last_tested_changeset_revision: + # Locate the test-data directory. + installed_tool_path = os.path.join( installed_tool_path_items[ 0 ], 'repos', repository_owner, repository_name, changeset_revision ) + for root, dirs, files in os.walk( installed_tool_path ): + if 'test-data' in dirs: + return os.path.join( root, 'test-data' ), repository_name, changeset_revision + return None, repository_name, changeset_revision + return last_galaxy_test_file_dir, last_tested_repository_name, last_tested_changeset_revision + +def run_tests( test_config ): + loader = nose.loader.TestLoader( config=test_config ) + plug_loader = test_config.plugins.prepareTestLoader( loader ) + if plug_loader is not None: + loader = plug_loader + tests = loader.loadTestsFromNames( test_config.testNames ) + test_runner = nose.core.TextTestRunner( stream=test_config.stream, + verbosity=test_config.verbosity, + config=test_config ) + plug_runner = test_config.plugins.prepareTestRunner( test_runner ) + if plug_runner is not None: + test_runner = plug_runner + return test_runner.run( tests ) + +def main(): # ---- Configuration ------------------------------------------------------ - galaxy_test_host = os.environ.get( 'GALAXY_TEST_HOST', default_galaxy_test_host ) galaxy_test_port = os.environ.get( 'GALAXY_TEST_PORT', None ) galaxy_test_save = os.environ.get( 'GALAXY_TEST_SAVE', None) - - if 'HTTP_ACCEPT_LANGUAGE' not in os.environ: - os.environ['HTTP_ACCEPT_LANGUAGE'] = default_galaxy_locales - galaxy_test_file_dir = os.environ.get( 'GALAXY_TEST_FILE_DIR', default_galaxy_test_file_dir ) - if not os.path.isabs( galaxy_test_file_dir ): - galaxy_test_file_dir = os.path.join( os.getcwd(), galaxy_test_file_dir ) - start_server = 'GALAXY_TEST_EXTERNAL' not in os.environ tool_path = os.environ.get( 'GALAXY_TEST_TOOL_PATH', 'tools' ) - tool_config_file = os.environ.get( 'GALAXY_TEST_TOOL_CONF', 'tool_conf.xml.sample' ) - tool_data_table_config_path = 'tool_data_table_conf.xml' - tool_dependency_dir = os.environ.get( 'GALAXY_TOOL_DEPENDENCY_DIR', None ) - use_distributed_object_store = os.environ.get( 'GALAXY_USE_DISTRIBUTED_OBJECT_STORE', False ) + if 'HTTP_ACCEPT_LANGUAGE' not in os.environ: + os.environ[ 'HTTP_ACCEPT_LANGUAGE' ] = default_galaxy_locales + testing_migrated_tools = '-migrated' in sys.argv + testing_installed_tools = '-installed' in sys.argv + + if testing_migrated_tools or testing_installed_tools: + sys.argv.pop() + # Store a jsonified dictionary of tool_id : GALAXY_TEST_FILE_DIR pairs. + galaxy_tool_shed_test_file = 'shed_tools_dict' + # We need the upload tool for functional tests, so we'll create a temporary tool panel config that defines it. + fd, tmp_tool_panel_conf = tempfile.mkstemp() + os.write( fd, '\n' ) + os.write( fd, '\n' ) + os.write( fd, '\n' ) + os.write( fd, '\n' ) + os.close( fd ) + tool_config_file = tmp_tool_panel_conf + galaxy_test_file_dir = None + library_import_dir = None + user_library_import_dir = None + # Exclude all files except test_toolbox.py. + ignore_files = ( re.compile( r'^test_[adghlmsu]*' ), re.compile( r'^test_ta*' ) ) + else: + tool_config_file = os.environ.get( 'GALAXY_TEST_TOOL_CONF', 'tool_conf.xml.sample' ) + galaxy_test_file_dir = os.environ.get( 'GALAXY_TEST_FILE_DIR', default_galaxy_test_file_dir ) + if not os.path.isabs( galaxy_test_file_dir ): + galaxy_test_file_dir = os.path.join( os.getcwd(), galaxy_test_file_dir ) + library_import_dir = galaxy_test_file_dir + user_library_import_dir = os.path.join( galaxy_test_file_dir, 'users' ) + ignore_files = () + + start_server = 'GALAXY_TEST_EXTERNAL' not in os.environ if os.path.exists( 'tool_data_table_conf.test.xml' ): tool_data_table_config_path = 'tool_data_table_conf.test.xml' + else: + tool_data_table_config_path = 'tool_data_table_conf.xml' + tool_dependency_dir = os.environ.get( 'GALAXY_TOOL_DEPENDENCY_DIR', None ) + use_distributed_object_store = os.environ.get( 'GALAXY_USE_DISTRIBUTED_OBJECT_STORE', False ) + if start_server: psu_production = False galaxy_test_proxy_port = None @@ -126,38 +236,21 @@ def main(): try: os.makedirs( dir ) except OSError: - pass - + pass print "Database connection:", database_connection - - # What requires these? - # handy for (eg) functional tests to save outputs? - if galaxy_test_save: - os.environ['GALAXY_TEST_SAVE'] = galaxy_test_save - # pass in through script setenv - # will leave a copy of ALL test validate files - os.environ['GALAXY_TEST_HOST'] = galaxy_test_host - os.environ['GALAXY_TEST_FILE_DIR'] = galaxy_test_file_dir - # ---- Build Application -------------------------------------------------- - - app = None - + # ---- Build Application -------------------------------------------------- + app = None if start_server: - global_conf = { '__file__' : 'universe_wsgi.ini.sample' } if psu_production: global_conf = None - if not database_connection.startswith( 'sqlite://' ): - kwargs['database_engine_option_max_overflow'] = '20' - + kwargs[ 'database_engine_option_max_overflow' ] = '20' if tool_dependency_dir is not None: - kwargs['tool_dependency_dir'] = tool_dependency_dir - + kwargs[ 'tool_dependency_dir' ] = tool_dependency_dir if use_distributed_object_store: - kwargs['object_store'] = 'distributed' - kwargs['distributed_object_store_config_file'] = 'distributed_object_store_conf.xml.sample' - + kwargs[ 'object_store' ] = 'distributed' + kwargs[ 'distributed_object_store_config_file' ] = 'distributed_object_store_conf.xml.sample' # Build the Universe Application app = UniverseApplication( job_queue_workers = 5, id_secret = 'changethisinproductiontoo', @@ -179,21 +272,17 @@ def main(): allow_user_deletion = True, admin_users = 'test@bx.psu.edu', allow_library_path_paste = True, - library_import_dir = galaxy_test_file_dir, - user_library_import_dir = os.path.join( galaxy_test_file_dir, 'users' ), + library_import_dir = library_import_dir, + user_library_import_dir = user_library_import_dir, global_conf = global_conf, + running_functional_tests=True, **kwargs ) - - log.info( "Embedded Universe application started" ); - + log.info( "Embedded Universe application started" ) # ---- Run webserver ------------------------------------------------------ - server = None if start_server: - - webapp = buildapp.app_factory( dict(), use_translogger = False, static_enabled = False, app=app ) - + webapp = buildapp.app_factory( dict(), use_translogger=False, static_enabled=False, app=app ) if galaxy_test_port is not None: server = httpserver.serve( webapp, host=galaxy_test_host, port=galaxy_test_port, start_loop=False ) else: @@ -217,7 +306,6 @@ def main(): t = threading.Thread( target=server.serve_forever ) t.start() - # Test if the server is up for i in range( 10 ): conn = httplib.HTTPConnection( galaxy_test_host, galaxy_test_port ) # directly test the app, not the proxy @@ -227,85 +315,83 @@ def main(): time.sleep( 0.1 ) else: raise Exception( "Test HTTP server did not return '200 OK' after 10 tries" ) - # Test if the proxy server is up if psu_production: conn = httplib.HTTPConnection( galaxy_test_host, galaxy_test_proxy_port ) # directly test the app, not the proxy conn.request( "GET", "/" ) if not conn.getresponse().status == 200: raise Exception( "Test HTTP proxy server did not return '200 OK'" ) - log.info( "Embedded web server started" ) - - # ---- Load toolbox for generated tests ----------------------------------- - # We don't add the tests to the path until everything is up and running new_path = [ os.path.join( cwd, "test" ) ] new_path.extend( sys.path[1:] ) sys.path = new_path - import functional.test_toolbox - - if app: - # TODO: provisions for loading toolbox from file when using external server - functional.test_toolbox.toolbox = app.toolbox - functional.test_toolbox.build_tests() - else: - # FIXME: This doesn't work at all now that toolbox requires an 'app' instance - # (to get at datatypes, might just pass a datatype registry directly) - datatypes_registry = galaxy.datatypes.registry.Registry() - datatypes_registry.load_datatypes() - my_app = bunch.Bunch( datatypes_registry ) - test_toolbox.toolbox = tools.ToolBox( 'tool_conf.xml.test', 'tools', my_app ) - # ---- Find tests --------------------------------------------------------- - if galaxy_test_proxy_port: log.info( "Functional tests will be run against %s:%s" % ( galaxy_test_host, galaxy_test_proxy_port ) ) else: log.info( "Functional tests will be run against %s:%s" % ( galaxy_test_host, galaxy_test_port ) ) - success = False - try: - - import nose.core - import nose.config - import nose.loader - import nose.plugins.manager - - test_config = nose.config.Config( env = os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) - test_config.configure( sys.argv ) - - loader = nose.loader.TestLoader( config = test_config ) - - plug_loader = test_config.plugins.prepareTestLoader( loader ) - if plug_loader is not None: - loader = plug_loader - - tests = loader.loadTestsFromNames( test_config.testNames ) - - test_runner = nose.core.TextTestRunner( - stream = test_config.stream, - verbosity = test_config.verbosity, - config = test_config) - - plug_runner = test_config.plugins.prepareTestRunner( test_runner ) - if plug_runner is not None: - test_runner = plug_runner - - result = test_runner.run( tests ) - - success = result.wasSuccessful() - + # What requires these? Handy for (eg) functional tests to save outputs? + if galaxy_test_save: + os.environ[ 'GALAXY_TEST_SAVE' ] = galaxy_test_save + # Pass in through script setenv, will leave a copy of ALL test validate files + os.environ[ 'GALAXY_TEST_HOST' ] = galaxy_test_host + if testing_migrated_tools or testing_installed_tools: + shed_tools_dict = {} + if testing_migrated_tools: + shed_tools_dict = parse_tool_panel_config( migrated_tool_panel_config, shed_tools_dict ) + elif testing_installed_tools: + for shed_tool_config in installed_tool_panel_configs: + shed_tools_dict = parse_tool_panel_config( shed_tool_config, shed_tools_dict ) + # Persist the shed_tools_dict to the galaxy_tool_shed_test_file. + shed_tools_file = open( galaxy_tool_shed_test_file, 'w' ) + shed_tools_file.write( to_json_string( shed_tools_dict ) ) + shed_tools_file.close() + if not os.path.isabs( galaxy_tool_shed_test_file ): + galaxy_tool_shed_test_file = os.path.join( os.getcwd(), galaxy_tool_shed_test_file ) + os.environ[ 'GALAXY_TOOL_SHED_TEST_FILE' ] = galaxy_tool_shed_test_file + if testing_installed_tools: + # Eliminate the migrated_tool_panel_config from the app's tool_configs, append the list of installed_tool_panel_configs, + # and reload the app's toolbox. + relative_migrated_tool_panel_config = os.path.join( app.config.root, migrated_tool_panel_config ) + tool_configs = app.config.tool_configs + if relative_migrated_tool_panel_config in tool_configs: + tool_configs.remove( relative_migrated_tool_panel_config ) + for installed_tool_panel_config in installed_tool_panel_configs: + tool_configs.append( installed_tool_panel_config ) + app.toolbox = tools.ToolBox( tool_configs, app.config.tool_path, app ) + functional.test_toolbox.toolbox = app.toolbox + functional.test_toolbox.build_tests( testing_shed_tools=True ) + test_config = nose.config.Config( env=os.environ, ignoreFiles=ignore_files, plugins=nose.plugins.manager.DefaultPluginManager() ) + test_config.configure( sys.argv ) + result = run_tests( test_config ) + success = result.wasSuccessful() + try: + os.unlink( tmp_tool_panel_conf ) + except: + log.info( "Unable to remove temporary file: %s" % tmp_tool_panel_conf ) + try: + os.unlink( galaxy_tool_shed_test_file ) + except: + log.info( "Unable to remove file: %s" % galaxy_tool_shed_test_file ) + else: + functional.test_toolbox.toolbox = app.toolbox + functional.test_toolbox.build_tests() + if galaxy_test_file_dir: + os.environ[ 'GALAXY_TEST_FILE_DIR' ] = galaxy_test_file_dir + test_config = nose.config.Config( env=os.environ, ignoreFiles=ignore_files, plugins=nose.plugins.manager.DefaultPluginManager() ) + test_config.configure( sys.argv ) + result = run_tests( test_config ) + success = result.wasSuccessful() except: log.exception( "Failure running tests" ) log.info( "Shutting down" ) - - # ---- Teardown ----------------------------------------------------------- - + # ---- Tear down ----------------------------------------------------------- if server: log.info( "Shutting down embedded web server" ) server.server_close() @@ -330,7 +416,6 @@ def main(): shutil.rmtree( dir ) except: pass - if success: return 0 else: diff --git a/scripts/galaxy_messaging/server/data_transfer.py b/scripts/galaxy_messaging/server/data_transfer.py index fab9cba7f43..619100a676d 100755 --- a/scripts/galaxy_messaging/server/data_transfer.py +++ b/scripts/galaxy_messaging/server/data_transfer.py @@ -177,7 +177,7 @@ class DataTransfer( object ): self.update_status( SampleDataset.transfer_status.ADD_TO_LIBRARY ) try: data = {} - data[ 'folder_id' ] = api.encode_id( self.config_id_secret, '%s.%s' % ( 'folder', self.folder_id ) ) + data[ 'folder_id' ] = 'F%s' % api.encode_id( self.config_id_secret, self.folder_id ) data[ 'file_type' ] = 'auto' data[ 'server_dir' ] = self.server_dir data[ 'dbkey' ] = '' diff --git a/static/june_2007_style/base.less b/static/june_2007_style/base.less index a55e96193ae..3b5dcf58d8c 100644 --- a/static/june_2007_style/base.less +++ b/static/june_2007_style/base.less @@ -314,6 +314,11 @@ div.unified-panel-body { -webkit-border-radius: 1em; } +#dialog-box.dialog-box{ + min-width: 660px; + margin: -250px 0 0 -330px; +} + .dialog-box { border: solid #999 1px; background: white; @@ -407,14 +412,26 @@ div.unified-panel-body { } .title { - .navbar.brand(); + + position: absolute; + left: 0; + top: 0; + font-family: verdana; font-weight: bold; + font-size: 20px; + line-height: 1; color: white; // Override margin and padding due to shorter navbar height padding: 5px 20px 12px; margin-left: -15px; z-index: 2000; + + img { + display: inline; + width: 26px; + vertical-align: top; + } a { color: white; text-decoration: none; @@ -1210,6 +1227,10 @@ a.action-button { .menubutton { .btn(); + &:hover { .btn.hover(); } + &:active { .btn.active(); } + &:focus { .tab-focus(); } + display: inline-block; cursor: pointer; position: relative; @@ -1223,11 +1244,21 @@ a.action-button { // padding: 1px 0.25em; // margin: -1px -0.25em; + a { + text-decoration: none; + } + .label { position: relative; - // display: block; + display: inline-block; border-right: none; text-decoration: none; + text-align: left; + // The following properties truncate the text and force the button to have one line + max-height: 2*@baseLineHeight; + line-height: @baseLineHeight; + overflow: hidden; + text-overflow: ellipsis; } &.popup .label { @@ -1235,21 +1266,16 @@ a.action-button { padding-right: 6px; } - &.popup { + &.popup, &.popup.split { + padding-right: 18px; &:after { margin-top: 6px; + position: absolute; + top: 2px; + right: 6px; .caret(); } } - - &.popup.split { - &:after { - margin-top: 6px; - margin-left: 0px; - .caret(); - } - } - } // A split menu button, the main button has an action, the arrow causes the @@ -1401,8 +1427,6 @@ div.permissionContainer { padding: 0; } -@import "base_sprites"; - .tipsy { padding: 5px; font-size: 10px; @@ -1521,19 +1545,19 @@ div.historyItem-failed_metadata { div.historyItem-error { // border-color: @history_error_border; background: @history_error_bg; - .state-icon { - -sprite-group: history-states; - -sprite-image: data_error.png; - } + // .state-icon { + // -sprite-group: history-states; + // -sprite-image: data_error.png; + // } } div.historyItem-empty { // border-color: @history_error_border; background: @history_error_bg; - .state-icon { - -sprite-group: history-states; - -sprite-image: data_empty.png; - } + // .state-icon { + // -sprite-group: history-states; + // -sprite-image: data_empty.png; + // } } div.historyItem-running { @@ -1561,11 +1585,11 @@ div.historyItem-upload { } div.historyItem-queued { - // background: @history_queued_bg; - .state-icon { - -sprite-group: history-states; - -sprite-image: data_queued.png; - } + background: @history_queued_bg; + // .state-icon { + // -sprite-group: history-states; + // -sprite-image: data_queued.png; + // } } div.historyItem-noPermission { @@ -1606,21 +1630,15 @@ pre.peek { // ==== Tool menu styles -body.toolMenuPage +.toolMenuContainer { - background: white; color: @base_text; background: @menu_bg_over; - margin: 5px; - margin-right: 10px; - margin-left: 10px; + margin-top: 5px; } -hr -{ - border: none; - height: 0px; - margin-top: 0px; +.unified-panel-body .toolMenu { + padding: 10px; } div.toolSectionPad @@ -1632,7 +1650,7 @@ div.toolSectionPad } div.toolSectionWrapper { - // margin-bottom: 5px; + margin-bottom: 5px; } div.toolSectionDetailsInner @@ -1648,8 +1666,8 @@ div.toolSectionTitle div.toolPanelLabel { - //padding-top: 10px; - //padding-bottom: 5px; + padding-top: 10px; + padding-bottom: 5px; font-weight: bold; color: gray; text-transform: uppercase; @@ -1677,7 +1695,44 @@ div.toolSectionBody div.toolPanelLabel div.toolTitleNoSection { - padding-bottom: 0px; + padding-bottom: 5px; font-weight: bold; } +#tool-search { + padding-top: 5px; + padding-bottom: 10px; + position: relative; +} + +// Dataset Display Styles + +#loading_indicator{ + position:fixed; + right:10px; + top:10px; + height:32px; + width:32px; + background:url(largespinner.gif); +} + +#content_table td{ + text-align:right; + white-space:nowrap; + padding:2px 10px; +} + +#content_table td.stringalign{ + text-align:left; +} + +// ==== Integrated tool form styles + +.toolMenuAndView .toolForm +{ + float: left; + background-color: white; + margin: 10px; +} + +@import "base_sprites"; diff --git a/static/june_2007_style/base_sprites.less.tmpl b/static/june_2007_style/base_sprites.less.tmpl index d1e16d0cce5..8c0dee0c7c4 100644 --- a/static/june_2007_style/base_sprites.less.tmpl +++ b/static/june_2007_style/base_sprites.less.tmpl @@ -87,6 +87,10 @@ -sprite-image: fugue/plus-circle.png; } .icon-button.plus-button { + -sprite-group: fugue; + -sprite-image: fugue/plus-button-bw.png; +} +.icon-button.plus-button:hover { -sprite-group: fugue; -sprite-image: fugue/plus-button.png; } @@ -94,9 +98,29 @@ -sprite-group: fugue; -sprite-image: fugue/gear.png; } +.icon-button.chart_curve { + -sprite-group: fugue; + -sprite-image: silk/chart_curve.png; +} .text-and-autocomplete-select { -sprite-group: fugue; -sprite-image: fugue/control-270.png; -sprite-horiz-position: right; +} + + +div.historyItem-error .state-icon { + -sprite-group: history-states; + -sprite-image: data_error.png; +} + +div.historyItem-empty .state-icon { + -sprite-group: history-states; + -sprite-image: data_empty.png; +} + +div.historyItem-queued .state-icon { + -sprite-group: history-states; + -sprite-image: data_queued.png; } \ No newline at end of file diff --git a/static/june_2007_style/blue/base.css b/static/june_2007_style/blue/base.css index 6380a94d441..94c40ecfe03 100644 --- a/static/june_2007_style/blue/base.css +++ b/static/june_2007_style/blue/base.css @@ -9,12 +9,12 @@ sup{top:-0.5em;} sub{bottom:-0.25em;} img{max-width:100%;height:auto;border:0;-ms-interpolation-mode:bicubic;} button,input,select,textarea{margin:0;font-size:100%;vertical-align:middle;} +input[type="checkbox"],input[type="radio"]{margin-left:0.5ex;margin-right:0.5ex;} button,input{*overflow:visible;line-height:normal;} button::-moz-focus-inner,input::-moz-focus-inner{padding:0;border:0;} button,input[type="button"],input[type="reset"],input[type="submit"]{cursor:pointer;-webkit-appearance:button;} input[type="search"]{-webkit-appearance:textfield;-webkit-box-sizing:content-box;-moz-box-sizing:content-box;box-sizing:content-box;} input[type="search"]::-webkit-search-decoration,input[type="search"]::-webkit-search-cancel-button{-webkit-appearance:none;} -input[type="checkbox"],input[type="radio"]{margin-left:0.5ex;margin-right:0.5ex;} textarea{overflow:auto;vertical-align:top;} body{margin:0;font-family:"Lucida Grande",verdana,arial,helvetica,sans-serif;font-size:12px;line-height:16px;color:#111111;background-color:#ffffff;} a{color:#303030;text-decoration:none;} @@ -225,7 +225,7 @@ button.btn.small,input[type="submit"].btn.small{*padding-top:3px;*padding-bottom .navbar-form .radio,.navbar-form .checkbox{margin-top:5px;} .navbar-form input[type="image"],.navbar-form input[type="checkbox"],.navbar-form input[type="radio"]{margin-top:3px;} .navbar-search{position:relative;float:left;margin-top:6px;margin-bottom:0;}.navbar-search .search-query{padding:4px 9px;font-family:"Lucida Grande",verdana,arial,helvetica,sans-serif;font-size:13px;font-weight:normal;line-height:1;color:#ffffff;color:rgba(255, 255, 255, 0.75);background:#666;background:rgba(255, 255, 255, 0.3);border:1px solid #111;-webkit-box-shadow:inset 0 1px 2px rgba(0, 0, 0, 0.1),0 1px 0px rgba(255, 255, 255, 0.15);-moz-box-shadow:inset 0 1px 2px rgba(0, 0, 0, 0.1),0 1px 0px rgba(255, 255, 255, 0.15);box-shadow:inset 0 1px 2px rgba(0, 0, 0, 0.1),0 1px 0px rgba(255, 255, 255, 0.15);-webkit-transition:none;-moz-transition:none;-ms-transition:none;-o-transition:none;transition:none;}.navbar-search .search-query :-moz-placeholder{color:#eeeeee;} -.navbar-search .search-query::-webkit-input-placeholder{color:#eeeeee;} +.navbar-search .search-query ::-webkit-input-placeholder{color:#eeeeee;} .navbar-search .search-query:hover{color:#ffffff;background-color:#999999;background-color:rgba(255, 255, 255, 0.5);} .navbar-search .search-query:focus,.navbar-search .search-query.focused{padding:5px 10px;color:#333333;text-shadow:0 1px 0 #ffffff;background-color:#ffffff;border:0;-webkit-box-shadow:0 0 3px rgba(0, 0, 0, 0.15);-moz-box-shadow:0 0 3px rgba(0, 0, 0, 0.15);box-shadow:0 0 3px rgba(0, 0, 0, 0.15);outline:0;} .navbar-fixed-top{position:fixed;top:0;right:0;left:0;z-index:1030;} @@ -520,6 +520,7 @@ div.unified-panel-body{position:absolute;top:30px;bottom:0;width:100%;margin-top #overlay.is_modal #overlay-background{background:rgba(0, 0, 0, 0.5);} .dialog-box-container{position:relative;margin-top:80px;margin-right:auto;margin-left:auto;} .dialog-box-wrapper{position:relative;padding:1em;background-color:rgba(0, 0, 0, 0.5);-moz-border-radius:1em;-webkit-border-radius:1em;} +#dialog-box.dialog-box{min-width:660px;margin:-250px 0 0 -330px;} .dialog-box{border:solid #999 1px;background:white;z-index:80000;} #overlay.is_modal .dialog-box .body{min-width:600px;} .dialog-box .body{padding:5px;overflow:auto;max-height:500px;min-width:300px;} @@ -535,7 +536,8 @@ div.unified-panel-body{position:absolute;top:30px;bottom:0;width:100%;margin-top #masthead .btn-group .btn{margin-top:0;} #masthead .nav>li>a{padding:6px 10px 8px;cursor:pointer;}#masthead .nav>li>a:hover{color:gold;} #masthead .dropdown-menu a,#masthead .dropdown-menu a:hover{text-decoration:none;} -#masthead .title{float:left;display:block;padding:8px 20px 12px;margin-left:-20px;font-size:20px;font-weight:200;line-height:1;color:#ffffff;font-family:verdana;font-weight:bold;color:white;padding:5px 20px 12px;margin-left:-15px;z-index:2000;}#masthead .title a{color:white;text-decoration:none;} +#masthead .title{position:absolute;left:0;top:0;font-family:verdana;font-weight:bold;font-size:20px;line-height:1;color:white;padding:5px 20px 12px;margin-left:-15px;z-index:2000;}#masthead .title img{display:inline;width:26px;vertical-align:top;} +#masthead .title a{color:white;text-decoration:none;} #masthead .masthead-inner{padding-left:20px;padding-right:20px;background-color:#303239;background-image:-moz-linear-gradient(top, #333333, #2c3143);background-image:-ms-linear-gradient(top, #333333, #2c3143);background-image:-webkit-gradient(linear, 0 0, 0 100%, from(#333333), to(#2c3143));background-image:-webkit-linear-gradient(top, #333333, #2c3143);background-image:-o-linear-gradient(top, #333333, #2c3143);background-image:linear-gradient(top, #333333, #2c3143);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#333333', endColorstr='#2c3143', GradientType=0);-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;-webkit-box-shadow:0 1px 3px rgba(0, 0, 0, 0.25),inset 0 -1px 0 rgba(0, 0, 0, 0.1);-moz-box-shadow:0 1px 3px rgba(0, 0, 0, 0.25),inset 0 -1px 0 rgba(0, 0, 0, 0.1);box-shadow:0 1px 3px rgba(0, 0, 0, 0.25),inset 0 -1px 0 rgba(0, 0, 0, 0.1);padding-left:0;padding-right:0;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0;height:32px;} #masthead a:hover{text-decoration:underline;} .quota-meter-container{position:absolute;top:0;right:0;height:32px;} @@ -660,10 +662,13 @@ a.action-button{text-decoration:none;} .action-button:active{background-image:none;-webkit-box-shadow:inset 0 2px 4px rgba(0, 0, 0, 0.15),0 1px 2px rgba(0, 0, 0, 0.05);-moz-box-shadow:inset 0 2px 4px rgba(0, 0, 0, 0.15),0 1px 2px rgba(0, 0, 0, 0.05);box-shadow:inset 0 2px 4px rgba(0, 0, 0, 0.15),0 1px 2px rgba(0, 0, 0, 0.05);background-color:#e6e6e6;background-color:#d9d9d9 \9;outline:0;} .menubutton{display:inline-block;padding:2px 10px 2px;font-size:12px;line-height:16px;color:#111111;text-align:center;text-shadow:0 1px 1px rgba(255, 255, 255, 0.75);background-color:#fafafa;background-image:-webkit-gradient(linear, 0 0, 0 100%, from(#ffffff), color-stop(25%, #ffffff), to(#e6e6e6));background-image:-webkit-linear-gradient(#ffffff, #ffffff 25%, #e6e6e6);background-image:-moz-linear-gradient(top, #ffffff, #ffffff 25%, #e6e6e6);background-image:-ms-linear-gradient(#ffffff, #ffffff 25%, #e6e6e6);background-image:-o-linear-gradient(#ffffff, #ffffff 25%, #e6e6e6);background-image:linear-gradient(#ffffff, #ffffff 25%, #e6e6e6);background-repeat:no-repeat;filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffffff', endColorstr='#e6e6e6', GradientType=0);border:1px solid #999999;border-bottom-color:#888888;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;-webkit-box-shadow:inset 0 1px 0 rgba(255, 255, 255, 0.2),0 1px 2px rgba(0, 0, 0, 0.05);-moz-box-shadow:inset 0 1px 0 rgba(255, 255, 255, 0.2),0 1px 2px rgba(0, 0, 0, 0.05);box-shadow:inset 0 1px 0 rgba(255, 255, 255, 0.2),0 1px 2px rgba(0, 0, 0, 0.05);cursor:pointer;*margin-left:.3em;display:inline-block;cursor:pointer;position:relative;-webkit-user-select:none;-moz-user-select:none;-o-user-select:none;user-select:none;}.menubutton:first-child{*margin-left:0;} .menubutton [class^="ficon"],.menubutton [class*=" ficon"]{line-height:.9em;} -.menubutton .label{position:relative;border-right:none;text-decoration:none;} +.menubutton:hover{color:#111111;text-decoration:none;background-color:#e6e6e6;background-position:0 -15px;-webkit-transition:background-position 0.1s linear;-moz-transition:background-position 0.1s linear;-ms-transition:background-position 0.1s linear;-o-transition:background-position 0.1s linear;transition:background-position 0.1s linear;} +.menubutton:active{background-image:none;-webkit-box-shadow:inset 0 2px 4px rgba(0, 0, 0, 0.15),0 1px 2px rgba(0, 0, 0, 0.05);-moz-box-shadow:inset 0 2px 4px rgba(0, 0, 0, 0.15),0 1px 2px rgba(0, 0, 0, 0.05);box-shadow:inset 0 2px 4px rgba(0, 0, 0, 0.15),0 1px 2px rgba(0, 0, 0, 0.05);background-color:#e6e6e6;background-color:#d9d9d9 \9;outline:0;} +.menubutton:focus{outline:thin dotted;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px;} +.menubutton a{text-decoration:none;} +.menubutton .label{position:relative;display:inline-block;border-right:none;text-decoration:none;text-align:left;max-height:32px;line-height:16px;overflow:hidden;text-overflow:ellipsis;} .menubutton.popup .label{border-right:solid #999999 1px;padding-right:6px;} -.menubutton.popup:after{margin-top:6px;display:inline-block;width:0;height:0;text-indent:-99999px;*text-indent:0;vertical-align:top;border-left:4px solid transparent;border-right:4px solid transparent;border-top:4px solid #000000;opacity:0.3;filter:alpha(opacity=30);content:"\2193";} -.menubutton.popup.split:after{margin-top:6px;margin-left:0px;display:inline-block;width:0;height:0;text-indent:-99999px;*text-indent:0;vertical-align:top;border-left:4px solid transparent;border-right:4px solid transparent;border-top:4px solid #000000;opacity:0.3;filter:alpha(opacity=30);content:"\2193";} +.menubutton.popup,.menubutton.popup.split{padding-right:18px;}.menubutton.popup:after,.menubutton.popup.split:after{margin-top:6px;position:absolute;top:2px;right:6px;display:inline-block;width:0;height:0;text-indent:-99999px;*text-indent:0;vertical-align:top;border-left:4px solid transparent;border-right:4px solid transparent;border-top:4px solid #000000;opacity:0.3;filter:alpha(opacity=30);content:"\2193";} div.popmenu-wrapper{position:absolute;top:100%;z-index:20000;}div.popmenu-wrapper ul.dropdown-menu{display:block;position:relative;float:none;} ul.dropdown-menu a{text-decoration:none;} ul.dropdown-menu li.head>a{text-shadow:0 1px 0 rgba(255, 255, 255, 0.5);font-size:11px;font-weight:bold;line-height:16px;color:#999999;text-transform:uppercase;} @@ -678,6 +683,51 @@ div.permissionContainer{padding-left:20px;} .text-content th,.text-content td{border-bottom:1px solid #ddd;border-right:1px solid #ccc;} .text-content th,.text-content td{padding:.8em;} .icon-button{width:16px;height:16px;display:block;float:left;margin-left:2px;text-indent:20px;background-repeat:no-repeat;background-position:0px 0px;padding:0;} +.tipsy{padding:5px;font-size:10px;filter:alpha(opacity=80);background-repeat:no-repeat;background-image:url(../images/tipsy.gif);} +.tipsy-inner{padding:5px 8px 4px 8px;background-color:black;color:white;max-width:200px;text-align:center;} +.tipsy-north{background-position:top center;} +.tipsy-south{background-position:bottom center;} +.tipsy-east{background-position:right center;} +.tipsy-west{background-position:left center;} +.editable-text{cursor:pointer;} +.editable-text:hover{cursor:text;border:dotted #999999 1px;} +.icon-button.multiinput{background:url(../images/documents-stack.png) no-repeat;cursor:pointer;float:none;display:inline-block;margin-left:10px;} +.icon-button.multiinput.disabled{background:url(../images/documents-stack-faded.png) no-repeat;cursor:auto;} +.workflow-invocation-complete{border:solid 1px #6A6;border-left-width:5px;margin:10px 0;padding-left:5px;} +body.historyPage{background:#dfe5f9;color:#303030;margin:5px;border:0;padding:0;} +div.historyLinks{margin:5px 5px;} +div.historyItem{margin:0px -5px;padding:8px 10px;border-top:solid #999999 1px;border-right:none;word-wrap:break-word;background:#eeeeee;}div.historyItem .state-icon{display:inline-block;vertical-align:middle;width:16px;height:16px;background-position:0 1px;background-repeat:no-repeat;} +div.historyItem .historyItemTitle{font-weight:bold;line-height:16px;} +.historyItemContainer:last-child div.historyItem{border-bottom:solid #999999 1px;} +div.historyItem div.historyItem{margin-right:-11px;} +div.historyItem-ok,div.historyItem-failed_metadata{background:#ccffcc;}div.historyItem-ok .state-icon,div.historyItem-failed_metadata .state-icon{display:none;} +div.historyItem-error{background:#ffcccc;} +div.historyItem-empty{background:#ffcccc;} +div.historyItem-running{background:#ffffcc;}div.historyItem-running .state-icon{background-image:url(data_running.gif);} +div.historyItem-setting_metadata{background:#ffffcc;}div.historyItem-setting_metadata .state-icon{background-image:url(data_running.gif);} +div.historyItem-upload{background:#ccccff;}div.historyItem-upload .state-icon{background-image:url(data_upload.gif);} +div.historyItem-queued{background:#eeeeee;} +div.historyItem-noPermission{filter:alpha(opacity=60);-moz-opacity:.60;opacity:.60;} +div.historyItemTitleBar.spinner .state-icon{background:url(data_running.gif) 0 1px no-repeat !important;} +div.historyItemButtons{float:right;} +div.historyItemBody div{padding-top:2px;} +pre.peek{background:white;color:black;width:100%;font-size:10px;overflow:auto;}pre.peek th{color:white;background:#023858;} +pre.peek table,pre.peek th,pre.peek tr,pre.peek td{font-family:Menlo,Monaco,"Courier New",monospace;font-size:10px;} +.toolMenuContainer{color:#303030;background:#dfe5f9;margin-top:5px;} +.unified-panel-body .toolMenu{padding:10px;} +div.toolSectionPad{margin:0;padding:0;height:5px;font-size:0px;} +div.toolSectionWrapper{margin-bottom:5px;} +div.toolSectionDetailsInner{margin-left:5px;margin-right:5px;} +div.toolSectionTitle{font-weight:bold;} +div.toolPanelLabel{padding-top:10px;padding-bottom:5px;font-weight:bold;color:gray;text-transform:uppercase;} +div.toolTitle{padding-top:5px;padding-bottom:5px;margin-left:16px;margin-right:10px;display:list-item;list-style:square outside;} +div.toolSectionBody div.toolPanelLabel{padding-top:5px;padding-bottom:5px;margin-left:16px;margin-right:10px;display:list-item;list-style:none outside;} +div.toolTitleNoSection{padding-bottom:5px;font-weight:bold;} +#tool-search{padding-top:5px;padding-bottom:10px;position:relative;} +#loading_indicator{position:fixed;right:10px;top:10px;height:32px;width:32px;background:url(largespinner.gif);} +#content_table td{text-align:right;white-space:nowrap;padding:2px 10px;} +#content_table td.stringalign{text-align:left;} +.toolMenuAndView .toolForm{float:left;background-color:white;margin:10px;} .icon-button.display{background:url(history-buttons.png) no-repeat 0px 0px;} .icon-button.display:hover{background:url(history-buttons.png) no-repeat 0px -26px;} .icon-button.display_disabled{background:url(history-buttons.png) no-repeat 0px -52px;} @@ -701,44 +751,10 @@ div.permissionContainer{padding-left:20px;} .icon-button.go-to-full-screen{background:url(fugue.png) no-repeat 0px -286px;} .icon-button.import{background:url(fugue.png) no-repeat 0px -312px;} .icon-button.plus-button{background:url(fugue.png) no-repeat 0px -338px;} -.icon-button.gear{background:url(fugue.png) no-repeat 0px -364px;} -.text-and-autocomplete-select{background:url(fugue.png) no-repeat right -390px;} -.tipsy{padding:5px;font-size:10px;filter:alpha(opacity=80);background-repeat:no-repeat;background-image:url(../images/tipsy.gif);} -.tipsy-inner{padding:5px 8px 4px 8px;background-color:black;color:white;max-width:200px;text-align:center;} -.tipsy-north{background-position:top center;} -.tipsy-south{background-position:bottom center;} -.tipsy-east{background-position:right center;} -.tipsy-west{background-position:left center;} -.editable-text{cursor:pointer;} -.editable-text:hover{cursor:text;border:dotted #999999 1px;} -.icon-button.multiinput{background:url(../images/documents-stack.png) no-repeat;cursor:pointer;float:none;display:inline-block;margin-left:10px;} -.icon-button.multiinput.disabled{background:url(../images/documents-stack-faded.png) no-repeat;cursor:auto;} -.workflow-invocation-complete{border:solid 1px #6A6;border-left-width:5px;margin:10px 0;padding-left:5px;} -body.historyPage{background:#dfe5f9;color:#303030;margin:5px;border:0;padding:0;} -div.historyLinks{margin:5px 5px;} -div.historyItem{margin:0px -5px;padding:8px 10px;border-top:solid #999999 1px;border-right:none;word-wrap:break-word;background:#eeeeee;}div.historyItem .state-icon{display:inline-block;vertical-align:middle;width:16px;height:16px;background-position:0 1px;background-repeat:no-repeat;} -div.historyItem .historyItemTitle{font-weight:bold;line-height:16px;} -.historyItemContainer:last-child div.historyItem{border-bottom:solid #999999 1px;} -div.historyItem div.historyItem{margin-right:-11px;} -div.historyItem-ok,div.historyItem-failed_metadata{background:#ccffcc;}div.historyItem-ok .state-icon,div.historyItem-failed_metadata .state-icon{display:none;} -div.historyItem-error{background:#ffcccc;}div.historyItem-error .state-icon{-sprite-group:history-states;-sprite-image:data_error.png;} -div.historyItem-empty{background:#ffcccc;}div.historyItem-empty .state-icon{-sprite-group:history-states;-sprite-image:data_empty.png;} -div.historyItem-running{background:#ffffcc;}div.historyItem-running .state-icon{background-image:url(data_running.gif);} -div.historyItem-setting_metadata{background:#ffffcc;}div.historyItem-setting_metadata .state-icon{background-image:url(data_running.gif);} -div.historyItem-upload{background:#ccccff;}div.historyItem-upload .state-icon{background-image:url(data_upload.gif);} -div.historyItem-queued .state-icon{-sprite-group:history-states;-sprite-image:data_queued.png;} -div.historyItem-noPermission{filter:alpha(opacity=60);-moz-opacity:.60;opacity:.60;} -div.historyItemTitleBar.spinner .state-icon{background:url(data_running.gif) 0 1px no-repeat !important;} -div.historyItemButtons{float:right;} -div.historyItemBody div{padding-top:2px;} -pre.peek{background:white;color:black;width:100%;font-size:10px;overflow:auto;}pre.peek th{color:white;background:#023858;} -pre.peek table,pre.peek th,pre.peek tr,pre.peek td{font-family:Menlo,Monaco,"Courier New",monospace;font-size:10px;} -body.toolMenuPage{background:white;color:#303030;background:#dfe5f9;margin:5px;margin-right:10px;margin-left:10px;} -hr{border:none;height:0px;margin-top:0px;} -div.toolSectionPad{margin:0;padding:0;height:5px;font-size:0px;} -div.toolSectionDetailsInner{margin-left:5px;margin-right:5px;} -div.toolSectionTitle{font-weight:bold;} -div.toolPanelLabel{font-weight:bold;color:gray;text-transform:uppercase;} -div.toolTitle{padding-top:5px;padding-bottom:5px;margin-left:16px;margin-right:10px;display:list-item;list-style:square outside;} -div.toolSectionBody div.toolPanelLabel{padding-top:5px;padding-bottom:5px;margin-left:16px;margin-right:10px;display:list-item;list-style:none outside;} -div.toolTitleNoSection{padding-bottom:0px;font-weight:bold;} +.icon-button.plus-button:hover{background:url(fugue.png) no-repeat 0px -364px;} +.icon-button.gear{background:url(fugue.png) no-repeat 0px -390px;} +.icon-button.chart_curve{background:url(fugue.png) no-repeat 0px -416px;} +.text-and-autocomplete-select{background:url(fugue.png) no-repeat right -442px;} +div.historyItem-error .state-icon{background:url(history-states.png) no-repeat 0px 0px;} +div.historyItem-empty .state-icon{background:url(history-states.png) no-repeat 0px -25px;} +div.historyItem-queued .state-icon{background:url(history-states.png) no-repeat 0px -50px;} diff --git a/static/june_2007_style/blue/fugue.png b/static/june_2007_style/blue/fugue.png index ba6dc59583c..6e69eaf0a1f 100644 Binary files a/static/june_2007_style/blue/fugue.png and b/static/june_2007_style/blue/fugue.png differ diff --git a/static/june_2007_style/blue/largespinner.gif b/static/june_2007_style/blue/largespinner.gif new file mode 100644 index 00000000000..3288d1035d7 Binary files /dev/null and b/static/june_2007_style/blue/largespinner.gif differ diff --git a/static/june_2007_style/blue/library.css b/static/june_2007_style/blue/library.css index 58afe2ccef5..3368c9ae6b3 100644 --- a/static/june_2007_style/blue/library.css +++ b/static/june_2007_style/blue/library.css @@ -1,9 +1,9 @@ .libraryRow{background-color:#ebd9b2;} -.datasetHighlighted{background-color:#C1C9E5;} +.datasetHighlighted{background-color:#DFE5F9;} .libraryItemDeleted-True{font-style:italic;} div.libraryItemBody{padding:4px 4px 2px 4px;} li.folderRow,li.datasetRow{border-top:solid 1px #ddd;} -li.folderRow:hover,li.datasetRow:hover{background-color:#C1C9E5;} +li.folderRow:hover,li.datasetRow:hover{background-color:#DFE5F9;} img.expanderIcon{padding-right:4px;} input.datasetCheckbox,li,ul{padding:0;margin:0;} .rowTitle{padding:2px;} diff --git a/static/june_2007_style/blue/trackster.css b/static/june_2007_style/blue/trackster.css index a260278d30a..e0b07d38775 100644 --- a/static/june_2007_style/blue/trackster.css +++ b/static/june_2007_style/blue/trackster.css @@ -79,17 +79,21 @@ input{font:10px verdana;} .filters-icon{background:transparent url(../images/fugue/ui-slider-050-bw.png) no-repeat;} .filters-icon:hover{background:transparent url(../images/fugue/ui-slider-050.png) no-repeat;} .remove-icon,.overview-close{background:transparent url(../images/fugue/cross-small-bw.png) no-repeat;} -.remove-icon:hover,.overview-close:hover{background:transparent url(../images/fugue/cross-circle.png) no-repeat;} -#close-icon{background:transparent url(../images/fugue/cross-circle-bw.png) no-repeat;margin-right:0px;} -#close-icon:hover{background:transparent url(../images/fugue/cross-circle.png) no-repeat;} -#add-tracks-icon{background:transparent url(../images/fugue/plus-button-bw.png) no-repeat;} -#add-tracks-icon:hover{background:transparent url(../images/fugue/plus-button.png) no-repeat;} -#add-group-icon{background:transparent url(../images/fugue/block--plus-bw.png) no-repeat;} -#add-group-icon:hover{background:transparent url(../images/fugue/block--plus.png) no-repeat;} -#bookmarks-icon{background:transparent url(../images/fugue/bookmarks-bw.png) no-repeat;} -#bookmarks-icon:hover{background:transparent url(../images/fugue/bookmarks.png) no-repeat;} -#save-icon{background:transparent url(../images/fugue/disk--arrow-bw.png) no-repeat;} -#save-icon:hover{background:transparent url(../images/fugue/disk--arrow.png) no-repeat;} +.icon-button.remove-icon:hover,.overview-close:hover{background:transparent url(../images/fugue/cross-circle.png) no-repeat;} +.icon-button.toggle{background:transparent url(../images/fugue/toggle-bw.png) no-repeat;margin-right:0px;} +.icon-button.toggle:hover{background:transparent url(../images/fugue/toggle.png) no-repeat;} +.icon-button.toggle-expand{background:transparent url(../images/fugue/toggle-expand-bw.png) no-repeat;margin-right:0px;} +.icon-button.toggle-expand:hover{background:transparent url(../images/fugue/toggle-expand.png) no-repeat;} +.icon-button.cross-circle{background:transparent url(../images/fugue/cross-circle-bw.png) no-repeat;margin-right:0px;} +.icon-button.cross-circle:hover{background:transparent url(../images/fugue/cross-circle.png) no-repeat;} +.icon-button.block--plus{background:transparent url(../images/fugue/block--plus-bw.png) no-repeat;} +.icon-button.block--plus:hover{background:transparent url(../images/fugue/block--plus.png) no-repeat;} +.icon-button.bookmarks{background:transparent url(../images/fugue/bookmarks-bw.png) no-repeat;} +.icon-button.bookmarks:hover{background:transparent url(../images/fugue/bookmarks.png) no-repeat;} +.icon-button.toolbox{background:transparent url(../images/fugue/toolbox-bw.png) no-repeat;} +.icon-button.toolbox:hover{background:transparent url(../images/fugue/toolbox.png) no-repeat;} +.icon-button.disk--arrow{background:transparent url(../images/fugue/disk--arrow-bw.png) no-repeat;} +.icon-button.disk--arrow:hover{background:transparent url(../images/fugue/disk--arrow.png) no-repeat;} .child-track-icon{background:url('../images/fugue/arrow-000-small-bw.png') no-repeat;width:30px;cursor:move;} .track-resize{background:white url('../images/visualization/draggable_vertical.png') no-repeat top center;position:absolute;right:3px;bottom:-4px;width:14px;height:7px;border:solid #999 1px;z-index:100;} .bookmark{background:white;border:solid #999 1px;border-right:none;margin:0.5em;margin-right:0;padding:0.5em;} diff --git a/static/june_2007_style/bootstrap/reset.less b/static/june_2007_style/bootstrap/reset.less index 28d8eb60ca4..b60881c82c0 100644 --- a/static/june_2007_style/bootstrap/reset.less +++ b/static/june_2007_style/bootstrap/reset.less @@ -93,6 +93,11 @@ textarea { font-size: 100%; vertical-align: middle; } +// Use Webkit margins for checkboxs and radio buttons. +input[type="checkbox"],input[type="radio"] { + margin-left: 0.5ex; + margin-right: 0.5ex; +} button, input { *overflow: visible; // Inner spacing ie IE6/7 diff --git a/static/june_2007_style/trackster.css.tmpl b/static/june_2007_style/trackster.css.tmpl index f42ae6f98b3..90331745628 100644 --- a/static/june_2007_style/trackster.css.tmpl +++ b/static/june_2007_style/trackster.css.tmpl @@ -390,37 +390,49 @@ input { .remove-icon, .overview-close { background: transparent url(../images/fugue/cross-small-bw.png) no-repeat; } -.remove-icon:hover, .overview-close:hover { +.icon-button.remove-icon:hover, .overview-close:hover { background: transparent url(../images/fugue/cross-circle.png) no-repeat; } -#close-icon { +.icon-button.toggle { + background: transparent url(../images/fugue/toggle-bw.png) no-repeat;margin-right:0px; +} +.icon-button.toggle:hover { + background: transparent url(../images/fugue/toggle.png) no-repeat; +} +.icon-button.toggle-expand { + background: transparent url(../images/fugue/toggle-expand-bw.png) no-repeat;margin-right:0px; +} +.icon-button.toggle-expand:hover { + background: transparent url(../images/fugue/toggle-expand.png) no-repeat; +} +.icon-button.cross-circle { background: transparent url(../images/fugue/cross-circle-bw.png) no-repeat;margin-right:0px; } -#close-icon:hover { +.icon-button.cross-circle:hover { background: transparent url(../images/fugue/cross-circle.png) no-repeat; } -#add-tracks-icon { - background: transparent url(../images/fugue/plus-button-bw.png) no-repeat; -} -#add-tracks-icon:hover { - background: transparent url(../images/fugue/plus-button.png) no-repeat; -} -#add-group-icon { +.icon-button.block--plus { background: transparent url(../images/fugue/block--plus-bw.png) no-repeat; } -#add-group-icon:hover { +.icon-button.block--plus:hover { background: transparent url(../images/fugue/block--plus.png) no-repeat; } -#bookmarks-icon { +.icon-button.bookmarks { background: transparent url(../images/fugue/bookmarks-bw.png) no-repeat; } -#bookmarks-icon:hover { +.icon-button.bookmarks:hover { background: transparent url(../images/fugue/bookmarks.png) no-repeat; } -#save-icon { +.icon-button.toolbox { + background: transparent url(../images/fugue/toolbox-bw.png) no-repeat; +} +.icon-button.toolbox:hover { + background: transparent url(../images/fugue/toolbox.png) no-repeat; +} +.icon-button.disk--arrow { background: transparent url(../images/fugue/disk--arrow-bw.png) no-repeat; } -#save-icon:hover { +.icon-button.disk--arrow:hover { background: transparent url(../images/fugue/disk--arrow.png) no-repeat; } .child-track-icon { diff --git a/static/scripts/galaxy.base.js b/static/scripts/galaxy.base.js index 1a8a253b76f..55b652bf509 100644 --- a/static/scripts/galaxy.base.js +++ b/static/scripts/galaxy.base.js @@ -623,7 +623,6 @@ function reset_tool_search( initValue ) { if (initValue) { var search_input = tool_menu_frame.find("#tool-search-query"); search_input.val("search tools"); - search_input.css("font-style", "italic"); } } diff --git a/static/scripts/galaxy.panels.js b/static/scripts/galaxy.panels.js index 69d3404e0c5..4e2f67c8924 100644 --- a/static/scripts/galaxy.panels.js +++ b/static/scripts/galaxy.panels.js @@ -217,7 +217,7 @@ function show_in_overlay( options ) { hide_modal(); $("#overlay-background").unbind( "click.overlay" ); }); - show_modal( null, $( "
    " ) ); + show_modal( null, $( "
    " ) ); $("#close_button").bind( "click", function() { hide_modal(); } ); } diff --git a/static/scripts/galaxy.workflow_editor.canvas.js b/static/scripts/galaxy.workflow_editor.canvas.js index ea60b600c03..8051ca011ed 100644 --- a/static/scripts/galaxy.workflow_editor.canvas.js +++ b/static/scripts/galaxy.workflow_editor.canvas.js @@ -188,7 +188,7 @@ $.extend( Node.prototype, { .appendTo( "body" ) .append( $("
    ").append( - $("").attr("src", image_path + '/delete_icon.png').click( function() { + $("").attr("src", galaxy_paths.attributes.image_path + '/delete_icon.png').click( function() { $.each( terminal.connectors, function( _, x ) { x.destroy(); }); @@ -331,17 +331,17 @@ $.extend( Node.prototype, { } var r = $("
    " + label + "
    " ); if (node.type == 'tool'){ - var callout = $("
    ") + var callout = $("
    ") .css( { display: 'none' } ) .append( $("
    ").append( - $("").attr('src', image_path + '/fugue/asterisk-small-outline.png').click( function() { + $("").attr('src', galaxy_paths.attributes.image_path + '/fugue/asterisk-small-outline.png').click( function() { if ($.inArray(output.name, node.workflow_outputs) != -1){ node.workflow_outputs.splice($.inArray(output.name, node.workflow_outputs), 1); - callout.find('img').attr('src', image_path + '/fugue/asterisk-small-outline.png'); + callout.find('img').attr('src', galaxy_paths.attributes.image_path + '/fugue/asterisk-small-outline.png'); }else{ node.workflow_outputs.push(output.name); - callout.find('img').attr('src', image_path + '/fugue/asterisk-small.png'); + callout.find('img').attr('src', galaxy_paths.attributes.image_path + '/fugue/asterisk-small.png'); } workflow.has_changes = true; canvas_manager.draw_overview(); @@ -355,20 +355,20 @@ $.extend( Node.prototype, { callout.show(); r.append(callout); if ($.inArray(output.name, node.workflow_outputs) === -1){ - callout.find('img').attr('src', image_path + '/fugue/asterisk-small-outline.png'); + callout.find('img').attr('src', galaxy_paths.attributes.image_path + '/fugue/asterisk-small-outline.png'); }else{ - callout.find('img').attr('src', image_path + '/fugue/asterisk-small.png'); + callout.find('img').attr('src', galaxy_paths.attributes.image_path + '/fugue/asterisk-small.png'); } r.hover( function(){ - callout.find('img').attr('src', image_path + '/fugue/asterisk-small-yellow.png'); + callout.find('img').attr('src', galaxy_paths.attributes.image_path + '/fugue/asterisk-small-yellow.png'); }, function(){ if ($.inArray(output.name, node.workflow_outputs) === -1){ - callout.find('img').attr('src', image_path + '/fugue/asterisk-small-outline.png'); + callout.find('img').attr('src', galaxy_paths.attributes.image_path + '/fugue/asterisk-small-outline.png'); }else{ - callout.find('img').attr('src', image_path + '/fugue/asterisk-small.png'); - } + callout.find('img').attr('src', galaxy_paths.attributes.image_path + '/fugue/asterisk-small.png'); + } }); } r.css({ position:'absolute', @@ -477,59 +477,65 @@ $.extend( Workflow.prototype, { }, rectify_workflow_outputs : function() { // Find out if we're using workflow_outputs or not. - var using_workflow_outputs = false; + var using_workflow_outputs, has_existing_pjas = false; $.each( this.nodes, function ( k, node ) { if (node.workflow_outputs && node.workflow_outputs.length > 0){ using_workflow_outputs = true; } + $.each(node.post_job_actions, function(pja_id, pja){ + if (pja.action_type === "HideDatasetAction"){ + has_existing_pjas = true; + } + }); }); - if (using_workflow_outputs == false){ - //We're done, leave PJAs alone. - return true; + if (using_workflow_outputs !== false || has_existing_pjas !== false){ + // Using workflow outputs, or has existing pjas. Remove all PJAs and recreate based on outputs. + $.each(this.nodes, function (k, node ){ + if (node.type === 'tool'){ + var node_changed = false; + if (node.post_job_actions == null){ + node.post_job_actions = {}; + node_changed = true; + } + var pjas_to_rem = []; + $.each(node.post_job_actions, function(pja_id, pja){ + if (pja.action_type == "HideDatasetAction"){ + pjas_to_rem.push(pja_id); + } + }); + if (pjas_to_rem.length > 0 && node == workflow.active_node) { + $.each(pjas_to_rem, function(i, pja_name){ + node_changed = true; + delete node.post_job_actions[pja_name]; + }); + } + if (using_workflow_outputs){ + $.each(node.output_terminals, function(ot_id, ot){ + var create_pja = true; + $.each(node.workflow_outputs, function(i, wo_name){ + if (ot.name === wo_name){ + create_pja = false; + } + }); + if (create_pja === true){ + node_changed = true; + var pja = { + action_type : "HideDatasetAction", + output_name : ot.name, + action_arguments : {} + } + node.post_job_actions['HideDatasetAction'+ot.name] = null; + node.post_job_actions['HideDatasetAction'+ot.name] = pja; + } + }); + } + // lastly, if this is the active node, and we made changes, reload the display at right. + if (workflow.active_node == node && node_changed === true) { + workflow.reload_active_node(); + } + } + }); } - wf = this; - $.each(this.nodes, function (k, node ){ - if (node.type == 'tool'){ - var node_changed = false; - if (node.post_job_actions == null){ - node.post_job_actions = {}; - } - var pjas_to_rem = []; - $.each(node.post_job_actions, function(pja_id, pja){ - if (pja.action_type == "HideDatasetAction"){ - pjas_to_rem.push(pja_id); - } - }); - if (pjas_to_rem.length > 0 && node == workflow.active_node) { - $.each(pjas_to_rem, function(i, pja_name){ - node_changed = true; - delete node.post_job_actions[pja_name]; - }); - } - $.each(node.output_terminals, function(ot_id, ot){ - var create_pja = true; - $.each(node.workflow_outputs, function(i, wo_name){ - if (ot.name == wo_name){ - create_pja = false; - } - }); - if (create_pja == true){ - node_changed = true; - var pja = { - action_type : "HideDatasetAction", - output_name : ot.name, - action_arguments : {} - } - node.post_job_actions['HideDatasetAction'+ot.name] = null; - node.post_job_actions['HideDatasetAction'+ot.name] = pja; - } - }); - // lastly, if this is the active node, and we made changes, reload the display at right. - if (wf.active_node == node && node_changed == true) { - wf.reload_active_node(); - } - } - }); }, to_simple : function () { var nodes = {}; @@ -580,6 +586,7 @@ $.extend( Workflow.prototype, { var max_id = 0; wf.name = data.name; // First pass, nodes + var using_workflow_outputs = false; $.each( data.steps, function( id, step ) { var node = prebuild_node( "tool", step.name, step.tool_id ); node.init_field_data( step ); @@ -589,6 +596,20 @@ $.extend( Workflow.prototype, { node.id = step.id; wf.nodes[ node.id ] = node; max_id = Math.max( max_id, parseInt( id ) ); + // For older workflows, it's possible to have HideDataset PJAs, but not WorkflowOutputs. + // Check for either, and then add outputs in the next pass. + if (!using_workflow_outputs && node.type === 'tool'){ + if (node.workflow_outputs.length > 0){ + using_workflow_outputs = true; + } + else{ + $.each(node.post_job_actions, function(pja_id, pja){ + if (pja.action_type === "HideDatasetAction"){ + using_workflow_outputs = true; + } + }); + } + } }); wf.id_counter = max_id + 1; // Second pass, connections @@ -603,6 +624,17 @@ $.extend( Workflow.prototype, { c.redraw(); } }); + if(using_workflow_outputs && node.type === 'tool'){ + // Ensure that every output terminal has a WorkflowOutput or HideDatasetAction. + $.each(node.output_terminals, function(ot_id, ot){ + if(node.post_job_actions['HideDatasetAction'+ot.name] === undefined){ + node.workflow_outputs.push(ot.name); + callout = $(node.element).find('.callout.'+ot.name); + callout.find('img').attr('src', galaxy_paths.attributes.image_path + '/fugue/asterisk-small.png'); + workflow.has_changes = true; + } + }); + } }); }, check_changes_in_active_form : function() { @@ -793,18 +825,18 @@ function prebuild_node( type, title_text, tool_id ) { f.append( title ); f.css( "left", $(window).scrollLeft() + 20 ); f.css( "top", $(window).scrollTop() + 20 ); var b = $("
    "); - var tmp = "
    loading tool info...
    "; + var tmp = "
    loading tool info...
    "; b.append( tmp ); node.form_html = tmp; f.append( b ); // Fix width to computed width // Now add floats var buttons = $("
    "); - buttons.append( $("").attr("src", image_path + '/delete_icon.png').click( function( e ) { + buttons.append( $("").attr("src", galaxy_paths.attributes.image_path + '/delete_icon.png').click( function( e ) { node.destroy(); } ).hover( - function() { $(this).attr( "src", image_path + "/delete_icon_dark.png" ); }, - function() { $(this).attr( "src", image_path + "/delete_icon.png" ); } + function() { $(this).attr( "src", galaxy_paths.attributes.image_path + "/delete_icon_dark.png" ); }, + function() { $(this).attr( "src", galaxy_paths.attributes.image_path + "/delete_icon.png" ); } ) ); // Place inside container f.appendTo( "#canvas-container" ); diff --git a/static/scripts/libs/backbone-relational.js b/static/scripts/libs/backbone-relational.js new file mode 100644 index 00000000000..5b049034537 --- /dev/null +++ b/static/scripts/libs/backbone-relational.js @@ -0,0 +1,1406 @@ +/** + * Backbone-relational.js 0.5.0 + * (c) 2011 Paul Uithol + * + * Backbone-relational may be freely distributed under the MIT license. + * For details and documentation: https://github.com/PaulUithol/Backbone-relational. + * Depends on Backbone: https://github.com/documentcloud/backbone. + */ +( function( undefined ) { + "use strict"; + + /** + * CommonJS shim + **/ + var _, Backbone, exports; + if ( typeof window === 'undefined' ) { + _ = require( 'underscore' ); + Backbone = require( 'backbone' ); + exports = module.exports = Backbone; + } + else { + var _ = window._; + Backbone = window.Backbone; + exports = window; + } + + Backbone.Relational = { + showWarnings: true + }; + + /** + * Semaphore mixin; can be used as both binary and counting. + **/ + Backbone.Semaphore = { + _permitsAvailable: null, + _permitsUsed: 0, + + acquire: function() { + if ( this._permitsAvailable && this._permitsUsed >= this._permitsAvailable ) { + throw new Error( 'Max permits acquired' ); + } + else { + this._permitsUsed++; + } + }, + + release: function() { + if ( this._permitsUsed === 0 ) { + throw new Error( 'All permits released' ); + } + else { + this._permitsUsed--; + } + }, + + isLocked: function() { + return this._permitsUsed > 0; + }, + + setAvailablePermits: function( amount ) { + if ( this._permitsUsed > amount ) { + throw new Error( 'Available permits cannot be less than used permits' ); + } + this._permitsAvailable = amount; + } + }; + + /** + * A BlockingQueue that accumulates items while blocked (via 'block'), + * and processes them when unblocked (via 'unblock'). + * Process can also be called manually (via 'process'). + */ + Backbone.BlockingQueue = function() { + this._queue = []; + }; + _.extend( Backbone.BlockingQueue.prototype, Backbone.Semaphore, { + _queue: null, + + add: function( func ) { + if ( this.isBlocked() ) { + this._queue.push( func ); + } + else { + func(); + } + }, + + process: function() { + while ( this._queue && this._queue.length ) { + this._queue.shift()(); + } + }, + + block: function() { + this.acquire(); + }, + + unblock: function() { + this.release(); + if ( !this.isBlocked() ) { + this.process(); + } + }, + + isBlocked: function() { + return this.isLocked(); + } + }); + /** + * Global event queue. Accumulates external events ('add:', 'remove:' and 'update:') + * until the top-level object is fully initialized (see 'Backbone.RelationalModel'). + */ + Backbone.Relational.eventQueue = new Backbone.BlockingQueue(); + + /** + * Backbone.Store keeps track of all created (and destruction of) Backbone.RelationalModel. + * Handles lookup for relations. + */ + Backbone.Store = function() { + this._collections = []; + this._reverseRelations = []; + }; + _.extend( Backbone.Store.prototype, Backbone.Events, { + _collections: null, + _reverseRelations: null, + + /** + * Add a reverse relation. Is added to the 'relations' property on model's prototype, and to + * existing instances of 'model' in the store as well. + * @param {object} relation + * @param {Backbone.RelationalModel} relation.model + * @param {String} relation.type + * @param {String} relation.key + * @param {String|object} relation.relatedModel + */ + addReverseRelation: function( relation ) { + var exists = _.any( this._reverseRelations, function( rel ) { + return _.all( relation, function( val, key ) { + return val === rel[ key ]; + }); + }); + + if ( !exists && relation.model && relation.type ) { + this._reverseRelations.push( relation ); + + if ( !relation.model.prototype.relations ) { + relation.model.prototype.relations = []; + } + relation.model.prototype.relations.push( relation ); + + this.retroFitRelation( relation ); + } + }, + + /** + * Add a 'relation' to all existing instances of 'relation.model' in the store + * @param {object} relation + */ + retroFitRelation: function( relation ) { + var coll = this.getCollection( relation.model ); + coll.each( function( model ) { + new relation.type( model, relation ); + }, this); + }, + + /** + * Find the Store's collection for a certain type of model. + * @param {Backbone.RelationalModel} model + * @return {Backbone.Collection} A collection if found (or applicable for 'model'), or null + */ + getCollection: function( model ) { + var coll = _.detect( this._collections, function( c ) { + // Check if model is the type itself (a ref to the constructor), or is of type c.model + return model === c.model || model.constructor === c.model; + }); + + if ( !coll ) { + coll = this._createCollection( model ); + } + + return coll; + }, + + /** + * Find a type on the global object by name. Splits name on dots. + * @param {String} name + * @return {Object} + */ + getObjectByName: function( name ) { + var type = _.reduce( name.split( '.' ), function( memo, val ) { + return memo[ val ]; + }, exports); + return type !== exports ? type: null; + }, + + _createCollection: function( type ) { + var coll; + + // If 'type' is an instance, take it's constructor + if ( type instanceof Backbone.RelationalModel ) { + type = type.constructor; + } + + // Type should inherit from Backbone.RelationalModel. + if ( type.prototype instanceof Backbone.RelationalModel.prototype.constructor ) { + coll = new Backbone.Collection(); + coll.model = type; + + this._collections.push( coll ); + } + + return coll; + }, + + /** + * Find an id + * @param type + * @param {String|Number|Object|Backbone.RelationalModel} item + */ + resolveIdForItem: function( type, item ) { + var id = _.isString( item ) || _.isNumber( item ) ? item : null; + + if ( id == null ) { + if ( item instanceof Backbone.RelationalModel ) { + id = item.id; + } + else if ( _.isObject( item ) ) { + id = item[ type.prototype.idAttribute ]; + } + } + + return id; + }, + + /** + * + * @param type + * @param {String|Number|Object|Backbone.RelationalModel} item + */ + find: function( type, item ) { + var id = this.resolveIdForItem( type, item ); + var coll = this.getCollection( type ); + return coll && coll.get( id ); + }, + + /** + * Add a 'model' to it's appropriate collection. Retain the original contents of 'model.collection'. + * @param {Backbone.RelationalModel} model + */ + register: function( model ) { + var modelColl = model.collection; + var coll = this.getCollection( model ); + coll && coll.add( model ); + model.bind( 'destroy', this.unregister, this ); + model.collection = modelColl; + }, + + /** + * Explicitly update a model's id in it's store collection + * @param {Backbone.RelationalModel} model + */ + update: function( model ) { + var coll = this.getCollection( model ); + coll._onModelEvent( 'change:' + model.idAttribute, model, coll ); + }, + + /** + * Remove a 'model' from the store. + * @param {Backbone.RelationalModel} model + */ + unregister: function( model ) { + model.unbind( 'destroy', this.unregister ); + var coll = this.getCollection( model ); + coll && coll.remove( model ); + } + }); + Backbone.Relational.store = new Backbone.Store(); + + /** + * The main Relation class, from which 'HasOne' and 'HasMany' inherit. Internally, 'relational:' events + * are used to regulate addition and removal of models from relations. + * + * @param {Backbone.RelationalModel} instance + * @param {object} options + * @param {string} options.key + * @param {Backbone.RelationalModel.constructor} options.relatedModel + * @param {Boolean|String} [options.includeInJSON=true] Serialize the given attribute for related model(s)' in toJSON, or just their ids. + * @param {Boolean} [options.createModels=true] Create objects from the contents of keys if the object is not found in Backbone.store. + * @param {object} [options.reverseRelation] Specify a bi-directional relation. If provided, Relation will reciprocate + * the relation to the 'relatedModel'. Required and optional properties match 'options', except that it also needs + * {Backbone.Relation|String} type ('HasOne' or 'HasMany'). + */ + Backbone.Relation = function( instance, options ) { + this.instance = instance; + // Make sure 'options' is sane, and fill with defaults from subclasses and this object's prototype + options = ( typeof options === 'object' && options ) || {}; + this.reverseRelation = _.defaults( options.reverseRelation || {}, this.options.reverseRelation ); + this.reverseRelation.type = !_.isString( this.reverseRelation.type ) ? this.reverseRelation.type : + Backbone[ this.reverseRelation.type ] || Backbone.Relational.store.getObjectByName( this.reverseRelation.type ); + this.model = options.model || this.instance.constructor; + this.options = _.defaults( options, this.options, Backbone.Relation.prototype.options ); + + this.key = this.options.key; + this.keySource = this.options.keySource || this.key; + this.keyDestination = this.options.keyDestination || this.options.keySource || this.key; + + // 'exports' should be the global object where 'relatedModel' can be found on if given as a string. + this.relatedModel = this.options.relatedModel; + if ( _.isString( this.relatedModel ) ) { + this.relatedModel = Backbone.Relational.store.getObjectByName( this.relatedModel ); + } + + if ( !this.checkPreconditions() ) { + return false; + } + + if ( instance ) { + this.keyContents = this.instance.get( this.keySource ); + + // Explicitly clear 'keySource', to prevent a leaky abstraction if 'keySource' differs from 'key'. + if ( this.key !== this.keySource ) { + this.instance.unset( this.keySource, { silent: true } ); + } + + // Add this Relation to instance._relations + this.instance._relations.push( this ); + } + + // Add the reverse relation on 'relatedModel' to the store's reverseRelations + if ( !this.options.isAutoRelation && this.reverseRelation.type && this.reverseRelation.key ) { + Backbone.Relational.store.addReverseRelation( _.defaults( { + isAutoRelation: true, + model: this.relatedModel, + relatedModel: this.model, + reverseRelation: this.options // current relation is the 'reverseRelation' for it's own reverseRelation + }, + this.reverseRelation // Take further properties from this.reverseRelation (type, key, etc.) + ) ); + } + + _.bindAll( this, '_modelRemovedFromCollection', '_relatedModelAdded', '_relatedModelRemoved' ); + + if( instance ) { + this.initialize(); + + // When a model in the store is destroyed, check if it is 'this.instance'. + Backbone.Relational.store.getCollection( this.instance ) + .bind( 'relational:remove', this._modelRemovedFromCollection ); + + // When 'relatedModel' are created or destroyed, check if it affects this relation. + Backbone.Relational.store.getCollection( this.relatedModel ) + .bind( 'relational:add', this._relatedModelAdded ) + .bind( 'relational:remove', this._relatedModelRemoved ); + } + }; + // Fix inheritance :\ + Backbone.Relation.extend = Backbone.Model.extend; + // Set up all inheritable **Backbone.Relation** properties and methods. + _.extend( Backbone.Relation.prototype, Backbone.Events, Backbone.Semaphore, { + options: { + createModels: true, + includeInJSON: true, + isAutoRelation: false + }, + + instance: null, + key: null, + keyContents: null, + relatedModel: null, + reverseRelation: null, + related: null, + + _relatedModelAdded: function( model, coll, options ) { + // Allow 'model' to set up it's relations, before calling 'tryAddRelated' + // (which can result in a call to 'addRelated' on a relation of 'model') + var dit = this; + model.queue( function() { + dit.tryAddRelated( model, options ); + }); + }, + + _relatedModelRemoved: function( model, coll, options ) { + this.removeRelated( model, options ); + }, + + _modelRemovedFromCollection: function( model ) { + if ( model === this.instance ) { + this.destroy(); + } + }, + + /** + * Check several pre-conditions. + * @return {Boolean} True if pre-conditions are satisfied, false if they're not. + */ + checkPreconditions: function() { + var i = this.instance, + k = this.key, + m = this.model, + rm = this.relatedModel, + warn = Backbone.Relational.showWarnings && typeof console !== 'undefined'; + + if ( !m || !k || !rm ) { + warn && console.warn( 'Relation=%o; no model, key or relatedModel (%o, %o, %o)', this, m, k, rm ); + return false; + } + // Check if the type in 'relatedModel' inherits from Backbone.RelationalModel + if ( !( m.prototype instanceof Backbone.RelationalModel.prototype.constructor ) ) { + warn && console.warn( 'Relation=%o; model does not inherit from Backbone.RelationalModel (%o)', this, i ); + return false; + } + // Check if the type in 'relatedModel' inherits from Backbone.RelationalModel + if ( !( rm.prototype instanceof Backbone.RelationalModel.prototype.constructor ) ) { + warn && console.warn( 'Relation=%o; relatedModel does not inherit from Backbone.RelationalModel (%o)', this, rm ); + return false; + } + // Check if this is not a HasMany, and the reverse relation is HasMany as well + if ( this instanceof Backbone.HasMany && this.reverseRelation.type === Backbone.HasMany.prototype.constructor ) { + warn && console.warn( 'Relation=%o; relation is a HasMany, and the reverseRelation is HasMany as well.', this ); + return false; + } + + // Check if we're not attempting to create a duplicate relationship + if( i && i._relations.length ) { + var exists = _.any( i._relations, function( rel ) { + var hasReverseRelation = this.reverseRelation.key && rel.reverseRelation.key; + return rel.relatedModel === rm && rel.key === k && + ( !hasReverseRelation || this.reverseRelation.key === rel.reverseRelation.key ); + }, this ); + + if ( exists ) { + warn && console.warn( 'Relation=%o between instance=%o.%s and relatedModel=%o.%s already exists', + this, i, k, rm, this.reverseRelation.key ); + return false; + } + } + + return true; + }, + + /** + * Set the related model(s) for this relation + * @param {Backbone.Mode|Backbone.Collection} related + * @param {Object} [options] + */ + setRelated: function( related, options ) { + this.related = related; + + this.instance.acquire(); + this.instance.set( this.key, related, _.defaults( options || {}, { silent: true } ) ); + this.instance.release(); + }, + + createModel: function( item ) { + if ( this.options.createModels && typeof( item ) === 'object' ) { + return new this.relatedModel( item ); + } + }, + + /** + * Determine if a relation (on a different RelationalModel) is the reverse + * relation of the current one. + * @param {Backbone.Relation} relation + * @return {Boolean} + */ + _isReverseRelation: function( relation ) { + if ( relation.instance instanceof this.relatedModel && this.reverseRelation.key === relation.key && + this.key === relation.reverseRelation.key ) { + return true; + } + return false; + }, + + /** + * Get the reverse relations (pointing back to 'this.key' on 'this.instance') for the currently related model(s). + * @param {Backbone.RelationalModel} [model] Get the reverse relations for a specific model. + * If not specified, 'this.related' is used. + * @return {Backbone.Relation[]} + */ + getReverseRelations: function( model ) { + var reverseRelations = []; + // Iterate over 'model', 'this.related.models' (if this.related is a Backbone.Collection), or wrap 'this.related' in an array. + var models = !_.isUndefined( model ) ? [ model ] : this.related && ( this.related.models || [ this.related ] ); + _.each( models , function( related ) { + _.each( related.getRelations(), function( relation ) { + if ( this._isReverseRelation( relation ) ) { + reverseRelations.push( relation ); + } + }, this ); + }, this ); + + return reverseRelations; + }, + + /** + * Rename options.silent to options.silentChange, so events propagate properly. + * (for example in HasMany, from 'addRelated'->'handleAddition') + * @param {Object} [options] + * @return {Object} + */ + sanitizeOptions: function( options ) { + options = options ? _.clone( options ) : {}; + if ( options.silent ) { + options = _.extend( {}, options, { silentChange: true } ); + delete options.silent; + } + return options; + }, + + /** + * Rename options.silentChange to options.silent, so events are silenced as intended in Backbone's + * original functions. + * @param {Object} [options] + * @return {Object} + */ + unsanitizeOptions: function( options ) { + options = options ? _.clone( options ) : {}; + if ( options.silentChange ) { + options = _.extend( {}, options, { silent: true } ); + delete options.silentChange; + } + return options; + }, + + // Cleanup. Get reverse relation, call removeRelated on each. + destroy: function() { + Backbone.Relational.store.getCollection( this.instance ) + .unbind( 'relational:remove', this._modelRemovedFromCollection ); + + Backbone.Relational.store.getCollection( this.relatedModel ) + .unbind( 'relational:add', this._relatedModelAdded ) + .unbind( 'relational:remove', this._relatedModelRemoved ); + + _.each( this.getReverseRelations(), function( relation ) { + relation.removeRelated( this.instance ); + }, this ); + } + }); + + Backbone.HasOne = Backbone.Relation.extend({ + options: { + reverseRelation: { type: 'HasMany' } + }, + + initialize: function() { + _.bindAll( this, 'onChange' ); + + this.instance.bind( 'relational:change:' + this.key, this.onChange ); + + var model = this.findRelated( { silent: true } ); + this.setRelated( model ); + + // Notify new 'related' object of the new relation. + var dit = this; + _.each( dit.getReverseRelations(), function( relation ) { + relation.addRelated( dit.instance ); + } ); + }, + + findRelated: function( options ) { + var item = this.keyContents; + var model = null; + + if ( item instanceof this.relatedModel ) { + model = item; + } + else if ( item ) { + // Try to find an instance of the appropriate 'relatedModel' in the store, or create it + model = Backbone.Relational.store.find( this.relatedModel, item ); + + if ( model && _.isObject( item ) ) { + model.set( item, options ); + } + else if ( !model ) { + model = this.createModel( item ); + } + } + + return model; + }, + + /** + * If the key is changed, notify old & new reverse relations and initialize the new relation + */ + onChange: function( model, attr, options ) { + // Don't accept recursive calls to onChange (like onChange->findRelated->createModel->initializeRelations->addRelated->onChange) + if ( this.isLocked() ) { + return; + } + this.acquire(); + options = this.sanitizeOptions( options ); + + // 'options._related' is set by 'addRelated'/'removeRelated'. If it is set, the change + // is the result of a call from a relation. If it's not, the change is the result of + // a 'set' call on this.instance. + var changed = _.isUndefined( options._related ); + var oldRelated = changed ? this.related : options._related; + + if ( changed ) { + this.keyContents = attr; + + // Set new 'related' + if ( attr instanceof this.relatedModel ) { + this.related = attr; + } + else if ( attr ) { + var related = this.findRelated( options ); + this.setRelated( related ); + } + else { + this.setRelated( null ); + } + } + + // Notify old 'related' object of the terminated relation + if ( oldRelated && this.related !== oldRelated ) { + _.each( this.getReverseRelations( oldRelated ), function( relation ) { + relation.removeRelated( this.instance, options ); + }, this ); + } + + // Notify new 'related' object of the new relation. Note we do re-apply even if this.related is oldRelated; + // that can be necessary for bi-directional relations if 'this.instance' was created after 'this.related'. + // In that case, 'this.instance' will already know 'this.related', but the reverse might not exist yet. + _.each( this.getReverseRelations(), function( relation ) { + relation.addRelated( this.instance, options ); + }, this); + + // Fire the 'update:' event if 'related' was updated + if ( !options.silentChange && this.related !== oldRelated ) { + var dit = this; + Backbone.Relational.eventQueue.add( function() { + dit.instance.trigger( 'update:' + dit.key, dit.instance, dit.related, options ); + }); + } + this.release(); + }, + + /** + * If a new 'this.relatedModel' appears in the 'store', try to match it to the last set 'keyContents' + */ + tryAddRelated: function( model, options ) { + if ( this.related ) { + return; + } + options = this.sanitizeOptions( options ); + + var item = this.keyContents; + if ( item ) { + var id = Backbone.Relational.store.resolveIdForItem( this.relatedModel, item ); + if ( model.id === id ) { + this.addRelated( model, options ); + } + } + }, + + addRelated: function( model, options ) { + if ( model !== this.related ) { + var oldRelated = this.related || null; + this.setRelated( model ); + this.onChange( this.instance, model, { _related: oldRelated } ); + } + }, + + removeRelated: function( model, options ) { + if ( !this.related ) { + return; + } + + if ( model === this.related ) { + var oldRelated = this.related || null; + this.setRelated( null ); + this.onChange( this.instance, model, { _related: oldRelated } ); + } + } + }); + + Backbone.HasMany = Backbone.Relation.extend({ + collectionType: null, + + options: { + reverseRelation: { type: 'HasOne' }, + collectionType: Backbone.Collection, + collectionKey: true, + collectionOptions: {} + }, + + initialize: function() { + _.bindAll( this, 'onChange', 'handleAddition', 'handleRemoval', 'handleReset' ); + this.instance.bind( 'relational:change:' + this.key, this.onChange ); + + // Handle a custom 'collectionType' + this.collectionType = this.options.collectionType; + if ( _( this.collectionType ).isString() ) { + this.collectionType = Backbone.Relational.store.getObjectByName( this.collectionType ); + } + if ( !this.collectionType.prototype instanceof Backbone.Collection.prototype.constructor ){ + throw new Error( 'collectionType must inherit from Backbone.Collection' ); + } + + // Handle cases where a model/relation is created with a collection passed straight into 'attributes' + if ( this.keyContents instanceof Backbone.Collection ) { + this.setRelated( this._prepareCollection( this.keyContents ) ); + } + else { + this.setRelated( this._prepareCollection() ); + } + + this.findRelated( { silent: true } ); + }, + + _getCollectionOptions: function() { + return _.isFunction( this.options.collectionOptions ) ? + this.options.collectionOptions( this.instance ) : + this.options.collectionOptions; + }, + + /** + * Bind events and setup collectionKeys for a collection that is to be used as the backing store for a HasMany. + * If no 'collection' is supplied, a new collection will be created of the specified 'collectionType' option. + * @param {Backbone.Collection} [collection] + */ + _prepareCollection: function( collection ) { + if ( this.related ) { + this.related + .unbind( 'relational:add', this.handleAddition ) + .unbind( 'relational:remove', this.handleRemoval ) + .unbind( 'relational:reset', this.handleReset ) + } + + if ( !collection || !( collection instanceof Backbone.Collection ) ) { + collection = new this.collectionType( [], this._getCollectionOptions() ); + } + + collection.model = this.relatedModel; + + if ( this.options.collectionKey ) { + var key = this.options.collectionKey === true ? this.options.reverseRelation.key : this.options.collectionKey; + + if (collection[ key ] && collection[ key ] !== this.instance ) { + if ( Backbone.Relational.showWarnings && typeof console !== 'undefined' ) { + console.warn( 'Relation=%o; collectionKey=%s already exists on collection=%o', this, key, this.options.collectionKey ); + } + } + else if (key) { + collection[ key ] = this.instance; + } + } + + collection + .bind( 'relational:add', this.handleAddition ) + .bind( 'relational:remove', this.handleRemoval ) + .bind( 'relational:reset', this.handleReset ); + + return collection; + }, + + findRelated: function( options ) { + if ( this.keyContents ) { + var models = []; + + if ( this.keyContents instanceof Backbone.Collection ) { + models = this.keyContents.models; + } + else { + // Handle cases the an API/user supplies just an Object/id instead of an Array + this.keyContents = _.isArray( this.keyContents ) ? this.keyContents : [ this.keyContents ]; + + // Try to find instances of the appropriate 'relatedModel' in the store + _.each( this.keyContents, function( item ) { + var model = Backbone.Relational.store.find( this.relatedModel, item ); + + if ( model && _.isObject( item ) ) { + model.set( item, options ); + } + else if ( !model ) { + model = this.createModel( item ); + } + + if ( model && !this.related.getByCid( model ) && !this.related.get( model ) ) { + models.push( model ); + } + }, this ); + } + + // Add all found 'models' in on go, so 'add' will only be called once (and thus 'sort', etc.) + if ( models.length ) { + options = this.unsanitizeOptions( options ); + this.related.add( models, options ); + } + } + }, + + /** + * If the key is changed, notify old & new reverse relations and initialize the new relation + */ + onChange: function( model, attr, options ) { + options = this.sanitizeOptions( options ); + this.keyContents = attr; + + // Notify old 'related' object of the terminated relation + _.each( this.getReverseRelations(), function( relation ) { + relation.removeRelated( this.instance, options ); + }, this ); + + // Replace 'this.related' by 'attr' if it is a Backbone.Collection + if ( attr instanceof Backbone.Collection ) { + this._prepareCollection( attr ); + this.related = attr; + } + // Otherwise, 'attr' should be an array of related object ids. + // Re-use the current 'this.related' if it is a Backbone.Collection, and remove any current entries. + // Otherwise, create a new collection. + else { + var coll; + + if ( this.related instanceof Backbone.Collection ) { + coll = this.related; + coll.reset( [], { silent: true } ); + } + else { + coll = this._prepareCollection(); + } + + this.setRelated( coll ); + this.findRelated( options ); + } + + // Notify new 'related' object of the new relation + _.each( this.getReverseRelations(), function( relation ) { + relation.addRelated( this.instance, options ); + }, this ); + + var dit = this; + Backbone.Relational.eventQueue.add( function() { + !options.silentChange && dit.instance.trigger( 'update:' + dit.key, dit.instance, dit.related, options ); + }); + }, + + tryAddRelated: function( model, options ) { + options = this.sanitizeOptions( options ); + if ( !this.related.getByCid( model ) && !this.related.get( model ) ) { + // Check if this new model was specified in 'this.keyContents' + var item = _.any( this.keyContents, function( item ) { + var id = Backbone.Relational.store.resolveIdForItem( this.relatedModel, item ); + return id && id === model.id; + }, this ); + + if ( item ) { + this.related.add( model, options ); + } + } + }, + + /** + * When a model is added to a 'HasMany', trigger 'add' on 'this.instance' and notify reverse relations. + * (should be 'HasOne', must set 'this.instance' as their related). + */ + handleAddition: function( model, coll, options ) { + //console.debug('handleAddition called; args=%o', arguments); + // Make sure the model is in fact a valid model before continuing. + // (it can be invalid as a result of failing validation in Backbone.Collection._prepareModel) + if( !( model instanceof Backbone.Model ) ) { + return; + } + + options = this.sanitizeOptions( options ); + + _.each( this.getReverseRelations( model ), function( relation ) { + relation.addRelated( this.instance, options ); + }, this ); + + // Only trigger 'add' once the newly added model is initialized (so, has it's relations set up) + var dit = this; + Backbone.Relational.eventQueue.add( function() { + !options.silentChange && dit.instance.trigger( 'add:' + dit.key, model, dit.related, options ); + }); + }, + + /** + * When a model is removed from a 'HasMany', trigger 'remove' on 'this.instance' and notify reverse relations. + * (should be 'HasOne', which should be nullified) + */ + handleRemoval: function( model, coll, options ) { + //console.debug('handleRemoval called; args=%o', arguments); + if( !( model instanceof Backbone.Model ) ) { + return; + } + + options = this.sanitizeOptions( options ); + + _.each( this.getReverseRelations( model ), function( relation ) { + relation.removeRelated( this.instance, options ); + }, this ); + + var dit = this; + Backbone.Relational.eventQueue.add( function() { + !options.silentChange && dit.instance.trigger( 'remove:' + dit.key, model, dit.related, options ); + }); + }, + + handleReset: function( coll, options ) { + options = this.sanitizeOptions( options ); + + var dit = this; + Backbone.Relational.eventQueue.add( function() { + !options.silentChange && dit.instance.trigger( 'reset:' + dit.key, dit.related, options ); + }); + }, + + addRelated: function( model, options ) { + var dit = this; + options = this.unsanitizeOptions( options ); + model.queue( function() { // Queued to avoid errors for adding 'model' to the 'this.related' set twice + if ( dit.related && !dit.related.getByCid( model ) && !dit.related.get( model ) ) { + dit.related.add( model, options ); + } + }); + }, + + removeRelated: function( model, options ) { + options = this.unsanitizeOptions( options ); + if ( this.related.getByCid( model ) || this.related.get( model ) ) { + this.related.remove( model, options ); + } + } + }); + + /** + * A type of Backbone.Model that also maintains relations to other models and collections. + * New events when compared to the original: + * - 'add:' (model, related collection, options) + * - 'remove:' (model, related collection, options) + * - 'update:' (model, related model or collection, options) + */ + Backbone.RelationalModel = Backbone.Model.extend({ + relations: null, // Relation descriptions on the prototype + _relations: null, // Relation instances + _isInitialized: false, + _deferProcessing: false, + _queue: null, + + constructor: function( attributes, options ) { + // Nasty hack, for cases like 'model.get( ).add( item )'. + // Defer 'processQueue', so that when 'Relation.createModels' is used we: + // a) Survive 'Backbone.Collection.add'; this takes care we won't error on "can't add model to a set twice" + // (by creating a model from properties, having the model add itself to the collection via one of + // it's relations, then trying to add it to the collection). + // b) Trigger 'HasMany' collection events only after the model is really fully set up. + // Example that triggers both a and b: "p.get('jobs').add( { company: c, person: p } )". + var dit = this; + if ( options && options.collection ) { + this._deferProcessing = true; + + var processQueue = function( model ) { + if ( model === dit ) { + dit._deferProcessing = false; + dit.processQueue(); + options.collection.unbind( 'relational:add', processQueue ); + } + }; + options.collection.bind( 'relational:add', processQueue ); + + // So we do process the queue eventually, regardless of whether this model really gets added to 'options.collection'. + _.defer( function() { + processQueue( dit ); + }); + } + + this._queue = new Backbone.BlockingQueue(); + this._queue.block(); + Backbone.Relational.eventQueue.block(); + + Backbone.Model.prototype.constructor.apply( this, arguments ); + + // Try to run the global queue holding external events + Backbone.Relational.eventQueue.unblock(); + }, + + /** + * Override 'trigger' to queue 'change' and 'change:*' events + */ + trigger: function( eventName ) { + if ( eventName.length > 5 && 'change' === eventName.substr( 0, 6 ) ) { + var dit = this, args = arguments; + Backbone.Relational.eventQueue.add( function() { + Backbone.Model.prototype.trigger.apply( dit, args ); + }); + } + else { + Backbone.Model.prototype.trigger.apply( this, arguments ); + } + + return this; + }, + + /** + * Initialize Relations present in this.relations; determine the type (HasOne/HasMany), then creates a new instance. + * Invoked in the first call so 'set' (which is made from the Backbone.Model constructor). + */ + initializeRelations: function() { + this.acquire(); // Setting up relations often also involve calls to 'set', and we only want to enter this function once + this._relations = []; + + _.each( this.relations, function( rel ) { + var type = !_.isString( rel.type ) ? rel.type : Backbone[ rel.type ] || Backbone.Relational.store.getObjectByName( rel.type ); + if ( type && type.prototype instanceof Backbone.Relation.prototype.constructor ) { + new type( this, rel ); // Also pushes the new Relation into _relations + } + else { + Backbone.Relational.showWarnings && typeof console !== 'undefined' && console.warn( 'Relation=%o; missing or invalid type!', rel ); + } + }, this ); + + this._isInitialized = true; + this.release(); + this.processQueue(); + }, + + /** + * When new values are set, notify this model's relations (also if options.silent is set). + * (Relation.setRelated locks this model before calling 'set' on it to prevent loops) + */ + updateRelations: function( options ) { + if( this._isInitialized && !this.isLocked() ) { + _.each( this._relations, function( rel ) { + var val = this.attributes[ rel.key ]; + if ( rel.related !== val ) { + this.trigger('relational:change:' + rel.key, this, val, options || {} ); + } + }, this ); + } + }, + + /** + * Either add to the queue (if we're not initialized yet), or execute right away. + */ + queue: function( func ) { + this._queue.add( func ); + }, + + /** + * Process _queue + */ + processQueue: function() { + if ( this._isInitialized && !this._deferProcessing && this._queue.isBlocked() ) { + this._queue.unblock(); + } + }, + + /** + * Get a specific relation. + * @param key {string} The relation key to look for. + * @return {Backbone.Relation} An instance of 'Backbone.Relation', if a relation was found for 'key', or null. + */ + getRelation: function( key ) { + return _.detect( this._relations, function( rel ) { + if ( rel.key === key ) { + return true; + } + }, this ); + }, + + /** + * Get all of the created relations. + * @return {Backbone.Relation[]} + */ + getRelations: function() { + return this._relations; + }, + + /** + * Retrieve related objects. + * @param key {string} The relation key to fetch models for. + * @param options {object} Options for 'Backbone.Model.fetch' and 'Backbone.sync'. + * @return {jQuery.when[]} An array of request objects + */ + fetchRelated: function( key, options ) { + options || ( options = {} ); + var setUrl, + requests = [], + rel = this.getRelation( key ), + keyContents = rel && rel.keyContents, + toFetch = keyContents && _.select( _.isArray( keyContents ) ? keyContents : [ keyContents ], function( item ) { + var id = Backbone.Relational.store.resolveIdForItem( rel.relatedModel, item ); + return id && !Backbone.Relational.store.find( rel.relatedModel, id ); + }, this ); + + if ( toFetch && toFetch.length ) { + // Create a model for each entry in 'keyContents' that is to be fetched + var models = _.map( toFetch, function( item ) { + var model; + + if ( typeof( item ) === 'object' ) { + model = new rel.relatedModel( item ); + } + else { + var attrs = {}; + attrs[ rel.relatedModel.prototype.idAttribute ] = item; + model = new rel.relatedModel( attrs ); + } + + return model; + }, this ); + + // Try if the 'collection' can provide a url to fetch a set of models in one request. + if ( rel.related instanceof Backbone.Collection && _.isFunction( rel.related.url ) ) { + setUrl = rel.related.url( models ); + } + + // An assumption is that when 'Backbone.Collection.url' is a function, it can handle building of set urls. + // To make sure it can, test if the url we got by supplying a list of models to fetch is different from + // the one supplied for the default fetch action (without args to 'url'). + if ( setUrl && setUrl !== rel.related.url() ) { + var opts = _.defaults( + { + error: function() { + var args = arguments; + _.each( models, function( model ) { + model.trigger( 'destroy', model, model.collection, options ); + options.error && options.error.apply( model, args ); + }); + }, + url: setUrl + }, + options, + { add: true } + ); + + requests = [ rel.related.fetch( opts ) ]; + } + else { + requests = _.map( models, function( model ) { + var opts = _.defaults( + { + error: function() { + model.trigger( 'destroy', model, model.collection, options ); + options.error && options.error.apply( model, arguments ); + } + }, + options + ); + return model.fetch( opts ); + }, this ); + } + } + + return requests; + }, + + set: function( key, value, options ) { + Backbone.Relational.eventQueue.block(); + + // Duplicate backbone's behavior to allow separate key/value parameters, instead of a single 'attributes' object + var attributes; + if (_.isObject( key ) || key == null) { + attributes = key; + options = value; + } + else { + attributes = {}; + attributes[ key ] = value; + } + + var result = Backbone.Model.prototype.set.apply( this, arguments ); + + // 'set' is called quite late in 'Backbone.Model.prototype.constructor', but before 'initialize'. + // Ideal place to set up relations :) + if ( !this._isInitialized && !this.isLocked() ) { + Backbone.Relational.store.register( this ); + this.initializeRelations(); + } + // Update the 'idAttribute' in Backbone.store if; we don't want it to miss an 'id' update due to {silent:true} + else if ( attributes && this.idAttribute in attributes ) { + Backbone.Relational.store.update( this ); + } + + if ( attributes ) { + this.updateRelations( options ); + } + + // Try to run the global queue holding external events + Backbone.Relational.eventQueue.unblock(); + + return result; + }, + + unset: function( attribute, options ) { + Backbone.Relational.eventQueue.block(); + + var result = Backbone.Model.prototype.unset.apply( this, arguments ); + this.updateRelations( options ); + + // Try to run the global queue holding external events + Backbone.Relational.eventQueue.unblock(); + + return result; + }, + + clear: function( options ) { + Backbone.Relational.eventQueue.block(); + + var result = Backbone.Model.prototype.clear.apply( this, arguments ); + this.updateRelations( options ); + + // Try to run the global queue holding external events + Backbone.Relational.eventQueue.unblock(); + + return result; + }, + + /** + * Override 'change', so the change will only execute after 'set' has finised (relations are updated), + * and 'previousAttributes' will be available when the event is fired. + */ + change: function( options ) { + var dit = this, args = arguments; + Backbone.Relational.eventQueue.add( function() { + Backbone.Model.prototype.change.apply( dit, args ); + }); + }, + + clone: function() { + var attributes = _.clone( this.attributes ); + if ( !_.isUndefined( attributes[ this.idAttribute ] ) ) { + attributes[ this.idAttribute ] = null; + } + + _.each( this.getRelations(), function( rel ) { + delete attributes[ rel.key ]; + }); + + return new this.constructor( attributes ); + }, + + /** + * Convert relations to JSON, omits them when required + */ + toJSON: function() { + // If this Model has already been fully serialized in this branch once, return to avoid loops + if ( this.isLocked() ) { + return this.id; + } + + this.acquire(); + var json = Backbone.Model.prototype.toJSON.call( this ); + + _.each( this._relations, function( rel ) { + var value = json[ rel.key ]; + + if ( rel.options.includeInJSON === true && value && _.isFunction( value.toJSON ) ) { + json[ rel.keyDestination ] = value.toJSON(); + } + else if ( _.isString( rel.options.includeInJSON ) ) { + if ( value instanceof Backbone.Collection ) { + json[ rel.keyDestination ] = value.pluck( rel.options.includeInJSON ); + } + else if ( value instanceof Backbone.Model ) { + json[ rel.keyDestination ] = value.get( rel.options.includeInJSON ); + } + } + else { + delete json[ rel.key ]; + } + + if ( rel.keyDestination !== rel.key ) { + delete json[ rel.key ]; + } + }, this ); + + this.release(); + return json; + } + }); + _.extend( Backbone.RelationalModel.prototype, Backbone.Semaphore ); + + /** + * Override Backbone.Collection.add, so objects fetched from the server multiple times will + * update the existing Model. Also, trigger 'relational:add'. + */ + var add = Backbone.Collection.prototype.__add = Backbone.Collection.prototype.add; + Backbone.Collection.prototype.add = function( models, options ) { + options || (options = {}); + if ( !_.isArray( models ) ) { + models = [ models ]; + } + + var modelsToAdd = []; + + //console.debug( 'calling add on coll=%o; model=%o, options=%o', this, models, options ); + _.each( models, function( model ) { + if ( !( model instanceof Backbone.Model ) ) { + // Try to find 'model' in Backbone.store. If it already exists, set the new properties on it. + var existingModel = Backbone.Relational.store.find( this.model, model[ this.model.prototype.idAttribute ] ); + if ( existingModel ) { + existingModel.set( existingModel.parse ? existingModel.parse( model ) : model, options ); + model = existingModel; + } + else { + model = Backbone.Collection.prototype._prepareModel.call( this, model, options ); + } + } + + if ( model instanceof Backbone.Model && !this.get( model ) && !this.getByCid( model ) ) { + modelsToAdd.push( model ); + } + }, this ); + + + // Add 'models' in a single batch, so the original add will only be called once (and thus 'sort', etc). + if ( modelsToAdd.length ) { + add.call( this, modelsToAdd, options ); + + _.each( modelsToAdd, function( model ) { + this.trigger('relational:add', model, this, options); + }, this ); + } + + return this; + }; + + /** + * Override 'Backbone.Collection.remove' to trigger 'relational:remove'. + */ + var remove = Backbone.Collection.prototype.__remove = Backbone.Collection.prototype.remove; + Backbone.Collection.prototype.remove = function( models, options ) { + options || (options = {}); + if (!_.isArray( models ) ) { + models = [ models ]; + } + + //console.debug('calling remove on coll=%o; models=%o, options=%o', this, models, options ); + _.each( models, function( model ) { + model = this.getByCid( model ) || this.get( model ); + + if ( model instanceof Backbone.Model ) { + remove.call( this, model, options ); + this.trigger('relational:remove', model, this, options); + } + }, this ); + + return this; + }; + + /** + * Override 'Backbone.Collection.reset' to trigger 'relational:reset'. + */ + var reset = Backbone.Collection.prototype.__reset = Backbone.Collection.prototype.reset; + Backbone.Collection.prototype.reset = function( models, options ) { + reset.call( this, models, options ); + this.trigger( 'relational:reset', models, options ); + + return this; + }; + + /** + * Override 'Backbone.Collection.trigger' so 'add', 'remove' and 'reset' events are queued until relations + * are ready. + */ + var trigger = Backbone.Collection.prototype.__trigger = Backbone.Collection.prototype.trigger; + Backbone.Collection.prototype.trigger = function( eventName ) { + if ( eventName === 'add' || eventName === 'remove' || eventName === 'reset' ) { + var dit = this, args = arguments; + Backbone.Relational.eventQueue.add( function() { + trigger.apply( dit, args ); + }); + } + else { + trigger.apply( this, arguments ); + } + + return this; + }; + + // Override .extend() to check for reverseRelations to initialize. + Backbone.RelationalModel.extend = function( protoProps, classProps ) { + var child = Backbone.Model.extend.apply( this, arguments ); + + var relations = ( protoProps && protoProps.relations ) || []; + _.each( relations, function( rel ) { + if( rel.reverseRelation ) { + rel.model = child; + + var preInitialize = true; + if ( _.isString( rel.relatedModel ) ) { + /** + * The related model might not be defined for two reasons + * 1. it never gets defined, e.g. a typo + * 2. it is related to itself + * In neither of these cases do we need to pre-initialize reverse relations. + */ + var relatedModel = Backbone.Relational.store.getObjectByName( rel.relatedModel ); + preInitialize = relatedModel && ( relatedModel.prototype instanceof Backbone.RelationalModel.prototype.constructor ); + } + + var type = !_.isString( rel.type ) ? rel.type : Backbone[ rel.type ] || Backbone.Relational.store.getObjectByName( rel.type ); + if ( preInitialize && type && type.prototype instanceof Backbone.Relation.prototype.constructor ) { + new type( null, rel ); + } + } + }); + + return child; + }; + +})(); diff --git a/static/scripts/libs/backbone.js b/static/scripts/libs/backbone.js new file mode 100644 index 00000000000..3373c952bfa --- /dev/null +++ b/static/scripts/libs/backbone.js @@ -0,0 +1,1431 @@ +// Backbone.js 0.9.2 + +// (c) 2010-2012 Jeremy Ashkenas, DocumentCloud Inc. +// Backbone may be freely distributed under the MIT license. +// For all details and documentation: +// http://backbonejs.org + +(function(){ + + // Initial Setup + // ------------- + + // Save a reference to the global object (`window` in the browser, `global` + // on the server). + var root = this; + + // Save the previous value of the `Backbone` variable, so that it can be + // restored later on, if `noConflict` is used. + var previousBackbone = root.Backbone; + + // Create a local reference to slice/splice. + var slice = Array.prototype.slice; + var splice = Array.prototype.splice; + + // The top-level namespace. All public Backbone classes and modules will + // be attached to this. Exported for both CommonJS and the browser. + var Backbone; + if (typeof exports !== 'undefined') { + Backbone = exports; + } else { + Backbone = root.Backbone = {}; + } + + // Current version of the library. Keep in sync with `package.json`. + Backbone.VERSION = '0.9.2'; + + // Require Underscore, if we're on the server, and it's not already present. + var _ = root._; + if (!_ && (typeof require !== 'undefined')) _ = require('underscore'); + + // For Backbone's purposes, jQuery, Zepto, or Ender owns the `$` variable. + var $ = root.jQuery || root.Zepto || root.ender; + + // Set the JavaScript library that will be used for DOM manipulation and + // Ajax calls (a.k.a. the `$` variable). By default Backbone will use: jQuery, + // Zepto, or Ender; but the `setDomLibrary()` method lets you inject an + // alternate JavaScript library (or a mock library for testing your views + // outside of a browser). + Backbone.setDomLibrary = function(lib) { + $ = lib; + }; + + // Runs Backbone.js in *noConflict* mode, returning the `Backbone` variable + // to its previous owner. Returns a reference to this Backbone object. + Backbone.noConflict = function() { + root.Backbone = previousBackbone; + return this; + }; + + // Turn on `emulateHTTP` to support legacy HTTP servers. Setting this option + // will fake `"PUT"` and `"DELETE"` requests via the `_method` parameter and + // set a `X-Http-Method-Override` header. + Backbone.emulateHTTP = false; + + // Turn on `emulateJSON` to support legacy servers that can't deal with direct + // `application/json` requests ... will encode the body as + // `application/x-www-form-urlencoded` instead and will send the model in a + // form param named `model`. + Backbone.emulateJSON = false; + + // Backbone.Events + // ----------------- + + // Regular expression used to split event strings + var eventSplitter = /\s+/; + + // A module that can be mixed in to *any object* in order to provide it with + // custom events. You may bind with `on` or remove with `off` callback functions + // to an event; trigger`-ing an event fires all callbacks in succession. + // + // var object = {}; + // _.extend(object, Backbone.Events); + // object.on('expand', function(){ alert('expanded'); }); + // object.trigger('expand'); + // + var Events = Backbone.Events = { + + // Bind one or more space separated events, `events`, to a `callback` + // function. Passing `"all"` will bind the callback to all events fired. + on: function(events, callback, context) { + + var calls, event, node, tail, list; + if (!callback) return this; + events = events.split(eventSplitter); + calls = this._callbacks || (this._callbacks = {}); + + // Create an immutable callback list, allowing traversal during + // modification. The tail is an empty object that will always be used + // as the next node. + while (event = events.shift()) { + list = calls[event]; + node = list ? list.tail : {}; + node.next = tail = {}; + node.context = context; + node.callback = callback; + calls[event] = {tail: tail, next: list ? list.next : node}; + } + + return this; + }, + + // Remove one or many callbacks. If `context` is null, removes all callbacks + // with that function. If `callback` is null, removes all callbacks for the + // event. If `events` is null, removes all bound callbacks for all events. + off: function(events, callback, context) { + var event, calls, node, tail, cb, ctx; + + // No events, or removing *all* events. + if (!(calls = this._callbacks)) return; + if (!(events || callback || context)) { + delete this._callbacks; + return this; + } + + // Loop through the listed events and contexts, splicing them out of the + // linked list of callbacks if appropriate. + events = events ? events.split(eventSplitter) : _.keys(calls); + while (event = events.shift()) { + node = calls[event]; + delete calls[event]; + if (!node || !(callback || context)) continue; + // Create a new list, omitting the indicated callbacks. + tail = node.tail; + while ((node = node.next) !== tail) { + cb = node.callback; + ctx = node.context; + if ((callback && cb !== callback) || (context && ctx !== context)) { + this.on(event, cb, ctx); + } + } + } + + return this; + }, + + // Trigger one or many events, firing all bound callbacks. Callbacks are + // passed the same arguments as `trigger` is, apart from the event name + // (unless you're listening on `"all"`, which will cause your callback to + // receive the true name of the event as the first argument). + trigger: function(events) { + var event, node, calls, tail, args, all, rest; + if (!(calls = this._callbacks)) return this; + all = calls.all; + events = events.split(eventSplitter); + rest = slice.call(arguments, 1); + + // For each event, walk through the linked list of callbacks twice, + // first to trigger the event, then to trigger any `"all"` callbacks. + while (event = events.shift()) { + if (node = calls[event]) { + tail = node.tail; + while ((node = node.next) !== tail) { + node.callback.apply(node.context || this, rest); + } + } + if (node = all) { + tail = node.tail; + args = [event].concat(rest); + while ((node = node.next) !== tail) { + node.callback.apply(node.context || this, args); + } + } + } + + return this; + } + + }; + + // Aliases for backwards compatibility. + Events.bind = Events.on; + Events.unbind = Events.off; + + // Backbone.Model + // -------------- + + // Create a new model, with defined attributes. A client id (`cid`) + // is automatically generated and assigned for you. + var Model = Backbone.Model = function(attributes, options) { + var defaults; + attributes || (attributes = {}); + if (options && options.parse) attributes = this.parse(attributes); + if (defaults = getValue(this, 'defaults')) { + attributes = _.extend({}, defaults, attributes); + } + if (options && options.collection) this.collection = options.collection; + this.attributes = {}; + this._escapedAttributes = {}; + this.cid = _.uniqueId('c'); + this.changed = {}; + this._silent = {}; + this._pending = {}; + this.set(attributes, {silent: true}); + // Reset change tracking. + this.changed = {}; + this._silent = {}; + this._pending = {}; + this._previousAttributes = _.clone(this.attributes); + this.initialize.apply(this, arguments); + }; + + // Attach all inheritable methods to the Model prototype. + _.extend(Model.prototype, Events, { + + // A hash of attributes whose current and previous value differ. + changed: null, + + // A hash of attributes that have silently changed since the last time + // `change` was called. Will become pending attributes on the next call. + _silent: null, + + // A hash of attributes that have changed since the last `'change'` event + // began. + _pending: null, + + // The default name for the JSON `id` attribute is `"id"`. MongoDB and + // CouchDB users may want to set this to `"_id"`. + idAttribute: 'id', + + // Initialize is an empty function by default. Override it with your own + // initialization logic. + initialize: function(){}, + + // Return a copy of the model's `attributes` object. + toJSON: function(options) { + return _.clone(this.attributes); + }, + + // Get the value of an attribute. + get: function(attr) { + return this.attributes[attr]; + }, + + // Get the HTML-escaped value of an attribute. + escape: function(attr) { + var html; + if (html = this._escapedAttributes[attr]) return html; + var val = this.get(attr); + return this._escapedAttributes[attr] = _.escape(val == null ? '' : '' + val); + }, + + // Returns `true` if the attribute contains a value that is not null + // or undefined. + has: function(attr) { + return this.get(attr) != null; + }, + + // Set a hash of model attributes on the object, firing `"change"` unless + // you choose to silence it. + set: function(key, value, options) { + var attrs, attr, val; + + // Handle both `"key", value` and `{key: value}` -style arguments. + if (_.isObject(key) || key == null) { + attrs = key; + options = value; + } else { + attrs = {}; + attrs[key] = value; + } + + // Extract attributes and options. + options || (options = {}); + if (!attrs) return this; + if (attrs instanceof Model) attrs = attrs.attributes; + if (options.unset) for (attr in attrs) attrs[attr] = void 0; + + // Run validation. + if (!this._validate(attrs, options)) return false; + + // Check for changes of `id`. + if (this.idAttribute in attrs) this.id = attrs[this.idAttribute]; + + var changes = options.changes = {}; + var now = this.attributes; + var escaped = this._escapedAttributes; + var prev = this._previousAttributes || {}; + + // For each `set` attribute... + for (attr in attrs) { + val = attrs[attr]; + + // If the new and current value differ, record the change. + if (!_.isEqual(now[attr], val) || (options.unset && _.has(now, attr))) { + delete escaped[attr]; + (options.silent ? this._silent : changes)[attr] = true; + } + + // Update or delete the current value. + options.unset ? delete now[attr] : now[attr] = val; + + // If the new and previous value differ, record the change. If not, + // then remove changes for this attribute. + if (!_.isEqual(prev[attr], val) || (_.has(now, attr) != _.has(prev, attr))) { + this.changed[attr] = val; + if (!options.silent) this._pending[attr] = true; + } else { + delete this.changed[attr]; + delete this._pending[attr]; + } + } + + // Fire the `"change"` events. + if (!options.silent) this.change(options); + return this; + }, + + // Remove an attribute from the model, firing `"change"` unless you choose + // to silence it. `unset` is a noop if the attribute doesn't exist. + unset: function(attr, options) { + (options || (options = {})).unset = true; + return this.set(attr, null, options); + }, + + // Clear all attributes on the model, firing `"change"` unless you choose + // to silence it. + clear: function(options) { + (options || (options = {})).unset = true; + return this.set(_.clone(this.attributes), options); + }, + + // Fetch the model from the server. If the server's representation of the + // model differs from its current attributes, they will be overriden, + // triggering a `"change"` event. + fetch: function(options) { + options = options ? _.clone(options) : {}; + var model = this; + var success = options.success; + options.success = function(resp, status, xhr) { + if (!model.set(model.parse(resp, xhr), options)) return false; + if (success) success(model, resp); + }; + options.error = Backbone.wrapError(options.error, model, options); + return (this.sync || Backbone.sync).call(this, 'read', this, options); + }, + + // Set a hash of model attributes, and sync the model to the server. + // If the server returns an attributes hash that differs, the model's + // state will be `set` again. + save: function(key, value, options) { + var attrs, current; + + // Handle both `("key", value)` and `({key: value})` -style calls. + if (_.isObject(key) || key == null) { + attrs = key; + options = value; + } else { + attrs = {}; + attrs[key] = value; + } + options = options ? _.clone(options) : {}; + + // If we're "wait"-ing to set changed attributes, validate early. + if (options.wait) { + if (!this._validate(attrs, options)) return false; + current = _.clone(this.attributes); + } + + // Regular saves `set` attributes before persisting to the server. + var silentOptions = _.extend({}, options, {silent: true}); + if (attrs && !this.set(attrs, options.wait ? silentOptions : options)) { + return false; + } + + // After a successful server-side save, the client is (optionally) + // updated with the server-side state. + var model = this; + var success = options.success; + options.success = function(resp, status, xhr) { + var serverAttrs = model.parse(resp, xhr); + if (options.wait) { + delete options.wait; + serverAttrs = _.extend(attrs || {}, serverAttrs); + } + if (!model.set(serverAttrs, options)) return false; + if (success) { + success(model, resp); + } else { + model.trigger('sync', model, resp, options); + } + }; + + // Finish configuring and sending the Ajax request. + options.error = Backbone.wrapError(options.error, model, options); + var method = this.isNew() ? 'create' : 'update'; + var xhr = (this.sync || Backbone.sync).call(this, method, this, options); + if (options.wait) this.set(current, silentOptions); + return xhr; + }, + + // Destroy this model on the server if it was already persisted. + // Optimistically removes the model from its collection, if it has one. + // If `wait: true` is passed, waits for the server to respond before removal. + destroy: function(options) { + options = options ? _.clone(options) : {}; + var model = this; + var success = options.success; + + var triggerDestroy = function() { + model.trigger('destroy', model, model.collection, options); + }; + + if (this.isNew()) { + triggerDestroy(); + return false; + } + + options.success = function(resp) { + if (options.wait) triggerDestroy(); + if (success) { + success(model, resp); + } else { + model.trigger('sync', model, resp, options); + } + }; + + options.error = Backbone.wrapError(options.error, model, options); + var xhr = (this.sync || Backbone.sync).call(this, 'delete', this, options); + if (!options.wait) triggerDestroy(); + return xhr; + }, + + // Default URL for the model's representation on the server -- if you're + // using Backbone's restful methods, override this to change the endpoint + // that will be called. + url: function() { + var base = getValue(this, 'urlRoot') || getValue(this.collection, 'url') || urlError(); + if (this.isNew()) return base; + return base + (base.charAt(base.length - 1) == '/' ? '' : '/') + encodeURIComponent(this.id); + }, + + // **parse** converts a response into the hash of attributes to be `set` on + // the model. The default implementation is just to pass the response along. + parse: function(resp, xhr) { + return resp; + }, + + // Create a new model with identical attributes to this one. + clone: function() { + return new this.constructor(this.attributes); + }, + + // A model is new if it has never been saved to the server, and lacks an id. + isNew: function() { + return this.id == null; + }, + + // Call this method to manually fire a `"change"` event for this model and + // a `"change:attribute"` event for each changed attribute. + // Calling this will cause all objects observing the model to update. + change: function(options) { + options || (options = {}); + var changing = this._changing; + this._changing = true; + + // Silent changes become pending changes. + for (var attr in this._silent) this._pending[attr] = true; + + // Silent changes are triggered. + var changes = _.extend({}, options.changes, this._silent); + this._silent = {}; + for (var attr in changes) { + this.trigger('change:' + attr, this, this.get(attr), options); + } + if (changing) return this; + + // Continue firing `"change"` events while there are pending changes. + while (!_.isEmpty(this._pending)) { + this._pending = {}; + this.trigger('change', this, options); + // Pending and silent changes still remain. + for (var attr in this.changed) { + if (this._pending[attr] || this._silent[attr]) continue; + delete this.changed[attr]; + } + this._previousAttributes = _.clone(this.attributes); + } + + this._changing = false; + return this; + }, + + // Determine if the model has changed since the last `"change"` event. + // If you specify an attribute name, determine if that attribute has changed. + hasChanged: function(attr) { + if (!arguments.length) return !_.isEmpty(this.changed); + return _.has(this.changed, attr); + }, + + // Return an object containing all the attributes that have changed, or + // false if there are no changed attributes. Useful for determining what + // parts of a view need to be updated and/or what attributes need to be + // persisted to the server. Unset attributes will be set to undefined. + // You can also pass an attributes object to diff against the model, + // determining if there *would be* a change. + changedAttributes: function(diff) { + if (!diff) return this.hasChanged() ? _.clone(this.changed) : false; + var val, changed = false, old = this._previousAttributes; + for (var attr in diff) { + if (_.isEqual(old[attr], (val = diff[attr]))) continue; + (changed || (changed = {}))[attr] = val; + } + return changed; + }, + + // Get the previous value of an attribute, recorded at the time the last + // `"change"` event was fired. + previous: function(attr) { + if (!arguments.length || !this._previousAttributes) return null; + return this._previousAttributes[attr]; + }, + + // Get all of the attributes of the model at the time of the previous + // `"change"` event. + previousAttributes: function() { + return _.clone(this._previousAttributes); + }, + + // Check if the model is currently in a valid state. It's only possible to + // get into an *invalid* state if you're using silent changes. + isValid: function() { + return !this.validate(this.attributes); + }, + + // Run validation against the next complete set of model attributes, + // returning `true` if all is well. If a specific `error` callback has + // been passed, call that instead of firing the general `"error"` event. + _validate: function(attrs, options) { + if (options.silent || !this.validate) return true; + attrs = _.extend({}, this.attributes, attrs); + var error = this.validate(attrs, options); + if (!error) return true; + if (options && options.error) { + options.error(this, error, options); + } else { + this.trigger('error', this, error, options); + } + return false; + } + + }); + + // Backbone.Collection + // ------------------- + + // Provides a standard collection class for our sets of models, ordered + // or unordered. If a `comparator` is specified, the Collection will maintain + // its models in sort order, as they're added and removed. + var Collection = Backbone.Collection = function(models, options) { + options || (options = {}); + if (options.model) this.model = options.model; + if (options.comparator) this.comparator = options.comparator; + this._reset(); + this.initialize.apply(this, arguments); + if (models) this.reset(models, {silent: true, parse: options.parse}); + }; + + // Define the Collection's inheritable methods. + _.extend(Collection.prototype, Events, { + + // The default model for a collection is just a **Backbone.Model**. + // This should be overridden in most cases. + model: Model, + + // Initialize is an empty function by default. Override it with your own + // initialization logic. + initialize: function(){}, + + // The JSON representation of a Collection is an array of the + // models' attributes. + toJSON: function(options) { + return this.map(function(model){ return model.toJSON(options); }); + }, + + // Add a model, or list of models to the set. Pass **silent** to avoid + // firing the `add` event for every new model. + add: function(models, options) { + var i, index, length, model, cid, id, cids = {}, ids = {}, dups = []; + options || (options = {}); + models = _.isArray(models) ? models.slice() : [models]; + + // Begin by turning bare objects into model references, and preventing + // invalid models or duplicate models from being added. + for (i = 0, length = models.length; i < length; i++) { + if (!(model = models[i] = this._prepareModel(models[i], options))) { + throw new Error("Can't add an invalid model to a collection"); + } + cid = model.cid; + id = model.id; + if (cids[cid] || this._byCid[cid] || ((id != null) && (ids[id] || this._byId[id]))) { + dups.push(i); + continue; + } + cids[cid] = ids[id] = model; + } + + // Remove duplicates. + i = dups.length; + while (i--) { + models.splice(dups[i], 1); + } + + // Listen to added models' events, and index models for lookup by + // `id` and by `cid`. + for (i = 0, length = models.length; i < length; i++) { + (model = models[i]).on('all', this._onModelEvent, this); + this._byCid[model.cid] = model; + if (model.id != null) this._byId[model.id] = model; + } + + // Insert models into the collection, re-sorting if needed, and triggering + // `add` events unless silenced. + this.length += length; + index = options.at != null ? options.at : this.models.length; + splice.apply(this.models, [index, 0].concat(models)); + if (this.comparator) this.sort({silent: true}); + if (options.silent) return this; + for (i = 0, length = this.models.length; i < length; i++) { + if (!cids[(model = this.models[i]).cid]) continue; + options.index = i; + model.trigger('add', model, this, options); + } + return this; + }, + + // Remove a model, or a list of models from the set. Pass silent to avoid + // firing the `remove` event for every model removed. + remove: function(models, options) { + var i, l, index, model; + options || (options = {}); + models = _.isArray(models) ? models.slice() : [models]; + for (i = 0, l = models.length; i < l; i++) { + model = this.getByCid(models[i]) || this.get(models[i]); + if (!model) continue; + delete this._byId[model.id]; + delete this._byCid[model.cid]; + index = this.indexOf(model); + this.models.splice(index, 1); + this.length--; + if (!options.silent) { + options.index = index; + model.trigger('remove', model, this, options); + } + this._removeReference(model); + } + return this; + }, + + // Add a model to the end of the collection. + push: function(model, options) { + model = this._prepareModel(model, options); + this.add(model, options); + return model; + }, + + // Remove a model from the end of the collection. + pop: function(options) { + var model = this.at(this.length - 1); + this.remove(model, options); + return model; + }, + + // Add a model to the beginning of the collection. + unshift: function(model, options) { + model = this._prepareModel(model, options); + this.add(model, _.extend({at: 0}, options)); + return model; + }, + + // Remove a model from the beginning of the collection. + shift: function(options) { + var model = this.at(0); + this.remove(model, options); + return model; + }, + + // Get a model from the set by id. + get: function(id) { + if (id == null) return void 0; + return this._byId[id.id != null ? id.id : id]; + }, + + // Get a model from the set by client id. + getByCid: function(cid) { + return cid && this._byCid[cid.cid || cid]; + }, + + // Get the model at the given index. + at: function(index) { + return this.models[index]; + }, + + // Return models with matching attributes. Useful for simple cases of `filter`. + where: function(attrs) { + if (_.isEmpty(attrs)) return []; + return this.filter(function(model) { + for (var key in attrs) { + if (attrs[key] !== model.get(key)) return false; + } + return true; + }); + }, + + // Force the collection to re-sort itself. You don't need to call this under + // normal circumstances, as the set will maintain sort order as each item + // is added. + sort: function(options) { + options || (options = {}); + if (!this.comparator) throw new Error('Cannot sort a set without a comparator'); + var boundComparator = _.bind(this.comparator, this); + if (this.comparator.length == 1) { + this.models = this.sortBy(boundComparator); + } else { + this.models.sort(boundComparator); + } + if (!options.silent) this.trigger('reset', this, options); + return this; + }, + + // Pluck an attribute from each model in the collection. + pluck: function(attr) { + return _.map(this.models, function(model){ return model.get(attr); }); + }, + + // When you have more items than you want to add or remove individually, + // you can reset the entire set with a new list of models, without firing + // any `add` or `remove` events. Fires `reset` when finished. + reset: function(models, options) { + models || (models = []); + options || (options = {}); + for (var i = 0, l = this.models.length; i < l; i++) { + this._removeReference(this.models[i]); + } + this._reset(); + this.add(models, _.extend({silent: true}, options)); + if (!options.silent) this.trigger('reset', this, options); + return this; + }, + + // Fetch the default set of models for this collection, resetting the + // collection when they arrive. If `add: true` is passed, appends the + // models to the collection instead of resetting. + fetch: function(options) { + options = options ? _.clone(options) : {}; + if (options.parse === undefined) options.parse = true; + var collection = this; + var success = options.success; + options.success = function(resp, status, xhr) { + collection[options.add ? 'add' : 'reset'](collection.parse(resp, xhr), options); + if (success) success(collection, resp); + }; + options.error = Backbone.wrapError(options.error, collection, options); + return (this.sync || Backbone.sync).call(this, 'read', this, options); + }, + + // Create a new instance of a model in this collection. Add the model to the + // collection immediately, unless `wait: true` is passed, in which case we + // wait for the server to agree. + create: function(model, options) { + var coll = this; + options = options ? _.clone(options) : {}; + model = this._prepareModel(model, options); + if (!model) return false; + if (!options.wait) coll.add(model, options); + var success = options.success; + options.success = function(nextModel, resp, xhr) { + if (options.wait) coll.add(nextModel, options); + if (success) { + success(nextModel, resp); + } else { + nextModel.trigger('sync', model, resp, options); + } + }; + model.save(null, options); + return model; + }, + + // **parse** converts a response into a list of models to be added to the + // collection. The default implementation is just to pass it through. + parse: function(resp, xhr) { + return resp; + }, + + // Proxy to _'s chain. Can't be proxied the same way the rest of the + // underscore methods are proxied because it relies on the underscore + // constructor. + chain: function () { + return _(this.models).chain(); + }, + + // Reset all internal state. Called when the collection is reset. + _reset: function(options) { + this.length = 0; + this.models = []; + this._byId = {}; + this._byCid = {}; + }, + + // Prepare a model or hash of attributes to be added to this collection. + _prepareModel: function(model, options) { + options || (options = {}); + if (!(model instanceof Model)) { + var attrs = model; + options.collection = this; + model = new this.model(attrs, options); + if (!model._validate(model.attributes, options)) model = false; + } else if (!model.collection) { + model.collection = this; + } + return model; + }, + + // Internal method to remove a model's ties to a collection. + _removeReference: function(model) { + if (this == model.collection) { + delete model.collection; + } + model.off('all', this._onModelEvent, this); + }, + + // Internal method called every time a model in the set fires an event. + // Sets need to update their indexes when models change ids. All other + // events simply proxy through. "add" and "remove" events that originate + // in other collections are ignored. + _onModelEvent: function(event, model, collection, options) { + if ((event == 'add' || event == 'remove') && collection != this) return; + if (event == 'destroy') { + this.remove(model, options); + } + if (model && event === 'change:' + model.idAttribute) { + delete this._byId[model.previous(model.idAttribute)]; + this._byId[model.id] = model; + } + this.trigger.apply(this, arguments); + } + + }); + + // Underscore methods that we want to implement on the Collection. + var methods = ['forEach', 'each', 'map', 'reduce', 'reduceRight', 'find', + 'detect', 'filter', 'select', 'reject', 'every', 'all', 'some', 'any', + 'include', 'contains', 'invoke', 'max', 'min', 'sortBy', 'sortedIndex', + 'toArray', 'size', 'first', 'initial', 'rest', 'last', 'without', 'indexOf', + 'shuffle', 'lastIndexOf', 'isEmpty', 'groupBy']; + + // Mix in each Underscore method as a proxy to `Collection#models`. + _.each(methods, function(method) { + Collection.prototype[method] = function() { + return _[method].apply(_, [this.models].concat(_.toArray(arguments))); + }; + }); + + // Backbone.Router + // ------------------- + + // Routers map faux-URLs to actions, and fire events when routes are + // matched. Creating a new one sets its `routes` hash, if not set statically. + var Router = Backbone.Router = function(options) { + options || (options = {}); + if (options.routes) this.routes = options.routes; + this._bindRoutes(); + this.initialize.apply(this, arguments); + }; + + // Cached regular expressions for matching named param parts and splatted + // parts of route strings. + var namedParam = /:\w+/g; + var splatParam = /\*\w+/g; + var escapeRegExp = /[-[\]{}()+?.,\\^$|#\s]/g; + + // Set up all inheritable **Backbone.Router** properties and methods. + _.extend(Router.prototype, Events, { + + // Initialize is an empty function by default. Override it with your own + // initialization logic. + initialize: function(){}, + + // Manually bind a single named route to a callback. For example: + // + // this.route('search/:query/p:num', 'search', function(query, num) { + // ... + // }); + // + route: function(route, name, callback) { + Backbone.history || (Backbone.history = new History); + if (!_.isRegExp(route)) route = this._routeToRegExp(route); + if (!callback) callback = this[name]; + Backbone.history.route(route, _.bind(function(fragment) { + var args = this._extractParameters(route, fragment); + callback && callback.apply(this, args); + this.trigger.apply(this, ['route:' + name].concat(args)); + Backbone.history.trigger('route', this, name, args); + }, this)); + return this; + }, + + // Simple proxy to `Backbone.history` to save a fragment into the history. + navigate: function(fragment, options) { + Backbone.history.navigate(fragment, options); + }, + + // Bind all defined routes to `Backbone.history`. We have to reverse the + // order of the routes here to support behavior where the most general + // routes can be defined at the bottom of the route map. + _bindRoutes: function() { + if (!this.routes) return; + var routes = []; + for (var route in this.routes) { + routes.unshift([route, this.routes[route]]); + } + for (var i = 0, l = routes.length; i < l; i++) { + this.route(routes[i][0], routes[i][1], this[routes[i][1]]); + } + }, + + // Convert a route string into a regular expression, suitable for matching + // against the current location hash. + _routeToRegExp: function(route) { + route = route.replace(escapeRegExp, '\\$&') + .replace(namedParam, '([^\/]+)') + .replace(splatParam, '(.*?)'); + return new RegExp('^' + route + '$'); + }, + + // Given a route, and a URL fragment that it matches, return the array of + // extracted parameters. + _extractParameters: function(route, fragment) { + return route.exec(fragment).slice(1); + } + + }); + + // Backbone.History + // ---------------- + + // Handles cross-browser history management, based on URL fragments. If the + // browser does not support `onhashchange`, falls back to polling. + var History = Backbone.History = function() { + this.handlers = []; + _.bindAll(this, 'checkUrl'); + }; + + // Cached regex for cleaning leading hashes and slashes . + var routeStripper = /^[#\/]/; + + // Cached regex for detecting MSIE. + var isExplorer = /msie [\w.]+/; + + // Has the history handling already been started? + History.started = false; + + // Set up all inheritable **Backbone.History** properties and methods. + _.extend(History.prototype, Events, { + + // The default interval to poll for hash changes, if necessary, is + // twenty times a second. + interval: 50, + + // Gets the true hash value. Cannot use location.hash directly due to bug + // in Firefox where location.hash will always be decoded. + getHash: function(windowOverride) { + var loc = windowOverride ? windowOverride.location : window.location; + var match = loc.href.match(/#(.*)$/); + return match ? match[1] : ''; + }, + + // Get the cross-browser normalized URL fragment, either from the URL, + // the hash, or the override. + getFragment: function(fragment, forcePushState) { + if (fragment == null) { + if (this._hasPushState || forcePushState) { + fragment = window.location.pathname; + var search = window.location.search; + if (search) fragment += search; + } else { + fragment = this.getHash(); + } + } + if (!fragment.indexOf(this.options.root)) fragment = fragment.substr(this.options.root.length); + return fragment.replace(routeStripper, ''); + }, + + // Start the hash change handling, returning `true` if the current URL matches + // an existing route, and `false` otherwise. + start: function(options) { + if (History.started) throw new Error("Backbone.history has already been started"); + History.started = true; + + // Figure out the initial configuration. Do we need an iframe? + // Is pushState desired ... is it available? + this.options = _.extend({}, {root: '/'}, this.options, options); + this._wantsHashChange = this.options.hashChange !== false; + this._wantsPushState = !!this.options.pushState; + this._hasPushState = !!(this.options.pushState && window.history && window.history.pushState); + var fragment = this.getFragment(); + var docMode = document.documentMode; + var oldIE = (isExplorer.exec(navigator.userAgent.toLowerCase()) && (!docMode || docMode <= 7)); + + if (oldIE) { + this.iframe = $('"));d("#close_button").bind("click",function(){a()})}function i(n,o){if(n){d(".loggedin-only").show();d(".loggedout-only").hide();d("#user-email").text(n);if(o){d(".admin-only").show()}}else{d(".loggedin-only").hide();d(".loggedout-only").show();d(".admin-only").hide()}}d(function(){var n=d("#masthead ul.nav > li.dropdown > .dropdown-menu");d("body").on("click.nav_popups",function(p){n.hide();d("#DD-helper").hide();if(d(p.target).closest("#masthead ul.nav > li.dropdown > .dropdown-menu").length){return}var o=d(p.target).closest("#masthead ul.nav > li.dropdown");if(o.length){d("#DD-helper").show();o.children(".dropdown-menu").show();p.preventDefault()}})});c.ensure_dd_helper=e;c.Panel=k;c.Modal=f;c.hide_modal=a;c.show_modal=l;c.show_message=g;c.show_in_overlay=j;c.user_changed=i}(window,window.jQuery); \ No newline at end of file +!function(c,d){var e=function(){if(d("#DD-helper").length==0){d("
    ").css({background:"white",opacity:0,zIndex:9000,position:"absolute",top:0,left:0,width:"100%",height:"100%"}).appendTo("body").hide()}};var b=150,h=800;var k=function(n){this.$panel=n.panel;this.$center=n.center;this.$drag=n.drag;this.$toggle=n.toggle;this.left=!n.right;this.hidden=false;this.hidden_by_tool=false;this.saved_size=null;this.init()};d.extend(k.prototype,{resize:function(n){this.$panel.css("width",n);if(this.left){this.$center.css("left",n)}else{this.$center.css("right",n)}if(document.recalc){document.recalc()}},do_toggle:function(){var n=this;if(this.hidden){this.$toggle.removeClass("hidden");if(this.left){this.$panel.css("left",-this.saved_size).show().animate({left:0},"fast",function(){n.resize(n.saved_size)})}else{this.$panel.css("right",-this.saved_size).show().animate({right:0},"fast",function(){n.resize(n.saved_size)})}n.hidden=false}else{n.saved_size=this.$panel.width();if(document.recalc){document.recalc()}if(this.left){this.$panel.animate({left:-this.saved_size},"fast")}else{this.$panel.animate({right:-this.saved_size},"fast")}if(this.left){this.$center.css("left",0)}else{this.$center.css("right",0)}n.hidden=true;n.$toggle.addClass("hidden")}this.hidden_by_tool=false},handle_minwidth_hint:function(n){var o=this.$center.width()-(this.hidden?this.saved_size:0);if(o ").text(r).click(s)).append(" ")});this.$footer.show()}var q=this.$footer.find(".extra_buttons").html("");if(p.extra_buttons){d.each(p.extra_buttons,function(r,s){q.append(d("").text(r).click(s)).append(" ")});this.$footer.show()}var n=p.body;if(n=="progress"){n=d("
    ")}this.$body.html(n)},show:function(n,o){if(!this.$dialog.is(":visible")){if(n.backdrop){this.$backdrop.addClass("in")}else{this.$backdrop.removeClass("in")}this.$overlay.show();this.$dialog.show();this.$body.css("min-width",this.$body.width())}if(o){o()}},hide:function(){var n=this;n.$dialog.fadeOut(function(){n.$overlay.hide();n.$backdrop.removeClass("in");n.$body.children().remove();n.$body.css("min-width",undefined)})}});var m;d(function(){m=new f({overlay:d("#overlay"),dialog:d("#dialog-box"),backdrop:d("#overlay-background")})});function a(){m.hide()}function l(r,n,p,o,q){m.setContent({title:r,body:n,buttons:p,extra_buttons:o});m.show({backdrop:true},q)}function g(r,n,p,o,q){m.setContent({title:r,body:n,buttons:p,extra_buttons:o});m.show({backdrop:false},q)}function j(p){var q=p.width||"600";var o=p.height||"400";var n=p.scroll||"auto";d("#overlay-background").bind("click.overlay",function(){a();d("#overlay-background").unbind("click.overlay")});l(null,d("
    "));d("#close_button").bind("click",function(){a()})}function i(n,o){if(n){d(".loggedin-only").show();d(".loggedout-only").hide();d("#user-email").text(n);if(o){d(".admin-only").show()}}else{d(".loggedin-only").hide();d(".loggedout-only").show();d(".admin-only").hide()}}d(function(){var n=d("#masthead ul.nav > li.dropdown > .dropdown-menu");d("body").on("click.nav_popups",function(p){n.hide();d("#DD-helper").hide();if(d(p.target).closest("#masthead ul.nav > li.dropdown > .dropdown-menu").length){return}var o=d(p.target).closest("#masthead ul.nav > li.dropdown");if(o.length){d("#DD-helper").show();o.children(".dropdown-menu").show();p.preventDefault()}})});c.ensure_dd_helper=e;c.Panel=k;c.Modal=f;c.hide_modal=a;c.show_modal=l;c.show_message=g;c.show_in_overlay=j;c.user_changed=i}(window,window.jQuery); \ No newline at end of file diff --git a/static/scripts/packed/galaxy.workflow_editor.canvas.js b/static/scripts/packed/galaxy.workflow_editor.canvas.js index f5fd235a17e..037e7fcec5b 100644 --- a/static/scripts/packed/galaxy.workflow_editor.canvas.js +++ b/static/scripts/packed/galaxy.workflow_editor.canvas.js @@ -1 +1 @@ -function Terminal(a){this.element=a;this.connectors=[]}$.extend(Terminal.prototype,{connect:function(a){this.connectors.push(a);if(this.node){this.node.changed()}},disconnect:function(a){this.connectors.splice($.inArray(a,this.connectors),1);if(this.node){this.node.changed()}},redraw:function(){$.each(this.connectors,function(a,b){b.redraw()})},destroy:function(){$.each(this.connectors.slice(),function(a,b){b.destroy()})}});function OutputTerminal(a,b){Terminal.call(this,a);this.datatypes=b}OutputTerminal.prototype=new Terminal();function InputTerminal(a,b){Terminal.call(this,a);this.datatypes=b}InputTerminal.prototype=new Terminal();$.extend(InputTerminal.prototype,{can_accept:function(a){if(this.connectors.length<1){for(var c in this.datatypes){var f=new Array();f=f.concat(a.datatypes);if(a.node.post_job_actions){for(var d in a.node.post_job_actions){var g=a.node.post_job_actions[d];if(g.action_type=="ChangeDatatypeAction"&&(g.output_name==""||g.output_name==a.name)&&g.action_arguments){f.push(g.action_arguments.newtype)}}}for(var b in f){if(f[b]=="input"||issubtype(f[b],this.datatypes[c])){return true}}}}return false}});function Connector(b,a){this.canvas=null;this.dragging=false;this.inner_color="#FFFFFF";this.outer_color="#D8B365";if(b&&a){this.connect(b,a)}}$.extend(Connector.prototype,{connect:function(b,a){this.handle1=b;this.handle1.connect(this);this.handle2=a;this.handle2.connect(this)},destroy:function(){if(this.handle1){this.handle1.disconnect(this)}if(this.handle2){this.handle2.disconnect(this)}$(this.canvas).remove()},redraw:function(){var d=$("#canvas-container");if(!this.canvas){this.canvas=document.createElement("canvas");if(window.G_vmlCanvasManager){G_vmlCanvasManager.initElement(this.canvas)}d.append($(this.canvas));if(this.dragging){this.canvas.style.zIndex="300"}}var n=function(c){return $(c).offset().left-d.offset().left};var i=function(c){return $(c).offset().top-d.offset().top};var h=n(this.handle1.element)+5;var g=i(this.handle1.element)+5;var p=n(this.handle2.element)+5;var m=i(this.handle2.element)+5;var f=100;var k=Math.min(h,p);var a=Math.max(h,p);var j=Math.min(g,m);var t=Math.max(g,m);var b=Math.min(Math.max(Math.abs(t-j)/2,100),300);var o=k-f;var s=j-f;var q=a-k+2*f;var l=t-j+2*f;this.canvas.style.left=o+"px";this.canvas.style.top=s+"px";this.canvas.setAttribute("width",q);this.canvas.setAttribute("height",l);h-=o;g-=s;p-=o;m-=s;var r=this.canvas.getContext("2d");r.lineCap="round";r.strokeStyle=this.outer_color;r.lineWidth=7;r.beginPath();r.moveTo(h,g);r.bezierCurveTo(h+b,g,p-b,m,p,m);r.stroke();r.strokeStyle=this.inner_color;r.lineWidth=5;r.beginPath();r.moveTo(h,g);r.bezierCurveTo(h+b,g,p-b,m,p,m);r.stroke()}});function Node(a){this.element=a;this.input_terminals={};this.output_terminals={};this.tool_errors={}}$.extend(Node.prototype,{enable_input_terminal:function(d,a,b){var c=this;$(d).each(function(){var f=this.terminal=new InputTerminal(this,b);f.node=c;f.name=a;$(this).bind("dropinit",function(g,h){return $(h.drag).hasClass("output-terminal")&&f.can_accept(h.drag.terminal)}).bind("dropstart",function(g,h){h.proxy.terminal.connectors[0].inner_color="#BBFFBB"}).bind("dropend",function(g,h){h.proxy.terminal.connectors[0].inner_color="#FFFFFF"}).bind("drop",function(g,h){(new Connector(h.drag.terminal,f)).redraw()}).bind("hover",function(){if(f.connectors.length>0){var g=$("
    ").css({display:"none"}).appendTo("body").append($("
    ").append($("").attr("src",image_path+"/delete_icon.png").click(function(){$.each(f.connectors,function(i,h){h.destroy()});g.remove()}))).bind("mouseleave",function(){$(this).remove()});g.css({top:$(this).offset().top-2,left:$(this).offset().left-g.width(),"padding-right":$(this).width()}).show()}});c.input_terminals[a]=f})},enable_output_terminal:function(d,a,b){var c=this;$(d).each(function(){var g=this;var f=this.terminal=new OutputTerminal(this,b);f.node=c;f.name=a;$(this).bind("dragstart",function(j,k){$(k.available).addClass("input-terminal-active");workflow.check_changes_in_active_form();var i=$('
    ').appendTo("#canvas-container").get(0);i.terminal=new OutputTerminal(i);var l=new Connector();l.dragging=true;l.connect(this.terminal,i.terminal);return i}).bind("drag",function(i,j){var h=function(){var l=$(j.proxy).offsetParent().offset(),k=j.offsetX-l.left,m=j.offsetY-l.top;$(j.proxy).css({left:k,top:m});j.proxy.terminal.redraw();canvas_manager.update_viewport_overlay()};h();$("#canvas-container").get(0).scroll_panel.test(i,h)}).bind("dragend",function(h,i){i.proxy.terminal.connectors[0].destroy();$(i.proxy).remove();$(i.available).removeClass("input-terminal-active");$("#canvas-container").get(0).scroll_panel.stop()});c.output_terminals[a]=f})},redraw:function(){$.each(this.input_terminals,function(a,b){b.redraw()});$.each(this.output_terminals,function(a,b){b.redraw()})},destroy:function(){$.each(this.input_terminals,function(a,b){b.destroy()});$.each(this.output_terminals,function(a,b){b.destroy()});workflow.remove_node(this);$(this.element).remove()},make_active:function(){$(this.element).addClass("toolForm-active")},make_inactive:function(){var a=this.element.get(0);(function(b){b.removeChild(a);b.appendChild(a)})(a.parentNode);$(a).removeClass("toolForm-active")},init_field_data:function(h){var g=this.element;if(h.type){this.type=h.type}this.name=h.name;this.form_html=h.form_html;this.tool_state=h.tool_state;this.tool_errors=h.tool_errors;this.tooltip=h.tooltip?h.tooltip:"";this.annotation=h.annotation;this.post_job_actions=h.post_job_actions?h.post_job_actions:{};this.workflow_outputs=h.workflow_outputs?h.workflow_outputs:[];if(this.tool_errors){g.addClass("tool-node-error")}else{g.removeClass("tool-node-error")}var d=this;var c=Math.max(150,g.width());var a=g.find(".toolFormBody");a.find("div").remove();var i=$("
    ").appendTo(a);$.each(h.data_inputs,function(k,f){var j=$("
    ");d.enable_input_terminal(j,f.name,f.extensions);var b=$("
    "+f.label+"
    ");b.css({position:"absolute",left:-1000,top:-1000,display:"none"});$("body").append(b);c=Math.max(c,b.outerWidth());b.css({position:"",left:"",top:"",display:""});b.remove();i.append(b.prepend(j))});if((h.data_inputs.length>0)&&(h.data_outputs.length>0)){a.append($("
    "))}$.each(h.data_outputs,function(k,b){var j=$("
    ");d.enable_output_terminal(j,b.name,b.extensions);var f=b.name;if(b.extensions.indexOf("input")<0){f=f+" ("+b.extensions.join(", ")+")"}var m=$("
    "+f+"
    ");if(d.type=="tool"){var l=$("
    ").css({display:"none"}).append($("
    ").append($("").attr("src",image_path+"/fugue/asterisk-small-outline.png").click(function(){if($.inArray(b.name,d.workflow_outputs)!=-1){d.workflow_outputs.splice($.inArray(b.name,d.workflow_outputs),1);l.find("img").attr("src",image_path+"/fugue/asterisk-small-outline.png")}else{d.workflow_outputs.push(b.name);l.find("img").attr("src",image_path+"/fugue/asterisk-small.png")}workflow.has_changes=true;canvas_manager.draw_overview()}))).tipsy({delayIn:500,fallback:"Flag this as a workflow output. All non-flagged outputs will be hidden."});l.css({top:"50%",margin:"-8px 0px 0px 0px",right:8});l.show();m.append(l);if($.inArray(b.name,d.workflow_outputs)===-1){l.find("img").attr("src",image_path+"/fugue/asterisk-small-outline.png")}else{l.find("img").attr("src",image_path+"/fugue/asterisk-small.png")}m.hover(function(){l.find("img").attr("src",image_path+"/fugue/asterisk-small-yellow.png")},function(){if($.inArray(b.name,d.workflow_outputs)===-1){l.find("img").attr("src",image_path+"/fugue/asterisk-small-outline.png")}else{l.find("img").attr("src",image_path+"/fugue/asterisk-small.png")}})}m.css({position:"absolute",left:-1000,top:-1000,display:"none"});$("body").append(m);c=Math.max(c,m.outerWidth()+17);m.css({position:"",left:"",top:"",display:""});m.detach();a.append(m.append(j))});g.css("width",Math.min(250,Math.max(g.width(),c)));workflow.node_changed(this)},update_field_data:function(f){var c=$(this.element),d=this;this.tool_state=f.tool_state;this.form_html=f.form_html;this.tool_errors=f.tool_errors;this.annotation=f.annotation;var g=$.parseJSON(f.post_job_actions);this.post_job_actions=g?g:{};if(this.tool_errors){c.addClass("tool-node-error")}else{c.removeClass("tool-node-error")}var h=c.find("div.inputs");var b=$("
    ");var a=h.find("div.input-data-row");$.each(f.data_inputs,function(l,j){var k=$("
    ");d.enable_input_terminal(k,j.name,j.extensions);h.find("div[name='"+j.name+"']").each(function(){$(this).find(".input-terminal").each(function(){var i=this.terminal.connectors[0];if(i){k[0].terminal.connectors[0]=i;i.handle2=k[0].terminal}});$(this).remove()});b.append($("
    "+j.label+"
    ").prepend(k))});h.replaceWith(b);h.find("div.input-data-row > .terminal").each(function(){this.terminal.destroy()});this.changed();this.redraw()},error:function(d){var a=$(this.element).find(".toolFormBody");a.find("div").remove();var c="
    "+d+"
    ";this.form_html=c;a.html(c);workflow.node_changed(this)},changed:function(){workflow.node_changed(this)}});function Workflow(a){this.canvas_container=a;this.id_counter=0;this.nodes={};this.name=null;this.has_changes=false;this.active_form_has_changes=false}$.extend(Workflow.prototype,{add_node:function(a){a.id=this.id_counter;a.element.attr("id","wf-node-step-"+a.id);this.id_counter++;this.nodes[a.id]=a;this.has_changes=true;a.workflow=this},remove_node:function(a){if(this.active_node==a){this.clear_active_node()}delete this.nodes[a.id];this.has_changes=true},remove_all:function(){wf=this;$.each(this.nodes,function(b,a){a.destroy();wf.remove_node(a)})},rectify_workflow_outputs:function(){var a=false;$.each(this.nodes,function(b,c){if(c.workflow_outputs&&c.workflow_outputs.length>0){a=true}});if(a==false){return true}wf=this;$.each(this.nodes,function(b,f){if(f.type=="tool"){var d=false;if(f.post_job_actions==null){f.post_job_actions={}}var c=[];$.each(f.post_job_actions,function(h,g){if(g.action_type=="HideDatasetAction"){c.push(h)}});if(c.length>0&&f==workflow.active_node){$.each(c,function(g,h){d=true;delete f.post_job_actions[h]})}$.each(f.output_terminals,function(h,i){var g=true;$.each(f.workflow_outputs,function(k,l){if(i.name==l){g=false}});if(g==true){d=true;var j={action_type:"HideDatasetAction",output_name:i.name,action_arguments:{}};f.post_job_actions["HideDatasetAction"+i.name]=null;f.post_job_actions["HideDatasetAction"+i.name]=j}});if(wf.active_node==f&&d==true){wf.reload_active_node()}}})},to_simple:function(){var a={};$.each(this.nodes,function(c,f){var g={};$.each(f.input_terminals,function(h,i){g[i.name]=null;$.each(i.connectors,function(j,k){g[i.name]={id:k.handle1.node.id,output_name:k.handle1.name}})});var b={};if(f.post_job_actions){$.each(f.post_job_actions,function(j,h){var k={action_type:h.action_type,output_name:h.output_name,action_arguments:h.action_arguments};b[h.action_type+h.output_name]=null;b[h.action_type+h.output_name]=k})}if(!f.workflow_outputs){f.workflow_outputs=[]}var d={id:f.id,type:f.type,tool_id:f.tool_id,tool_state:f.tool_state,tool_errors:f.tool_errors,input_connections:g,position:$(f.element).position(),annotation:f.annotation,post_job_actions:f.post_job_actions,workflow_outputs:f.workflow_outputs};a[f.id]=d});return{steps:a}},from_simple:function(a){wf=this;var b=0;wf.name=a.name;$.each(a.steps,function(f,d){var c=prebuild_node("tool",d.name,d.tool_id);c.init_field_data(d);if(d.position){c.element.css({top:d.position.top,left:d.position.left})}c.id=d.id;wf.nodes[c.id]=c;b=Math.max(b,parseInt(f))});wf.id_counter=b+1;$.each(a.steps,function(f,d){var c=wf.nodes[f];$.each(d.input_connections,function(h,g){if(g){var i=wf.nodes[g.id];var j=new Connector();j.connect(i.output_terminals[g.output_name],c.input_terminals[h]);j.redraw()}})})},check_changes_in_active_form:function(){if(this.active_form_has_changes){this.has_changes=true;$("#right-content").find("form").submit();this.active_form_has_changes=false}},reload_active_node:function(){if(this.active_node){var a=this.active_node;this.clear_active_node();this.activate_node(a)}},clear_active_node:function(){if(this.active_node){this.active_node.make_inactive();this.active_node=null}parent.show_form_for_tool("
    No node selected
    ")},activate_node:function(a){if(this.active_node!=a){this.check_changes_in_active_form();this.clear_active_node();parent.show_form_for_tool(a.form_html+a.tooltip,a);a.make_active();this.active_node=a}},node_changed:function(a){this.has_changes=true;if(this.active_node==a){this.check_changes_in_active_form();parent.show_form_for_tool(a.form_html+a.tooltip,a)}},layout:function(){this.check_changes_in_active_form();this.has_changes=true;var i={};var b={};$.each(this.nodes,function(l,k){if(i[l]===undefined){i[l]=0}if(b[l]===undefined){b[l]=[]}});$.each(this.nodes,function(l,k){$.each(k.input_terminals,function(m,n){$.each(n.connectors,function(p,q){var o=q.handle1.node;i[k.id]+=1;b[o.id].push(k.id)})})});node_ids_by_level=[];while(true){level_parents=[];for(var a in i){if(i[a]==0){level_parents.push(a)}}if(level_parents.length==0){break}node_ids_by_level.push(level_parents);for(var f in level_parents){var j=level_parents[f];delete i[j];for(var g in b[j]){i[b[j][g]]-=1}}}if(i.length){return}var d=this.nodes;var h=80;v_pad=30;var c=h;$.each(node_ids_by_level,function(k,l){l.sort(function(p,o){return $(d[p].element).position().top-$(d[o].element).position().top});var m=0;var n=v_pad;$.each(l,function(o,r){var q=d[r];var p=$(q.element);$(p).css({top:n,left:c});m=Math.max(m,$(p).width());n+=$(p).height()+v_pad});c+=m+h});$.each(d,function(k,l){l.redraw()})},bounds_for_all_nodes:function(){var d=Infinity,b=-Infinity,c=Infinity,a=-Infinity,f;$.each(this.nodes,function(h,g){e=$(g.element);f=e.position();d=Math.min(d,f.left);b=Math.max(b,f.left+e.width());c=Math.min(c,f.top);a=Math.max(a,f.top+e.width())});return{xmin:d,xmax:b,ymin:c,ymax:a}},fit_canvas_to_nodes:function(){var a=this.bounds_for_all_nodes();var f=this.canvas_container.position();var i=this.canvas_container.parent();var d=fix_delta(a.xmin,100);var h=fix_delta(a.ymin,100);d=Math.max(d,f.left);h=Math.max(h,f.top);var c=f.left-d;var g=f.top-h;var b=round_up(a.xmax+100,100)+d;var j=round_up(a.ymax+100,100)+h;b=Math.max(b,-c+i.width());j=Math.max(j,-g+i.height());this.canvas_container.css({left:c,top:g,width:b,height:j});this.canvas_container.children().each(function(){var k=$(this).position();$(this).css("left",k.left+d);$(this).css("top",k.top+h)})}});function fix_delta(a,b){if(a3*b){new_pos=(Math.ceil(((a%b))/b)+1)*b;return(-(a-new_pos))}return 0}function round_up(a,b){return Math.ceil(a/b)*b}function prebuild_node(l,j,r){var i=$("
    ");var g=new Node(i);g.type=l;if(l=="tool"){g.tool_id=r}var n=$("
    "+j+"
    ");i.append(n);i.css("left",$(window).scrollLeft()+20);i.css("top",$(window).scrollTop()+20);var m=$("
    ");var h="
    loading tool info...
    ";m.append(h);g.form_html=h;i.append(m);var k=$("
    ");k.append($("").attr("src",image_path+"/delete_icon.png").click(function(b){g.destroy()}).hover(function(){$(this).attr("src",image_path+"/delete_icon_dark.png")},function(){$(this).attr("src",image_path+"/delete_icon.png")}));i.appendTo("#canvas-container");var d=$("#canvas-container").position();var c=$("#canvas-container").parent();var a=i.width();var q=i.height();i.css({left:(-d.left)+(c.width()/2)-(a/2),top:(-d.top)+(c.height()/2)-(q/2)});k.prependTo(n);a+=(k.width()+10);i.css("width",a);$(i).bind("dragstart",function(){workflow.activate_node(g)}).bind("dragend",function(){workflow.node_changed(this);workflow.fit_canvas_to_nodes();canvas_manager.draw_overview()}).bind("dragclickonly",function(){workflow.activate_node(g)}).bind("drag",function(o,p){var f=$(this).offsetParent().offset(),b=p.offsetX-f.left,s=p.offsetY-f.top;$(this).css({left:b,top:s});$(this).find(".terminal").each(function(){this.terminal.redraw()})});return g}var ext_to_type=null;var type_to_type=null;function issubtype(b,a){b=ext_to_type[b];a=ext_to_type[a];return(type_to_type[b])&&(a in type_to_type[b])}function populate_datatype_info(a){ext_to_type=a.ext_to_class_name;type_to_type=a.class_to_classes}function ScrollPanel(a){this.panel=a}$.extend(ScrollPanel.prototype,{test:function(v,d){clearTimeout(this.timeout);var k=v.pageX,j=v.pageY,l=$(this.panel),c=l.position(),b=l.width(),i=l.height(),w=l.parent(),s=w.width(),a=w.height(),r=w.offset(),p=r.left,m=r.top,A=p+w.width(),u=m+w.height(),B=-(b-(s/2)),z=-(i-(a/2)),g=(s/2),f=(a/2),h=false,q=5,o=23;if(k-qA){if(c.left>B){var n=Math.min(o,c.left-B);l.css("left",c.left-n);h=true}}else{if(j-qu){if(c.top>z){var n=Math.min(o,c.top-B);l.css("top",(c.top-n)+"px");h=true}}}}}if(h){d();var l=this;this.timeout=setTimeout(function(){l.test(v,d)},50)}},stop:function(b,a){clearTimeout(this.timeout)}});function CanvasManager(b,a){this.cv=b;this.cc=this.cv.find("#canvas-container");this.oc=a.find("#overview-canvas");this.ov=a.find("#overview-viewport");this.init_drag()}$.extend(CanvasManager.prototype,{init_drag:function(){var b=this;var a=function(f,g){f=Math.min(f,b.cv.width()/2);f=Math.max(f,-b.cc.width()+b.cv.width()/2);g=Math.min(g,b.cv.height()/2);g=Math.max(g,-b.cc.height()+b.cv.height()/2);b.cc.css({left:f,top:g});b.update_viewport_overlay()};this.cc.each(function(){this.scroll_panel=new ScrollPanel(this)});var d,c;this.cv.bind("dragstart",function(){var g=$(this).offset();var f=b.cc.position();c=f.top-g.top;d=f.left-g.left}).bind("drag",function(f,g){a(g.offsetX+d,g.offsetY+c)}).bind("dragend",function(){workflow.fit_canvas_to_nodes();b.draw_overview()});this.ov.bind("drag",function(k,l){var h=b.cc.width(),n=b.cc.height(),m=b.oc.width(),j=b.oc.height(),f=$(this).offsetParent().offset(),i=l.offsetX-f.left,g=l.offsetY-f.top;a(-(i/m*h),-(g/j*n))}).bind("dragend",function(){workflow.fit_canvas_to_nodes();b.draw_overview()});$("#overview-border").bind("drag",function(g,i){var j=$(this).offsetParent();var h=j.offset();var f=Math.max(j.width()-(i.offsetX-h.left),j.height()-(i.offsetY-h.top));$(this).css({width:f,height:f});b.draw_overview()});$("#overview-border div").bind("drag",function(){})},update_viewport_overlay:function(){var b=this.cc,f=this.cv,a=this.oc,c=this.ov,d=b.width(),j=b.height(),i=a.width(),g=a.height(),h=b.position();c.css({left:-(h.left/d*i),top:-(h.top/j*g),width:(f.width()/d*i)-2,height:(f.height()/j*g)-2})},draw_overview:function(){var j=$("#overview-canvas"),m=j.parent().parent().width(),i=j.get(0).getContext("2d"),d=$("#canvas-container").width(),l=$("#canvas-container").height();var g,a,k,f;var h=this.cv.width();var b=this.cv.height();if(d0){i.fillStyle="#E8A92D";i.strokeStyle="#E8A92D"}}i.fillRect(c,r,o,p);i.strokeRect(c,r,o,p)});this.update_viewport_overlay()}}); \ No newline at end of file +function Terminal(a){this.element=a;this.connectors=[]}$.extend(Terminal.prototype,{connect:function(a){this.connectors.push(a);if(this.node){this.node.changed()}},disconnect:function(a){this.connectors.splice($.inArray(a,this.connectors),1);if(this.node){this.node.changed()}},redraw:function(){$.each(this.connectors,function(a,b){b.redraw()})},destroy:function(){$.each(this.connectors.slice(),function(a,b){b.destroy()})}});function OutputTerminal(a,b){Terminal.call(this,a);this.datatypes=b}OutputTerminal.prototype=new Terminal();function InputTerminal(a,b){Terminal.call(this,a);this.datatypes=b}InputTerminal.prototype=new Terminal();$.extend(InputTerminal.prototype,{can_accept:function(a){if(this.connectors.length<1){for(var c in this.datatypes){var f=new Array();f=f.concat(a.datatypes);if(a.node.post_job_actions){for(var d in a.node.post_job_actions){var g=a.node.post_job_actions[d];if(g.action_type=="ChangeDatatypeAction"&&(g.output_name==""||g.output_name==a.name)&&g.action_arguments){f.push(g.action_arguments.newtype)}}}for(var b in f){if(f[b]=="input"||issubtype(f[b],this.datatypes[c])){return true}}}}return false}});function Connector(b,a){this.canvas=null;this.dragging=false;this.inner_color="#FFFFFF";this.outer_color="#D8B365";if(b&&a){this.connect(b,a)}}$.extend(Connector.prototype,{connect:function(b,a){this.handle1=b;this.handle1.connect(this);this.handle2=a;this.handle2.connect(this)},destroy:function(){if(this.handle1){this.handle1.disconnect(this)}if(this.handle2){this.handle2.disconnect(this)}$(this.canvas).remove()},redraw:function(){var d=$("#canvas-container");if(!this.canvas){this.canvas=document.createElement("canvas");if(window.G_vmlCanvasManager){G_vmlCanvasManager.initElement(this.canvas)}d.append($(this.canvas));if(this.dragging){this.canvas.style.zIndex="300"}}var n=function(c){return $(c).offset().left-d.offset().left};var i=function(c){return $(c).offset().top-d.offset().top};var h=n(this.handle1.element)+5;var g=i(this.handle1.element)+5;var p=n(this.handle2.element)+5;var m=i(this.handle2.element)+5;var f=100;var k=Math.min(h,p);var a=Math.max(h,p);var j=Math.min(g,m);var t=Math.max(g,m);var b=Math.min(Math.max(Math.abs(t-j)/2,100),300);var o=k-f;var s=j-f;var q=a-k+2*f;var l=t-j+2*f;this.canvas.style.left=o+"px";this.canvas.style.top=s+"px";this.canvas.setAttribute("width",q);this.canvas.setAttribute("height",l);h-=o;g-=s;p-=o;m-=s;var r=this.canvas.getContext("2d");r.lineCap="round";r.strokeStyle=this.outer_color;r.lineWidth=7;r.beginPath();r.moveTo(h,g);r.bezierCurveTo(h+b,g,p-b,m,p,m);r.stroke();r.strokeStyle=this.inner_color;r.lineWidth=5;r.beginPath();r.moveTo(h,g);r.bezierCurveTo(h+b,g,p-b,m,p,m);r.stroke()}});function Node(a){this.element=a;this.input_terminals={};this.output_terminals={};this.tool_errors={}}$.extend(Node.prototype,{enable_input_terminal:function(d,a,b){var c=this;$(d).each(function(){var f=this.terminal=new InputTerminal(this,b);f.node=c;f.name=a;$(this).bind("dropinit",function(g,h){return $(h.drag).hasClass("output-terminal")&&f.can_accept(h.drag.terminal)}).bind("dropstart",function(g,h){h.proxy.terminal.connectors[0].inner_color="#BBFFBB"}).bind("dropend",function(g,h){h.proxy.terminal.connectors[0].inner_color="#FFFFFF"}).bind("drop",function(g,h){(new Connector(h.drag.terminal,f)).redraw()}).bind("hover",function(){if(f.connectors.length>0){var g=$("
    ").css({display:"none"}).appendTo("body").append($("
    ").append($("").attr("src",galaxy_paths.attributes.image_path+"/delete_icon.png").click(function(){$.each(f.connectors,function(i,h){h.destroy()});g.remove()}))).bind("mouseleave",function(){$(this).remove()});g.css({top:$(this).offset().top-2,left:$(this).offset().left-g.width(),"padding-right":$(this).width()}).show()}});c.input_terminals[a]=f})},enable_output_terminal:function(d,a,b){var c=this;$(d).each(function(){var g=this;var f=this.terminal=new OutputTerminal(this,b);f.node=c;f.name=a;$(this).bind("dragstart",function(j,k){$(k.available).addClass("input-terminal-active");workflow.check_changes_in_active_form();var i=$('
    ').appendTo("#canvas-container").get(0);i.terminal=new OutputTerminal(i);var l=new Connector();l.dragging=true;l.connect(this.terminal,i.terminal);return i}).bind("drag",function(i,j){var h=function(){var l=$(j.proxy).offsetParent().offset(),k=j.offsetX-l.left,m=j.offsetY-l.top;$(j.proxy).css({left:k,top:m});j.proxy.terminal.redraw();canvas_manager.update_viewport_overlay()};h();$("#canvas-container").get(0).scroll_panel.test(i,h)}).bind("dragend",function(h,i){i.proxy.terminal.connectors[0].destroy();$(i.proxy).remove();$(i.available).removeClass("input-terminal-active");$("#canvas-container").get(0).scroll_panel.stop()});c.output_terminals[a]=f})},redraw:function(){$.each(this.input_terminals,function(a,b){b.redraw()});$.each(this.output_terminals,function(a,b){b.redraw()})},destroy:function(){$.each(this.input_terminals,function(a,b){b.destroy()});$.each(this.output_terminals,function(a,b){b.destroy()});workflow.remove_node(this);$(this.element).remove()},make_active:function(){$(this.element).addClass("toolForm-active")},make_inactive:function(){var a=this.element.get(0);(function(b){b.removeChild(a);b.appendChild(a)})(a.parentNode);$(a).removeClass("toolForm-active")},init_field_data:function(h){var g=this.element;if(h.type){this.type=h.type}this.name=h.name;this.form_html=h.form_html;this.tool_state=h.tool_state;this.tool_errors=h.tool_errors;this.tooltip=h.tooltip?h.tooltip:"";this.annotation=h.annotation;this.post_job_actions=h.post_job_actions?h.post_job_actions:{};this.workflow_outputs=h.workflow_outputs?h.workflow_outputs:[];if(this.tool_errors){g.addClass("tool-node-error")}else{g.removeClass("tool-node-error")}var d=this;var c=Math.max(150,g.width());var a=g.find(".toolFormBody");a.find("div").remove();var i=$("
    ").appendTo(a);$.each(h.data_inputs,function(k,f){var j=$("
    ");d.enable_input_terminal(j,f.name,f.extensions);var b=$("
    "+f.label+"
    ");b.css({position:"absolute",left:-1000,top:-1000,display:"none"});$("body").append(b);c=Math.max(c,b.outerWidth());b.css({position:"",left:"",top:"",display:""});b.remove();i.append(b.prepend(j))});if((h.data_inputs.length>0)&&(h.data_outputs.length>0)){a.append($("
    "))}$.each(h.data_outputs,function(k,b){var j=$("
    ");d.enable_output_terminal(j,b.name,b.extensions);var f=b.name;if(b.extensions.indexOf("input")<0){f=f+" ("+b.extensions.join(", ")+")"}var m=$("
    "+f+"
    ");if(d.type=="tool"){var l=$("
    ").css({display:"none"}).append($("
    ").append($("").attr("src",galaxy_paths.attributes.image_path+"/fugue/asterisk-small-outline.png").click(function(){if($.inArray(b.name,d.workflow_outputs)!=-1){d.workflow_outputs.splice($.inArray(b.name,d.workflow_outputs),1);l.find("img").attr("src",galaxy_paths.attributes.image_path+"/fugue/asterisk-small-outline.png")}else{d.workflow_outputs.push(b.name);l.find("img").attr("src",galaxy_paths.attributes.image_path+"/fugue/asterisk-small.png")}workflow.has_changes=true;canvas_manager.draw_overview()}))).tipsy({delayIn:500,fallback:"Flag this as a workflow output. All non-flagged outputs will be hidden."});l.css({top:"50%",margin:"-8px 0px 0px 0px",right:8});l.show();m.append(l);if($.inArray(b.name,d.workflow_outputs)===-1){l.find("img").attr("src",galaxy_paths.attributes.image_path+"/fugue/asterisk-small-outline.png")}else{l.find("img").attr("src",galaxy_paths.attributes.image_path+"/fugue/asterisk-small.png")}m.hover(function(){l.find("img").attr("src",galaxy_paths.attributes.image_path+"/fugue/asterisk-small-yellow.png")},function(){if($.inArray(b.name,d.workflow_outputs)===-1){l.find("img").attr("src",galaxy_paths.attributes.image_path+"/fugue/asterisk-small-outline.png")}else{l.find("img").attr("src",galaxy_paths.attributes.image_path+"/fugue/asterisk-small.png")}})}m.css({position:"absolute",left:-1000,top:-1000,display:"none"});$("body").append(m);c=Math.max(c,m.outerWidth()+17);m.css({position:"",left:"",top:"",display:""});m.detach();a.append(m.append(j))});g.css("width",Math.min(250,Math.max(g.width(),c)));workflow.node_changed(this)},update_field_data:function(f){var c=$(this.element),d=this;this.tool_state=f.tool_state;this.form_html=f.form_html;this.tool_errors=f.tool_errors;this.annotation=f.annotation;var g=$.parseJSON(f.post_job_actions);this.post_job_actions=g?g:{};if(this.tool_errors){c.addClass("tool-node-error")}else{c.removeClass("tool-node-error")}var h=c.find("div.inputs");var b=$("
    ");var a=h.find("div.input-data-row");$.each(f.data_inputs,function(l,j){var k=$("
    ");d.enable_input_terminal(k,j.name,j.extensions);h.find("div[name='"+j.name+"']").each(function(){$(this).find(".input-terminal").each(function(){var i=this.terminal.connectors[0];if(i){k[0].terminal.connectors[0]=i;i.handle2=k[0].terminal}});$(this).remove()});b.append($("
    "+j.label+"
    ").prepend(k))});h.replaceWith(b);h.find("div.input-data-row > .terminal").each(function(){this.terminal.destroy()});this.changed();this.redraw()},error:function(d){var a=$(this.element).find(".toolFormBody");a.find("div").remove();var c="
    "+d+"
    ";this.form_html=c;a.html(c);workflow.node_changed(this)},changed:function(){workflow.node_changed(this)}});function Workflow(a){this.canvas_container=a;this.id_counter=0;this.nodes={};this.name=null;this.has_changes=false;this.active_form_has_changes=false}$.extend(Workflow.prototype,{add_node:function(a){a.id=this.id_counter;a.element.attr("id","wf-node-step-"+a.id);this.id_counter++;this.nodes[a.id]=a;this.has_changes=true;a.workflow=this},remove_node:function(a){if(this.active_node==a){this.clear_active_node()}delete this.nodes[a.id];this.has_changes=true},remove_all:function(){wf=this;$.each(this.nodes,function(b,a){a.destroy();wf.remove_node(a)})},rectify_workflow_outputs:function(){var b,a=false;$.each(this.nodes,function(c,d){if(d.workflow_outputs&&d.workflow_outputs.length>0){b=true}$.each(d.post_job_actions,function(g,f){if(f.action_type==="HideDatasetAction"){a=true}})});if(b!==false||a!==false){$.each(this.nodes,function(c,g){if(g.type==="tool"){var f=false;if(g.post_job_actions==null){g.post_job_actions={};f=true}var d=[];$.each(g.post_job_actions,function(i,h){if(h.action_type=="HideDatasetAction"){d.push(i)}});if(d.length>0&&g==workflow.active_node){$.each(d,function(h,j){f=true;delete g.post_job_actions[j]})}if(b){$.each(g.output_terminals,function(i,j){var h=true;$.each(g.workflow_outputs,function(l,m){if(j.name===m){h=false}});if(h===true){f=true;var k={action_type:"HideDatasetAction",output_name:j.name,action_arguments:{}};g.post_job_actions["HideDatasetAction"+j.name]=null;g.post_job_actions["HideDatasetAction"+j.name]=k}})}if(workflow.active_node==g&&f===true){workflow.reload_active_node()}}})}},to_simple:function(){var a={};$.each(this.nodes,function(c,f){var g={};$.each(f.input_terminals,function(h,i){g[i.name]=null;$.each(i.connectors,function(j,k){g[i.name]={id:k.handle1.node.id,output_name:k.handle1.name}})});var b={};if(f.post_job_actions){$.each(f.post_job_actions,function(j,h){var k={action_type:h.action_type,output_name:h.output_name,action_arguments:h.action_arguments};b[h.action_type+h.output_name]=null;b[h.action_type+h.output_name]=k})}if(!f.workflow_outputs){f.workflow_outputs=[]}var d={id:f.id,type:f.type,tool_id:f.tool_id,tool_state:f.tool_state,tool_errors:f.tool_errors,input_connections:g,position:$(f.element).position(),annotation:f.annotation,post_job_actions:f.post_job_actions,workflow_outputs:f.workflow_outputs};a[f.id]=d});return{steps:a}},from_simple:function(b){wf=this;var c=0;wf.name=b.name;var a=false;$.each(b.steps,function(g,f){var d=prebuild_node("tool",f.name,f.tool_id);d.init_field_data(f);if(f.position){d.element.css({top:f.position.top,left:f.position.left})}d.id=f.id;wf.nodes[d.id]=d;c=Math.max(c,parseInt(g));if(!a&&d.type==="tool"){if(d.workflow_outputs.length>0){a=true}else{$.each(d.post_job_actions,function(i,h){if(h.action_type==="HideDatasetAction"){a=true}})}}});wf.id_counter=c+1;$.each(b.steps,function(g,f){var d=wf.nodes[g];$.each(f.input_connections,function(i,h){if(h){var j=wf.nodes[h.id];var l=new Connector();l.connect(j.output_terminals[h.output_name],d.input_terminals[i]);l.redraw()}});if(a&&d.type==="tool"){$.each(d.output_terminals,function(h,i){if(d.post_job_actions["HideDatasetAction"+i.name]===undefined){d.workflow_outputs.push(i.name);callout=$(d.element).find(".callout."+i.name);callout.find("img").attr("src",galaxy_paths.attributes.image_path+"/fugue/asterisk-small.png");workflow.has_changes=true}})}})},check_changes_in_active_form:function(){if(this.active_form_has_changes){this.has_changes=true;$("#right-content").find("form").submit();this.active_form_has_changes=false}},reload_active_node:function(){if(this.active_node){var a=this.active_node;this.clear_active_node();this.activate_node(a)}},clear_active_node:function(){if(this.active_node){this.active_node.make_inactive();this.active_node=null}parent.show_form_for_tool("
    No node selected
    ")},activate_node:function(a){if(this.active_node!=a){this.check_changes_in_active_form();this.clear_active_node();parent.show_form_for_tool(a.form_html+a.tooltip,a);a.make_active();this.active_node=a}},node_changed:function(a){this.has_changes=true;if(this.active_node==a){this.check_changes_in_active_form();parent.show_form_for_tool(a.form_html+a.tooltip,a)}},layout:function(){this.check_changes_in_active_form();this.has_changes=true;var i={};var b={};$.each(this.nodes,function(l,k){if(i[l]===undefined){i[l]=0}if(b[l]===undefined){b[l]=[]}});$.each(this.nodes,function(l,k){$.each(k.input_terminals,function(m,n){$.each(n.connectors,function(p,q){var o=q.handle1.node;i[k.id]+=1;b[o.id].push(k.id)})})});node_ids_by_level=[];while(true){level_parents=[];for(var a in i){if(i[a]==0){level_parents.push(a)}}if(level_parents.length==0){break}node_ids_by_level.push(level_parents);for(var f in level_parents){var j=level_parents[f];delete i[j];for(var g in b[j]){i[b[j][g]]-=1}}}if(i.length){return}var d=this.nodes;var h=80;v_pad=30;var c=h;$.each(node_ids_by_level,function(k,l){l.sort(function(p,o){return $(d[p].element).position().top-$(d[o].element).position().top});var m=0;var n=v_pad;$.each(l,function(o,r){var q=d[r];var p=$(q.element);$(p).css({top:n,left:c});m=Math.max(m,$(p).width());n+=$(p).height()+v_pad});c+=m+h});$.each(d,function(k,l){l.redraw()})},bounds_for_all_nodes:function(){var d=Infinity,b=-Infinity,c=Infinity,a=-Infinity,f;$.each(this.nodes,function(h,g){e=$(g.element);f=e.position();d=Math.min(d,f.left);b=Math.max(b,f.left+e.width());c=Math.min(c,f.top);a=Math.max(a,f.top+e.width())});return{xmin:d,xmax:b,ymin:c,ymax:a}},fit_canvas_to_nodes:function(){var a=this.bounds_for_all_nodes();var f=this.canvas_container.position();var i=this.canvas_container.parent();var d=fix_delta(a.xmin,100);var h=fix_delta(a.ymin,100);d=Math.max(d,f.left);h=Math.max(h,f.top);var c=f.left-d;var g=f.top-h;var b=round_up(a.xmax+100,100)+d;var j=round_up(a.ymax+100,100)+h;b=Math.max(b,-c+i.width());j=Math.max(j,-g+i.height());this.canvas_container.css({left:c,top:g,width:b,height:j});this.canvas_container.children().each(function(){var k=$(this).position();$(this).css("left",k.left+d);$(this).css("top",k.top+h)})}});function fix_delta(a,b){if(a3*b){new_pos=(Math.ceil(((a%b))/b)+1)*b;return(-(a-new_pos))}return 0}function round_up(a,b){return Math.ceil(a/b)*b}function prebuild_node(l,j,r){var i=$("
    ");var g=new Node(i);g.type=l;if(l=="tool"){g.tool_id=r}var n=$("
    "+j+"
    ");i.append(n);i.css("left",$(window).scrollLeft()+20);i.css("top",$(window).scrollTop()+20);var m=$("
    ");var h="
    loading tool info...
    ";m.append(h);g.form_html=h;i.append(m);var k=$("
    ");k.append($("").attr("src",galaxy_paths.attributes.image_path+"/delete_icon.png").click(function(b){g.destroy()}).hover(function(){$(this).attr("src",galaxy_paths.attributes.image_path+"/delete_icon_dark.png")},function(){$(this).attr("src",galaxy_paths.attributes.image_path+"/delete_icon.png")}));i.appendTo("#canvas-container");var d=$("#canvas-container").position();var c=$("#canvas-container").parent();var a=i.width();var q=i.height();i.css({left:(-d.left)+(c.width()/2)-(a/2),top:(-d.top)+(c.height()/2)-(q/2)});k.prependTo(n);a+=(k.width()+10);i.css("width",a);$(i).bind("dragstart",function(){workflow.activate_node(g)}).bind("dragend",function(){workflow.node_changed(this);workflow.fit_canvas_to_nodes();canvas_manager.draw_overview()}).bind("dragclickonly",function(){workflow.activate_node(g)}).bind("drag",function(o,p){var f=$(this).offsetParent().offset(),b=p.offsetX-f.left,s=p.offsetY-f.top;$(this).css({left:b,top:s});$(this).find(".terminal").each(function(){this.terminal.redraw()})});return g}var ext_to_type=null;var type_to_type=null;function issubtype(b,a){b=ext_to_type[b];a=ext_to_type[a];return(type_to_type[b])&&(a in type_to_type[b])}function populate_datatype_info(a){ext_to_type=a.ext_to_class_name;type_to_type=a.class_to_classes}function ScrollPanel(a){this.panel=a}$.extend(ScrollPanel.prototype,{test:function(v,d){clearTimeout(this.timeout);var k=v.pageX,j=v.pageY,l=$(this.panel),c=l.position(),b=l.width(),i=l.height(),w=l.parent(),s=w.width(),a=w.height(),r=w.offset(),p=r.left,m=r.top,A=p+w.width(),u=m+w.height(),B=-(b-(s/2)),z=-(i-(a/2)),g=(s/2),f=(a/2),h=false,q=5,o=23;if(k-qA){if(c.left>B){var n=Math.min(o,c.left-B);l.css("left",c.left-n);h=true}}else{if(j-qu){if(c.top>z){var n=Math.min(o,c.top-B);l.css("top",(c.top-n)+"px");h=true}}}}}if(h){d();var l=this;this.timeout=setTimeout(function(){l.test(v,d)},50)}},stop:function(b,a){clearTimeout(this.timeout)}});function CanvasManager(b,a){this.cv=b;this.cc=this.cv.find("#canvas-container");this.oc=a.find("#overview-canvas");this.ov=a.find("#overview-viewport");this.init_drag()}$.extend(CanvasManager.prototype,{init_drag:function(){var b=this;var a=function(f,g){f=Math.min(f,b.cv.width()/2);f=Math.max(f,-b.cc.width()+b.cv.width()/2);g=Math.min(g,b.cv.height()/2);g=Math.max(g,-b.cc.height()+b.cv.height()/2);b.cc.css({left:f,top:g});b.update_viewport_overlay()};this.cc.each(function(){this.scroll_panel=new ScrollPanel(this)});var d,c;this.cv.bind("dragstart",function(){var g=$(this).offset();var f=b.cc.position();c=f.top-g.top;d=f.left-g.left}).bind("drag",function(f,g){a(g.offsetX+d,g.offsetY+c)}).bind("dragend",function(){workflow.fit_canvas_to_nodes();b.draw_overview()});this.ov.bind("drag",function(k,l){var h=b.cc.width(),n=b.cc.height(),m=b.oc.width(),j=b.oc.height(),f=$(this).offsetParent().offset(),i=l.offsetX-f.left,g=l.offsetY-f.top;a(-(i/m*h),-(g/j*n))}).bind("dragend",function(){workflow.fit_canvas_to_nodes();b.draw_overview()});$("#overview-border").bind("drag",function(g,i){var j=$(this).offsetParent();var h=j.offset();var f=Math.max(j.width()-(i.offsetX-h.left),j.height()-(i.offsetY-h.top));$(this).css({width:f,height:f});b.draw_overview()});$("#overview-border div").bind("drag",function(){})},update_viewport_overlay:function(){var b=this.cc,f=this.cv,a=this.oc,c=this.ov,d=b.width(),j=b.height(),i=a.width(),g=a.height(),h=b.position();c.css({left:-(h.left/d*i),top:-(h.top/j*g),width:(f.width()/d*i)-2,height:(f.height()/j*g)-2})},draw_overview:function(){var j=$("#overview-canvas"),m=j.parent().parent().width(),i=j.get(0).getContext("2d"),d=$("#canvas-container").width(),l=$("#canvas-container").height();var g,a,k,f;var h=this.cv.width();var b=this.cv.height();if(d0){i.fillStyle="#E8A92D";i.strokeStyle="#E8A92D"}}i.fillRect(c,r,o,p);i.strokeRect(c,r,o,p)});this.update_viewport_overlay()}}); \ No newline at end of file diff --git a/static/scripts/packed/libs/backbone-relational.js b/static/scripts/packed/libs/backbone-relational.js new file mode 100644 index 00000000000..b386486dd96 --- /dev/null +++ b/static/scripts/packed/libs/backbone-relational.js @@ -0,0 +1 @@ +(function(g){var d,h,b;if(typeof window==="undefined"){d=require("underscore");h=require("backbone");b=module.exports=h}else{var d=window._;h=window.Backbone;b=window}h.Relational={showWarnings:true};h.Semaphore={_permitsAvailable:null,_permitsUsed:0,acquire:function(){if(this._permitsAvailable&&this._permitsUsed>=this._permitsAvailable){throw new Error("Max permits acquired")}else{this._permitsUsed++}},release:function(){if(this._permitsUsed===0){throw new Error("All permits released")}else{this._permitsUsed--}},isLocked:function(){return this._permitsUsed>0},setAvailablePermits:function(i){if(this._permitsUsed>i){throw new Error("Available permits cannot be less than used permits")}this._permitsAvailable=i}};h.BlockingQueue=function(){this._queue=[]};d.extend(h.BlockingQueue.prototype,h.Semaphore,{_queue:null,add:function(i){if(this.isBlocked()){this._queue.push(i)}else{i()}},process:function(){while(this._queue&&this._queue.length){this._queue.shift()()}},block:function(){this.acquire()},unblock:function(){this.release();if(!this.isBlocked()){this.process()}},isBlocked:function(){return this.isLocked()}});h.Relational.eventQueue=new h.BlockingQueue();h.Store=function(){this._collections=[];this._reverseRelations=[]};d.extend(h.Store.prototype,h.Events,{_collections:null,_reverseRelations:null,addReverseRelation:function(j){var i=d.any(this._reverseRelations,function(k){return d.all(j,function(m,l){return m===k[l]})});if(!i&&j.model&&j.type){this._reverseRelations.push(j);if(!j.model.prototype.relations){j.model.prototype.relations=[]}j.model.prototype.relations.push(j);this.retroFitRelation(j)}},retroFitRelation:function(j){var i=this.getCollection(j.model);i.each(function(k){new j.type(k,j)},this)},getCollection:function(i){var j=d.detect(this._collections,function(k){return i===k.model||i.constructor===k.model});if(!j){j=this._createCollection(i)}return j},getObjectByName:function(i){var j=d.reduce(i.split("."),function(k,l){return k[l]},b);return j!==b?j:null},_createCollection:function(j){var i;if(j instanceof h.RelationalModel){j=j.constructor}if(j.prototype instanceof h.RelationalModel.prototype.constructor){i=new h.Collection();i.model=j;this._collections.push(i)}return i},resolveIdForItem:function(i,j){var k=d.isString(j)||d.isNumber(j)?j:null;if(k==null){if(j instanceof h.RelationalModel){k=j.id}else{if(d.isObject(j)){k=j[i.prototype.idAttribute]}}}return k},find:function(j,k){var l=this.resolveIdForItem(j,k);var i=this.getCollection(j);return i&&i.get(l)},register:function(j){var i=j.collection;var k=this.getCollection(j);k&&k.add(j);j.bind("destroy",this.unregister,this);j.collection=i},update:function(i){var j=this.getCollection(i);j._onModelEvent("change:"+i.idAttribute,i,j)},unregister:function(i){i.unbind("destroy",this.unregister);var j=this.getCollection(i);j&&j.remove(i)}});h.Relational.store=new h.Store();h.Relation=function(i,j){this.instance=i;j=(typeof j==="object"&&j)||{};this.reverseRelation=d.defaults(j.reverseRelation||{},this.options.reverseRelation);this.reverseRelation.type=!d.isString(this.reverseRelation.type)?this.reverseRelation.type:h[this.reverseRelation.type]||h.Relational.store.getObjectByName(this.reverseRelation.type);this.model=j.model||this.instance.constructor;this.options=d.defaults(j,this.options,h.Relation.prototype.options);this.key=this.options.key;this.keySource=this.options.keySource||this.key;this.keyDestination=this.options.keyDestination||this.options.keySource||this.key;this.relatedModel=this.options.relatedModel;if(d.isString(this.relatedModel)){this.relatedModel=h.Relational.store.getObjectByName(this.relatedModel)}if(!this.checkPreconditions()){return false}if(i){this.keyContents=this.instance.get(this.keySource);if(this.key!==this.keySource){this.instance.unset(this.keySource,{silent:true})}this.instance._relations.push(this)}if(!this.options.isAutoRelation&&this.reverseRelation.type&&this.reverseRelation.key){h.Relational.store.addReverseRelation(d.defaults({isAutoRelation:true,model:this.relatedModel,relatedModel:this.model,reverseRelation:this.options},this.reverseRelation))}d.bindAll(this,"_modelRemovedFromCollection","_relatedModelAdded","_relatedModelRemoved");if(i){this.initialize();h.Relational.store.getCollection(this.instance).bind("relational:remove",this._modelRemovedFromCollection);h.Relational.store.getCollection(this.relatedModel).bind("relational:add",this._relatedModelAdded).bind("relational:remove",this._relatedModelRemoved)}};h.Relation.extend=h.Model.extend;d.extend(h.Relation.prototype,h.Events,h.Semaphore,{options:{createModels:true,includeInJSON:true,isAutoRelation:false},instance:null,key:null,keyContents:null,relatedModel:null,reverseRelation:null,related:null,_relatedModelAdded:function(k,l,j){var i=this;k.queue(function(){i.tryAddRelated(k,j)})},_relatedModelRemoved:function(j,k,i){this.removeRelated(j,i)},_modelRemovedFromCollection:function(i){if(i===this.instance){this.destroy()}},checkPreconditions:function(){var n=this.instance,l=this.key,j=this.model,p=this.relatedModel,q=h.Relational.showWarnings&&typeof console!=="undefined";if(!j||!l||!p){q&&console.warn("Relation=%o; no model, key or relatedModel (%o, %o, %o)",this,j,l,p);return false}if(!(j.prototype instanceof h.RelationalModel.prototype.constructor)){q&&console.warn("Relation=%o; model does not inherit from Backbone.RelationalModel (%o)",this,n);return false}if(!(p.prototype instanceof h.RelationalModel.prototype.constructor)){q&&console.warn("Relation=%o; relatedModel does not inherit from Backbone.RelationalModel (%o)",this,p);return false}if(this instanceof h.HasMany&&this.reverseRelation.type===h.HasMany.prototype.constructor){q&&console.warn("Relation=%o; relation is a HasMany, and the reverseRelation is HasMany as well.",this);return false}if(n&&n._relations.length){var o=d.any(n._relations,function(i){var k=this.reverseRelation.key&&i.reverseRelation.key;return i.relatedModel===p&&i.key===l&&(!k||this.reverseRelation.key===i.reverseRelation.key)},this);if(o){q&&console.warn("Relation=%o between instance=%o.%s and relatedModel=%o.%s already exists",this,n,l,p,this.reverseRelation.key);return false}}return true},setRelated:function(j,i){this.related=j;this.instance.acquire();this.instance.set(this.key,j,d.defaults(i||{},{silent:true}));this.instance.release()},createModel:function(i){if(this.options.createModels&&typeof(i)==="object"){return new this.relatedModel(i)}},_isReverseRelation:function(i){if(i.instance instanceof this.relatedModel&&this.reverseRelation.key===i.key&&this.key===i.reverseRelation.key){return true}return false},getReverseRelations:function(i){var j=[];var k=!d.isUndefined(i)?[i]:this.related&&(this.related.models||[this.related]);d.each(k,function(l){d.each(l.getRelations(),function(m){if(this._isReverseRelation(m)){j.push(m)}},this)},this);return j},sanitizeOptions:function(i){i=i?d.clone(i):{};if(i.silent){i=d.extend({},i,{silentChange:true});delete i.silent}return i},unsanitizeOptions:function(i){i=i?d.clone(i):{};if(i.silentChange){i=d.extend({},i,{silent:true});delete i.silentChange}return i},destroy:function(){h.Relational.store.getCollection(this.instance).unbind("relational:remove",this._modelRemovedFromCollection);h.Relational.store.getCollection(this.relatedModel).unbind("relational:add",this._relatedModelAdded).unbind("relational:remove",this._relatedModelRemoved);d.each(this.getReverseRelations(),function(i){i.removeRelated(this.instance)},this)}});h.HasOne=h.Relation.extend({options:{reverseRelation:{type:"HasMany"}},initialize:function(){d.bindAll(this,"onChange");this.instance.bind("relational:change:"+this.key,this.onChange);var j=this.findRelated({silent:true});this.setRelated(j);var i=this;d.each(i.getReverseRelations(),function(k){k.addRelated(i.instance)})},findRelated:function(j){var k=this.keyContents;var i=null;if(k instanceof this.relatedModel){i=k}else{if(k){i=h.Relational.store.find(this.relatedModel,k);if(i&&d.isObject(k)){i.set(k,j)}else{if(!i){i=this.createModel(k)}}}}return i},onChange:function(l,i,k){if(this.isLocked()){return}this.acquire();k=this.sanitizeOptions(k);var o=d.isUndefined(k._related);var m=o?this.related:k._related;if(o){this.keyContents=i;if(i instanceof this.relatedModel){this.related=i}else{if(i){var n=this.findRelated(k);this.setRelated(n)}else{this.setRelated(null)}}}if(m&&this.related!==m){d.each(this.getReverseRelations(m),function(p){p.removeRelated(this.instance,k)},this)}d.each(this.getReverseRelations(),function(p){p.addRelated(this.instance,k)},this);if(!k.silentChange&&this.related!==m){var j=this;h.Relational.eventQueue.add(function(){j.instance.trigger("update:"+j.key,j.instance,j.related,k)})}this.release()},tryAddRelated:function(j,i){if(this.related){return}i=this.sanitizeOptions(i);var k=this.keyContents;if(k){var l=h.Relational.store.resolveIdForItem(this.relatedModel,k);if(j.id===l){this.addRelated(j,i)}}},addRelated:function(j,i){if(j!==this.related){var k=this.related||null;this.setRelated(j);this.onChange(this.instance,j,{_related:k})}},removeRelated:function(j,i){if(!this.related){return}if(j===this.related){var k=this.related||null;this.setRelated(null);this.onChange(this.instance,j,{_related:k})}}});h.HasMany=h.Relation.extend({collectionType:null,options:{reverseRelation:{type:"HasOne"},collectionType:h.Collection,collectionKey:true,collectionOptions:{}},initialize:function(){d.bindAll(this,"onChange","handleAddition","handleRemoval","handleReset");this.instance.bind("relational:change:"+this.key,this.onChange);this.collectionType=this.options.collectionType;if(d(this.collectionType).isString()){this.collectionType=h.Relational.store.getObjectByName(this.collectionType)}if(!this.collectionType.prototype instanceof h.Collection.prototype.constructor){throw new Error("collectionType must inherit from Backbone.Collection")}if(this.keyContents instanceof h.Collection){this.setRelated(this._prepareCollection(this.keyContents))}else{this.setRelated(this._prepareCollection())}this.findRelated({silent:true})},_getCollectionOptions:function(){return d.isFunction(this.options.collectionOptions)?this.options.collectionOptions(this.instance):this.options.collectionOptions},_prepareCollection:function(j){if(this.related){this.related.unbind("relational:add",this.handleAddition).unbind("relational:remove",this.handleRemoval).unbind("relational:reset",this.handleReset)}if(!j||!(j instanceof h.Collection)){j=new this.collectionType([],this._getCollectionOptions())}j.model=this.relatedModel;if(this.options.collectionKey){var i=this.options.collectionKey===true?this.options.reverseRelation.key:this.options.collectionKey;if(j[i]&&j[i]!==this.instance){if(h.Relational.showWarnings&&typeof console!=="undefined"){console.warn("Relation=%o; collectionKey=%s already exists on collection=%o",this,i,this.options.collectionKey)}}else{if(i){j[i]=this.instance}}}j.bind("relational:add",this.handleAddition).bind("relational:remove",this.handleRemoval).bind("relational:reset",this.handleReset);return j},findRelated:function(i){if(this.keyContents){var j=[];if(this.keyContents instanceof h.Collection){j=this.keyContents.models}else{this.keyContents=d.isArray(this.keyContents)?this.keyContents:[this.keyContents];d.each(this.keyContents,function(l){var k=h.Relational.store.find(this.relatedModel,l);if(k&&d.isObject(l)){k.set(l,i)}else{if(!k){k=this.createModel(l)}}if(k&&!this.related.getByCid(k)&&!this.related.get(k)){j.push(k)}},this)}if(j.length){i=this.unsanitizeOptions(i);this.related.add(j,i)}}},onChange:function(l,i,k){k=this.sanitizeOptions(k);this.keyContents=i;d.each(this.getReverseRelations(),function(n){n.removeRelated(this.instance,k)},this);if(i instanceof h.Collection){this._prepareCollection(i);this.related=i}else{var m;if(this.related instanceof h.Collection){m=this.related;m.reset([],{silent:true})}else{m=this._prepareCollection()}this.setRelated(m);this.findRelated(k)}d.each(this.getReverseRelations(),function(n){n.addRelated(this.instance,k)},this);var j=this;h.Relational.eventQueue.add(function(){!k.silentChange&&j.instance.trigger("update:"+j.key,j.instance,j.related,k)})},tryAddRelated:function(j,i){i=this.sanitizeOptions(i);if(!this.related.getByCid(j)&&!this.related.get(j)){var k=d.any(this.keyContents,function(l){var m=h.Relational.store.resolveIdForItem(this.relatedModel,l);return m&&m===j.id},this);if(k){this.related.add(j,i)}}},handleAddition:function(k,l,j){if(!(k instanceof h.Model)){return}j=this.sanitizeOptions(j);d.each(this.getReverseRelations(k),function(m){m.addRelated(this.instance,j)},this);var i=this;h.Relational.eventQueue.add(function(){!j.silentChange&&i.instance.trigger("add:"+i.key,k,i.related,j)})},handleRemoval:function(k,l,j){if(!(k instanceof h.Model)){return}j=this.sanitizeOptions(j);d.each(this.getReverseRelations(k),function(m){m.removeRelated(this.instance,j)},this);var i=this;h.Relational.eventQueue.add(function(){!j.silentChange&&i.instance.trigger("remove:"+i.key,k,i.related,j)})},handleReset:function(k,j){j=this.sanitizeOptions(j);var i=this;h.Relational.eventQueue.add(function(){!j.silentChange&&i.instance.trigger("reset:"+i.key,i.related,j)})},addRelated:function(k,j){var i=this;j=this.unsanitizeOptions(j);k.queue(function(){if(i.related&&!i.related.getByCid(k)&&!i.related.get(k)){i.related.add(k,j)}})},removeRelated:function(j,i){i=this.unsanitizeOptions(i);if(this.related.getByCid(j)||this.related.get(j)){this.related.remove(j,i)}}});h.RelationalModel=h.Model.extend({relations:null,_relations:null,_isInitialized:false,_deferProcessing:false,_queue:null,constructor:function(j,k){var i=this;if(k&&k.collection){this._deferProcessing=true;var l=function(m){if(m===i){i._deferProcessing=false;i.processQueue();k.collection.unbind("relational:add",l)}};k.collection.bind("relational:add",l);d.defer(function(){l(i)})}this._queue=new h.BlockingQueue();this._queue.block();h.Relational.eventQueue.block();h.Model.prototype.constructor.apply(this,arguments);h.Relational.eventQueue.unblock()},trigger:function(j){if(j.length>5&&"change"===j.substr(0,6)){var i=this,k=arguments;h.Relational.eventQueue.add(function(){h.Model.prototype.trigger.apply(i,k)})}else{h.Model.prototype.trigger.apply(this,arguments)}return this},initializeRelations:function(){this.acquire();this._relations=[];d.each(this.relations,function(i){var j=!d.isString(i.type)?i.type:h[i.type]||h.Relational.store.getObjectByName(i.type);if(j&&j.prototype instanceof h.Relation.prototype.constructor){new j(this,i)}else{h.Relational.showWarnings&&typeof console!=="undefined"&&console.warn("Relation=%o; missing or invalid type!",i)}},this);this._isInitialized=true;this.release();this.processQueue()},updateRelations:function(i){if(this._isInitialized&&!this.isLocked()){d.each(this._relations,function(j){var k=this.attributes[j.key];if(j.related!==k){this.trigger("relational:change:"+j.key,this,k,i||{})}},this)}},queue:function(i){this._queue.add(i)},processQueue:function(){if(this._isInitialized&&!this._deferProcessing&&this._queue.isBlocked()){this._queue.unblock()}},getRelation:function(i){return d.detect(this._relations,function(j){if(j.key===i){return true}},this)},getRelations:function(){return this._relations},fetchRelated:function(n,p){p||(p={});var l,j=[],o=this.getRelation(n),q=o&&o.keyContents,m=q&&d.select(d.isArray(q)?q:[q],function(r){var s=h.Relational.store.resolveIdForItem(o.relatedModel,r);return s&&!h.Relational.store.find(o.relatedModel,s)},this);if(m&&m.length){var k=d.map(m,function(t){var s;if(typeof(t)==="object"){s=new o.relatedModel(t)}else{var r={};r[o.relatedModel.prototype.idAttribute]=t;s=new o.relatedModel(r)}return s},this);if(o.related instanceof h.Collection&&d.isFunction(o.related.url)){l=o.related.url(k)}if(l&&l!==o.related.url()){var i=d.defaults({error:function(){var r=arguments;d.each(k,function(s){s.trigger("destroy",s,s.collection,p);p.error&&p.error.apply(s,r)})},url:l},p,{add:true});j=[o.related.fetch(i)]}else{j=d.map(k,function(r){var s=d.defaults({error:function(){r.trigger("destroy",r,r.collection,p);p.error&&p.error.apply(r,arguments)}},p);return r.fetch(s)},this)}}return j},set:function(l,m,k){h.Relational.eventQueue.block();var j;if(d.isObject(l)||l==null){j=l;k=m}else{j={};j[l]=m}var i=h.Model.prototype.set.apply(this,arguments);if(!this._isInitialized&&!this.isLocked()){h.Relational.store.register(this);this.initializeRelations()}else{if(j&&this.idAttribute in j){h.Relational.store.update(this)}}if(j){this.updateRelations(k)}h.Relational.eventQueue.unblock();return i},unset:function(k,j){h.Relational.eventQueue.block();var i=h.Model.prototype.unset.apply(this,arguments);this.updateRelations(j);h.Relational.eventQueue.unblock();return i},clear:function(j){h.Relational.eventQueue.block();var i=h.Model.prototype.clear.apply(this,arguments);this.updateRelations(j);h.Relational.eventQueue.unblock();return i},change:function(k){var i=this,j=arguments;h.Relational.eventQueue.add(function(){h.Model.prototype.change.apply(i,j)})},clone:function(){var i=d.clone(this.attributes);if(!d.isUndefined(i[this.idAttribute])){i[this.idAttribute]=null}d.each(this.getRelations(),function(j){delete i[j.key]});return new this.constructor(i)},toJSON:function(){if(this.isLocked()){return this.id}this.acquire();var i=h.Model.prototype.toJSON.call(this);d.each(this._relations,function(j){var k=i[j.key];if(j.options.includeInJSON===true&&k&&d.isFunction(k.toJSON)){i[j.keyDestination]=k.toJSON()}else{if(d.isString(j.options.includeInJSON)){if(k instanceof h.Collection){i[j.keyDestination]=k.pluck(j.options.includeInJSON)}else{if(k instanceof h.Model){i[j.keyDestination]=k.get(j.options.includeInJSON)}}}else{delete i[j.key]}}if(j.keyDestination!==j.key){delete i[j.key]}},this);this.release();return i}});d.extend(h.RelationalModel.prototype,h.Semaphore);var f=h.Collection.prototype.__add=h.Collection.prototype.add;h.Collection.prototype.add=function(k,i){i||(i={});if(!d.isArray(k)){k=[k]}var j=[];d.each(k,function(m){if(!(m instanceof h.Model)){var l=h.Relational.store.find(this.model,m[this.model.prototype.idAttribute]);if(l){l.set(l.parse?l.parse(m):m,i);m=l}else{m=h.Collection.prototype._prepareModel.call(this,m,i)}}if(m instanceof h.Model&&!this.get(m)&&!this.getByCid(m)){j.push(m)}},this);if(j.length){f.call(this,j,i);d.each(j,function(l){this.trigger("relational:add",l,this,i)},this)}return this};var a=h.Collection.prototype.__remove=h.Collection.prototype.remove;h.Collection.prototype.remove=function(j,i){i||(i={});if(!d.isArray(j)){j=[j]}d.each(j,function(k){k=this.getByCid(k)||this.get(k);if(k instanceof h.Model){a.call(this,k,i);this.trigger("relational:remove",k,this,i)}},this);return this};var e=h.Collection.prototype.__reset=h.Collection.prototype.reset;h.Collection.prototype.reset=function(j,i){e.call(this,j,i);this.trigger("relational:reset",j,i);return this};var c=h.Collection.prototype.__trigger=h.Collection.prototype.trigger;h.Collection.prototype.trigger=function(j){if(j==="add"||j==="remove"||j==="reset"){var i=this,k=arguments;h.Relational.eventQueue.add(function(){c.apply(i,k)})}else{c.apply(this,arguments)}return this};h.RelationalModel.extend=function(j,k){var l=h.Model.extend.apply(this,arguments);var i=(j&&j.relations)||[];d.each(i,function(m){if(m.reverseRelation){m.model=l;var o=true;if(d.isString(m.relatedModel)){var n=h.Relational.store.getObjectByName(m.relatedModel);o=n&&(n.prototype instanceof h.RelationalModel.prototype.constructor)}var p=!d.isString(m.type)?m.type:h[m.type]||h.Relational.store.getObjectByName(m.type);if(o&&p&&p.prototype instanceof h.Relation.prototype.constructor){new p(null,m)}}});return l}})(); \ No newline at end of file diff --git a/static/scripts/packed/libs/backbone.js b/static/scripts/packed/libs/backbone.js new file mode 100644 index 00000000000..5238e90fb48 --- /dev/null +++ b/static/scripts/packed/libs/backbone.js @@ -0,0 +1 @@ +(function(){var q=this;var o=q.Backbone;var p=Array.prototype.slice;var y=Array.prototype.splice;var c;if(typeof exports!=="undefined"){c=exports}else{c=q.Backbone={}}c.VERSION="0.9.2";var A=q._;if(!A&&(typeof require!=="undefined")){A=require("underscore")}var g=q.jQuery||q.Zepto||q.ender;c.setDomLibrary=function(C){g=C};c.noConflict=function(){q.Backbone=o;return this};c.emulateHTTP=false;c.emulateJSON=false;var a=/\s+/;var n=c.Events={on:function(F,J,E){var D,H,G,C,I;if(!J){return this}F=F.split(a);D=this._callbacks||(this._callbacks={});while(H=F.shift()){I=D[H];G=I?I.tail:{};G.next=C={};G.context=E;G.callback=J;D[H]={tail:C,next:I?I.next:G}}return this},off:function(J,H,D){var C,K,E,G,F,I;if(!(K=this._callbacks)){return}if(!(J||H||D)){delete this._callbacks;return this}J=J?J.split(a):A.keys(K);while(C=J.shift()){E=K[C];delete K[C];if(!E||!(H||D)){continue}G=E.tail;while((E=E.next)!==G){F=E.callback;I=E.context;if((H&&F!==H)||(D&&I!==D)){this.on(C,F,I)}}}return this},trigger:function(F){var J,I,E,D,C,H,G;if(!(E=this._callbacks)){return this}H=E.all;F=F.split(a);G=p.call(arguments,1);while(J=F.shift()){if(I=E[J]){D=I.tail;while((I=I.next)!==D){I.callback.apply(I.context||this,G)}}if(I=H){D=I.tail;C=[J].concat(G);while((I=I.next)!==D){I.callback.apply(I.context||this,C)}}}return this}};n.bind=n.on;n.unbind=n.off;var k=c.Model=function(C,D){var E;C||(C={});if(D&&D.parse){C=this.parse(C)}if(E=d(this,"defaults")){C=A.extend({},E,C)}if(D&&D.collection){this.collection=D.collection}this.attributes={};this._escapedAttributes={};this.cid=A.uniqueId("c");this.changed={};this._silent={};this._pending={};this.set(C,{silent:true});this.changed={};this._silent={};this._pending={};this._previousAttributes=A.clone(this.attributes);this.initialize.apply(this,arguments)};A.extend(k.prototype,n,{changed:null,_silent:null,_pending:null,idAttribute:"id",initialize:function(){},toJSON:function(C){return A.clone(this.attributes)},get:function(C){return this.attributes[C]},escape:function(C){var D;if(D=this._escapedAttributes[C]){return D}var E=this.get(C);return this._escapedAttributes[C]=A.escape(E==null?"":""+E)},has:function(C){return this.get(C)!=null},set:function(J,I,L){var K,G,E;if(A.isObject(J)||J==null){K=J;L=I}else{K={};K[J]=I}L||(L={});if(!K){return this}if(K instanceof k){K=K.attributes}if(L.unset){for(G in K){K[G]=void 0}}if(!this._validate(K,L)){return false}if(this.idAttribute in K){this.id=K[this.idAttribute]}var H=L.changes={};var D=this.attributes;var C=this._escapedAttributes;var F=this._previousAttributes||{};for(G in K){E=K[G];if(!A.isEqual(D[G],E)||(L.unset&&A.has(D,G))){delete C[G];(L.silent?this._silent:H)[G]=true}L.unset?delete D[G]:D[G]=E;if(!A.isEqual(F[G],E)||(A.has(D,G)!=A.has(F,G))){this.changed[G]=E;if(!L.silent){this._pending[G]=true}}else{delete this.changed[G];delete this._pending[G]}}if(!L.silent){this.change(L)}return this},unset:function(C,D){(D||(D={})).unset=true;return this.set(C,null,D)},clear:function(C){(C||(C={})).unset=true;return this.set(A.clone(this.attributes),C)},fetch:function(D){D=D?A.clone(D):{};var C=this;var E=D.success;D.success=function(H,F,G){if(!C.set(C.parse(H,G),D)){return false}if(E){E(C,H)}};D.error=c.wrapError(D.error,C,D);return(this.sync||c.sync).call(this,"read",this,D)},save:function(H,G,L){var I,F;if(A.isObject(H)||H==null){I=H;L=G}else{I={};I[H]=G}L=L?A.clone(L):{};if(L.wait){if(!this._validate(I,L)){return false}F=A.clone(this.attributes)}var D=A.extend({},L,{silent:true});if(I&&!this.set(I,L.wait?D:L)){return false}var E=this;var J=L.success;L.success=function(P,M,O){var N=E.parse(P,O);if(L.wait){delete L.wait;N=A.extend(I||{},N)}if(!E.set(N,L)){return false}if(J){J(E,P)}else{E.trigger("sync",E,P,L)}};L.error=c.wrapError(L.error,E,L);var C=this.isNew()?"create":"update";var K=(this.sync||c.sync).call(this,C,this,L);if(L.wait){this.set(F,D)}return K},destroy:function(D){D=D?A.clone(D):{};var C=this;var G=D.success;var F=function(){C.trigger("destroy",C,C.collection,D)};if(this.isNew()){F();return false}D.success=function(H){if(D.wait){F()}if(G){G(C,H)}else{C.trigger("sync",C,H,D)}};D.error=c.wrapError(D.error,C,D);var E=(this.sync||c.sync).call(this,"delete",this,D);if(!D.wait){F()}return E},url:function(){var C=d(this,"urlRoot")||d(this.collection,"url")||t();if(this.isNew()){return C}return C+(C.charAt(C.length-1)=="/"?"":"/")+encodeURIComponent(this.id)},parse:function(D,C){return D},clone:function(){return new this.constructor(this.attributes)},isNew:function(){return this.id==null},change:function(D){D||(D={});var F=this._changing;this._changing=true;for(var C in this._silent){this._pending[C]=true}var E=A.extend({},D.changes,this._silent);this._silent={};for(var C in E){this.trigger("change:"+C,this,this.get(C),D)}if(F){return this}while(!A.isEmpty(this._pending)){this._pending={};this.trigger("change",this,D);for(var C in this.changed){if(this._pending[C]||this._silent[C]){continue}delete this.changed[C]}this._previousAttributes=A.clone(this.attributes)}this._changing=false;return this},hasChanged:function(C){if(!arguments.length){return !A.isEmpty(this.changed)}return A.has(this.changed,C)},changedAttributes:function(E){if(!E){return this.hasChanged()?A.clone(this.changed):false}var G,F=false,D=this._previousAttributes;for(var C in E){if(A.isEqual(D[C],(G=E[C]))){continue}(F||(F={}))[C]=G}return F},previous:function(C){if(!arguments.length||!this._previousAttributes){return null}return this._previousAttributes[C]},previousAttributes:function(){return A.clone(this._previousAttributes)},isValid:function(){return !this.validate(this.attributes)},_validate:function(E,D){if(D.silent||!this.validate){return true}E=A.extend({},this.attributes,E);var C=this.validate(E,D);if(!C){return true}if(D&&D.error){D.error(this,C,D)}else{this.trigger("error",this,C,D)}return false}});var B=c.Collection=function(D,C){C||(C={});if(C.model){this.model=C.model}if(C.comparator){this.comparator=C.comparator}this._reset();this.initialize.apply(this,arguments);if(D){this.reset(D,{silent:true,parse:C.parse})}};A.extend(B.prototype,n,{model:k,initialize:function(){},toJSON:function(C){return this.map(function(D){return D.toJSON(C)})},add:function(D,M){var I,K,F,J,L,E,G={},C={},H=[];M||(M={});D=A.isArray(D)?D.slice():[D];for(I=0,F=D.length;I').hide().appendTo("body")[0].contentWindow;this.navigate(D)}if(this._hasPushState){g(window).bind("popstate",this.checkUrl)}else{if(this._wantsHashChange&&("onhashchange" in window)&&!G){g(window).bind("hashchange",this.checkUrl)}else{if(this._wantsHashChange){this._checkUrlInterval=setInterval(this.checkUrl,this.interval)}}}this.fragment=D;var H=window.location;var F=H.pathname==this.options.root;if(this._wantsHashChange&&this._wantsPushState&&!this._hasPushState&&!F){this.fragment=this.getFragment(null,true);window.location.replace(this.options.root+"#"+this.fragment);return true}else{if(this._wantsPushState&&this._hasPushState&&F&&H.hash){this.fragment=this.getHash().replace(m,"");window.history.replaceState({},document.title,H.protocol+"//"+H.host+this.options.root+this.fragment)}}if(!this.options.silent){return this.loadUrl()}},stop:function(){g(window).unbind("popstate",this.checkUrl).unbind("hashchange",this.checkUrl);clearInterval(this._checkUrlInterval);b.started=false},route:function(C,D){this.handlers.unshift({route:C,callback:D})},checkUrl:function(D){var C=this.getFragment();if(C==this.fragment&&this.iframe){C=this.getFragment(this.getHash(this.iframe))}if(C==this.fragment){return false}if(this.iframe){this.navigate(C)}this.loadUrl()||this.loadUrl(this.getHash())},loadUrl:function(E){var D=this.fragment=this.getFragment(E);var C=A.any(this.handlers,function(F){if(F.route.test(D)){F.callback(D);return true}});return C},navigate:function(D,C){if(!b.started){return false}if(!C||C===true){C={trigger:C}}var E=(D||"").replace(m,"");if(this.fragment==E){return}if(this._hasPushState){if(E.indexOf(this.options.root)!=0){E=this.options.root+E}this.fragment=E;window.history[C.replace?"replaceState":"pushState"]({},document.title,E)}else{if(this._wantsHashChange){this.fragment=E;this._updateHash(window.location,E,C.replace);if(this.iframe&&(E!=this.getFragment(this.getHash(this.iframe)))){if(!C.replace){this.iframe.document.open().close()}this._updateHash(this.iframe.location,E,C.replace)}}else{window.location.assign(this.options.root+D)}}if(C.trigger){this.loadUrl(D)}},_updateHash:function(C,D,E){if(E){C.replace(C.toString().replace(/(javascript:|#).*$/,"")+"#"+D)}else{C.hash=D}}});var s=c.View=function(C){this.cid=A.uniqueId("view");this._configure(C||{});this._ensureElement();this.initialize.apply(this,arguments);this.delegateEvents()};var f=/^(\S+)\s*(.*)$/;var u=["model","collection","el","id","attributes","className","tagName"];A.extend(s.prototype,n,{tagName:"div",$:function(C){return this.$el.find(C)},initialize:function(){},render:function(){return this},remove:function(){this.$el.remove();return this},make:function(D,C,F){var E=document.createElement(D);if(C){g(E).attr(C)}if(F){g(E).html(F)}return E},setElement:function(C,D){if(this.$el){this.undelegateEvents()}this.$el=(C instanceof g)?C:g(C);this.el=this.$el[0];if(D!==false){this.delegateEvents()}return this},delegateEvents:function(G){if(!(G||(G=d(this,"events")))){return}this.undelegateEvents();for(var F in G){var H=G[F];if(!A.isFunction(H)){H=this[G[F]]}if(!H){throw new Error('Method "'+G[F]+'" does not exist')}var E=F.match(f);var D=E[1],C=E[2];H=A.bind(H,this);D+=".delegateEvents"+this.cid;if(C===""){this.$el.bind(D,H)}else{this.$el.delegate(C,D,H)}}},undelegateEvents:function(){this.$el.unbind(".delegateEvents"+this.cid)},_configure:function(E){if(this.options){E=A.extend({},this.options,E)}for(var F=0,D=u.length;F0){for(var e=0,b=f.length;e0){for(var e=0,b=f.length;e":">",'"':""","'":"'","`":"`"};var d=/&(?!\w+;)|[<>"'`]/g;var b=/[&<>"'`]/;var a=function(e){return c[e]||"&"};Handlebars.Utils={escapeExpression:function(e){if(e instanceof Handlebars.SafeString){return e.toString()}else{if(e==null||e===false){return""}}if(!b.test(e)){return e}return e.replace(d,a)},isEmpty:function(e){if(typeof e==="undefined"){return true}else{if(e===null){return true}else{if(e===false){return true}else{if(Object.prototype.toString.call(e)==="[object Array]"&&e.length===0){return true}else{return false}}}}}}})();Handlebars.VM={template:function(a){var b={escapeExpression:Handlebars.Utils.escapeExpression,invokePartial:Handlebars.VM.invokePartial,programs:[],program:function(d,e,f){var c=this.programs[d];if(f){return Handlebars.VM.program(e,f)}else{if(c){return c}else{c=this.programs[d]=Handlebars.VM.program(e);return c}}},programWithDepth:Handlebars.VM.programWithDepth,noop:Handlebars.VM.noop};return function(d,c){c=c||{};return a.call(b,Handlebars,d,c.helpers,c.partials,c.data)}},programWithDepth:function(b,d,c){var a=Array.prototype.slice.call(arguments,2);return function(f,e){e=e||{};return b.apply(this,[f,e.data||d].concat(a))}},program:function(a,b){return function(d,c){c=c||{};return a(d,c.data||b)}},noop:function(){return""},invokePartial:function(a,b,d,e,c,f){options={helpers:e,partials:c,data:f};if(a===undefined){throw new Handlebars.Exception("The partial "+b+" could not be found")}else{if(a instanceof Function){return a(d,options)}else{if(!Handlebars.compile){throw new Handlebars.Exception("The partial "+b+" could not be compiled when running in runtime-only mode")}else{c[b]=Handlebars.compile(a);return c[b](d,options)}}}}};Handlebars.template=Handlebars.VM.template; \ No newline at end of file diff --git a/static/scripts/packed/libs/underscore.js b/static/scripts/packed/libs/underscore.js new file mode 100644 index 00000000000..1cc8c740dac --- /dev/null +++ b/static/scripts/packed/libs/underscore.js @@ -0,0 +1 @@ +(function(){var x=this;var t=x._;var b={};var j=Array.prototype,D=Object.prototype,G=Function.prototype;var v=j.slice,z=j.unshift,y=D.toString,p=D.hasOwnProperty;var n=j.forEach,i=j.map,B=j.reduce,e=j.reduceRight,m=j.filter,a=j.every,A=j.some,w=j.indexOf,f=j.lastIndexOf,c=Array.isArray,C=Object.keys,k=G.bind;var F=function(I){return new g(I)};if(typeof exports!=="undefined"){if(typeof module!=="undefined"&&module.exports){exports=module.exports=F}exports._=F}else{x._=F}F.VERSION="1.3.1";var d=F.each=F.forEach=function(N,M,L){if(N==null){return}if(n&&N.forEach===n){N.forEach(M,L)}else{if(N.length===+N.length){for(var K=0,I=N.length;K2;if(M==null){M=[]}if(B&&M.reduce===B){if(K){L=F.bind(L,K)}return J?M.reduce(L,I):M.reduce(L)}d(M,function(P,N,O){if(!J){I=P;J=true}else{I=L.call(K,I,P,N,O)}});if(!J){throw new TypeError("Reduce of empty array with no initial value")}return I};F.reduceRight=F.foldr=function(M,L,I,K){var J=arguments.length>2;if(M==null){M=[]}if(e&&M.reduceRight===e){if(K){L=F.bind(L,K)}return J?M.reduceRight(L,I):M.reduceRight(L)}var N=F.toArray(M).reverse();if(K&&!J){L=F.bind(L,K)}return J?F.reduce(N,L,I,K):F.reduce(N,L)};F.find=F.detect=function(L,K,J){var I;r(L,function(O,M,N){if(K.call(J,O,M,N)){I=O;return true}});return I};F.filter=F.select=function(L,K,J){var I=[];if(L==null){return I}if(m&&L.filter===m){return L.filter(K,J)}d(L,function(O,M,N){if(K.call(J,O,M,N)){I[I.length]=O}});return I};F.reject=function(L,K,J){var I=[];if(L==null){return I}d(L,function(O,M,N){if(!K.call(J,O,M,N)){I[I.length]=O}});return I};F.every=F.all=function(L,K,J){var I=true;if(L==null){return I}if(a&&L.every===a){return L.every(K,J)}d(L,function(O,M,N){if(!(I=I&&K.call(J,O,M,N))){return b}});return I};var r=F.some=F.any=function(L,K,J){K||(K=F.identity);var I=false;if(L==null){return I}if(A&&L.some===A){return L.some(K,J)}d(L,function(O,M,N){if(I||(I=K.call(J,O,M,N))){return b}});return !!I};F.include=F.contains=function(K,J){var I=false;if(K==null){return I}if(w&&K.indexOf===w){return K.indexOf(J)!=-1}I=r(K,function(L){return L===J});return I};F.invoke=function(J,K){var I=v.call(arguments,2);return F.map(J,function(L){return(F.isFunction(K)?K||L:L[K]).apply(L,I)})};F.pluck=function(J,I){return F.map(J,function(K){return K[I]})};F.max=function(L,K,J){if(!K&&F.isArray(L)){return Math.max.apply(Math,L)}if(!K&&F.isEmpty(L)){return -Infinity}var I={computed:-Infinity};d(L,function(P,M,O){var N=K?K.call(J,P,M,O):P;N>=I.computed&&(I={value:P,computed:N})});return I.value};F.min=function(L,K,J){if(!K&&F.isArray(L)){return Math.min.apply(Math,L)}if(!K&&F.isEmpty(L)){return Infinity}var I={computed:Infinity};d(L,function(P,M,O){var N=K?K.call(J,P,M,O):P;NL?1:0}),"value")};F.groupBy=function(K,L){var I={};var J=F.isFunction(L)?L:function(M){return M[L]};d(K,function(O,M){var N=J(O,M);(I[N]||(I[N]=[])).push(O)});return I};F.sortedIndex=function(N,M,K){K||(K=F.identity);var I=0,L=N.length;while(I>1;K(N[J])=0})})};F.difference=function(J){var I=F.flatten(v.call(arguments,1));return F.filter(J,function(K){return !F.include(I,K)})};F.zip=function(){var I=v.call(arguments);var L=F.max(F.pluck(I,"length"));var K=new Array(L);for(var J=0;J=0;K--){J=[I[K].apply(this,J)]}return J[0]}};F.after=function(J,I){if(J<=0){return I()}return function(){if(--J<1){return I.apply(this,arguments)}}};F.keys=C||function(K){if(K!==Object(K)){throw new TypeError("Invalid object")}var J=[];for(var I in K){if(F.has(K,I)){J[J.length]=I}}return J};F.values=function(I){return F.map(I,F.identity)};F.functions=F.methods=function(K){var J=[];for(var I in K){if(F.isFunction(K[I])){J.push(I)}}return J.sort()};F.extend=function(I){d(v.call(arguments,1),function(J){for(var K in J){I[K]=J[K]}});return I};F.defaults=function(I){d(v.call(arguments,1),function(J){for(var K in J){if(I[K]==null){I[K]=J[K]}}});return I};F.clone=function(I){if(!F.isObject(I)){return I}return F.isArray(I)?I.slice():F.extend({},I)};F.tap=function(J,I){I(J);return J};function E(L,K,J){if(L===K){return L!==0||1/L==1/K}if(L==null||K==null){return L===K}if(L._chain){L=L._wrapped}if(K._chain){K=K._wrapped}if(L.isEqual&&F.isFunction(L.isEqual)){return L.isEqual(K)}if(K.isEqual&&F.isFunction(K.isEqual)){return K.isEqual(L)}var O=y.call(L);if(O!=y.call(K)){return false}switch(O){case"[object String]":return L==String(K);case"[object Number]":return L!=+L?K!=+K:(L==0?1/L==1/K:L==+K);case"[object Date]":case"[object Boolean]":return +L==+K;case"[object RegExp]":return L.source==K.source&&L.global==K.global&&L.multiline==K.multiline&&L.ignoreCase==K.ignoreCase}if(typeof L!="object"||typeof K!="object"){return false}var P=J.length;while(P--){if(J[P]==L){return true}}J.push(L);var N=0,I=true;if(O=="[object Array]"){N=L.length;I=N==K.length;if(I){while(N--){if(!(I=N in L==N in K&&E(L[N],K[N],J))){break}}}}else{if("constructor" in L!="constructor" in K||L.constructor!=K.constructor){return false}for(var M in L){if(F.has(L,M)){N++;if(!(I=F.has(K,M)&&E(L[M],K[M],J))){break}}}if(I){for(M in K){if(F.has(K,M)&&!(N--)){break}}I=!N}}J.pop();return I}F.isEqual=function(J,I){return E(J,I,[])};F.isEmpty=function(J){if(F.isArray(J)||F.isString(J)){return J.length===0}for(var I in J){if(F.has(J,I)){return false}}return true};F.isElement=function(I){return !!(I&&I.nodeType==1)};F.isArray=c||function(I){return y.call(I)=="[object Array]"};F.isObject=function(I){return I===Object(I)};F.isArguments=function(I){return y.call(I)=="[object Arguments]"};if(!F.isArguments(arguments)){F.isArguments=function(I){return !!(I&&F.has(I,"callee"))}}F.isFunction=function(I){return y.call(I)=="[object Function]"};F.isString=function(I){return y.call(I)=="[object String]"};F.isNumber=function(I){return y.call(I)=="[object Number]"};F.isNaN=function(I){return I!==I};F.isBoolean=function(I){return I===true||I===false||y.call(I)=="[object Boolean]"};F.isDate=function(I){return y.call(I)=="[object Date]"};F.isRegExp=function(I){return y.call(I)=="[object RegExp]"};F.isNull=function(I){return I===null};F.isUndefined=function(I){return I===void 0};F.has=function(J,I){return p.call(J,I)};F.noConflict=function(){x._=t;return this};F.identity=function(I){return I};F.times=function(L,K,J){for(var I=0;I/g,">").replace(/"/g,""").replace(/'/g,"'").replace(/\//g,"/")};F.mixin=function(I){d(F.functions(I),function(J){s(J,F[J]=I[J])})};var l=0;F.uniqueId=function(I){var J=l++;return I?I+J:J};F.templateSettings={evaluate:/<%([\s\S]+?)%>/g,interpolate:/<%=([\s\S]+?)%>/g,escape:/<%-([\s\S]+?)%>/g};var u=/.^/;var q=function(I){return I.replace(/\\\\/g,"\\").replace(/\\'/g,"'")};F.template=function(L,K){var M=F.templateSettings;var I="var __p=[],print=function(){__p.push.apply(__p,arguments);};with(obj||{}){__p.push('"+L.replace(/\\/g,"\\\\").replace(/'/g,"\\'").replace(M.escape||u,function(N,O){return"',_.escape("+q(O)+"),'"}).replace(M.interpolate||u,function(N,O){return"',"+q(O)+",'"}).replace(M.evaluate||u,function(N,O){return"');"+q(O).replace(/[\r\n\t]/g," ")+";__p.push('"}).replace(/\r/g,"\\r").replace(/\n/g,"\\n").replace(/\t/g,"\\t")+"');}return __p.join('');";var J=new Function("obj","_",I);if(K){return J(K,F)}return function(N){return J.call(this,N,F)}};F.chain=function(I){return F(I).chain()};var g=function(I){this._wrapped=I};F.prototype=g.prototype;var o=function(J,I){return I?F(J).chain():J};var s=function(I,J){g.prototype[I]=function(){var K=v.call(arguments);z.call(K,this._wrapped);return o(J.apply(F,K),this._chain)}};F.mixin(F);d(["pop","push","reverse","shift","sort","splice","unshift"],function(I){var J=j[I];g.prototype[I]=function(){var K=this._wrapped;J.apply(K,arguments);var L=K.length;if((I=="shift"||I=="splice")&&L===0){delete K[0]}return o(K,this._chain)}});d(["concat","join","slice"],function(I){var J=j[I];g.prototype[I]=function(){return o(J.apply(this._wrapped,arguments),this._chain)}});g.prototype.chain=function(){this._chain=true;return this};g.prototype.value=function(){return this._wrapped}}).call(this); \ No newline at end of file diff --git a/static/scripts/packed/mvc/tools.js b/static/scripts/packed/mvc/tools.js new file mode 100644 index 00000000000..a83db63386f --- /dev/null +++ b/static/scripts/packed/mvc/tools.js @@ -0,0 +1 @@ +var BaseModel=Backbone.RelationalModel.extend({defaults:{name:null,hidden:false},show:function(){this.set("hidden",false)},hide:function(){this.set("hidden",true)},is_visible:function(){return !this.attributes.hidden}});var Tool=BaseModel.extend({defaults:{description:null,target:null,inputs:[]},relations:[{type:Backbone.HasMany,key:"inputs",relatedModel:"ToolInput",reverseRelation:{key:"tool"}}],urlRoot:galaxy_paths.attributes.root_path+"api/tools",apply_search_results:function(a){(_.indexOf(a,this.attributes.id)!==-1?this.show():this.hide());return this.is_visible()}});var ToolInput=Backbone.RelationalModel.extend({initialize:function(){this.attributes.html=unescape(this.attributes.html)}});var ToolCollection=Backbone.Collection.extend({model:Tool});var ToolPanelLabel=BaseModel.extend({});var ToolPanelSection=BaseModel.extend({defaults:{elems:[],open:false},clear_search_results:function(){_.each(this.attributes.elems,function(a){a.show()});this.show();this.set("open",false)},apply_search_results:function(b){var c=true,a;_.each(this.attributes.elems,function(d){if(d instanceof ToolPanelLabel){a=d;a.hide()}else{if(d instanceof Tool){if(d.apply_search_results(b)){c=false;if(a){a.show()}}}}});if(c){this.hide()}else{this.show();this.set("open",true)}}});var ToolSearch=BaseModel.extend({defaults:{spinner_url:"",search_url:"",visible:true,query:"",results:null},initialize:function(){this.on("change:query",this.do_search)},do_search:function(){var c=this.attributes.query;if(c.length<3){this.set("results",null);return}var b=c+"*";if(this.timer){clearTimeout(this.timer)}$("#search-spinner").show();var a=this;this.timer=setTimeout(function(){$.get(a.attributes.search_url,{query:b},function(d){a.set("results",d);$("#search-spinner").hide()},"json")},200)}});var ToolPanel=Backbone.Collection.extend({url:"/tools",tools:new ToolCollection(),parse:function(a){var b=function(e){var d=e.type;if(d==="tool"){return new Tool(e)}else{if(d==="section"){var c=_.map(e.elems,b);e.elems=c;return new ToolPanelSection(e)}else{if(d==="label"){return new ToolPanelLabel(e)}}}};return _.map(a,b)},initialize:function(a){this.tool_search=a.tool_search;this.tool_search.on("change:results",this.apply_search_results,this);this.on("reset",this.populate_tools,this)},populate_tools:function(){var a=this;a.tools=new ToolCollection();this.each(function(b){if(b instanceof ToolPanelSection){_.each(b.attributes.elems,function(c){if(c instanceof Tool){a.tools.push(c)}})}else{if(b instanceof Tool){a.tools.push(b)}}})},clear_search_results:function(){this.each(function(a){if(a instanceof ToolPanelSection){a.clear_search_results()}else{a.show()}})},apply_search_results:function(){var b=this.tool_search.attributes.results;if(b===null){this.clear_search_results();return}var a=null;this.each(function(c){if(c instanceof ToolPanelLabel){a=c;a.hide()}else{if(c instanceof Tool){if(c.apply_search_results(b)){if(a){a.show()}}}else{a=null;c.apply_search_results(b)}}})}});var BaseView=Backbone.View.extend({initialize:function(){this.model.on("change:hidden",this.update_visible,this);this.update_visible()},update_visible:function(){(this.model.attributes.hidden?this.$el.hide():this.$el.show())}});var ToolLinkView=BaseView.extend({tagName:"div",template:Handlebars.templates.tool_link,render:function(){this.$el.append(this.template(this.model.toJSON()));return this}});var ToolPanelLabelView=BaseView.extend({tagName:"div",className:"toolPanelLabel",render:function(){this.$el.append($("").text(this.model.attributes.name));return this}});var ToolPanelSectionView=BaseView.extend({tagName:"div",className:"toolSectionWrapper",template:Handlebars.templates.panel_section,initialize:function(){BaseView.prototype.initialize.call(this);this.model.on("change:open",this.update_open,this)},render:function(){this.$el.append(this.template(this.model.toJSON()));var a=this.$el.find(".toolSectionBody");_.each(this.model.attributes.elems,function(b){if(b instanceof Tool){var c=new ToolLinkView({model:b,className:"toolTitle"});c.render();a.append(c.$el)}else{if(b instanceof ToolPanelLabel){var d=new ToolPanelLabelView({model:b});d.render();a.append(d.$el)}else{}}});return this},events:{"click .toolSectionTitle > a":"toggle"},toggle:function(){this.model.set("open",!this.model.attributes.open)},update_open:function(){(this.model.attributes.open?this.$el.children(".toolSectionBody").slideDown("fast"):this.$el.children(".toolSectionBody").slideUp("fast"))}});var ToolSearchView=Backbone.View.extend({tagName:"div",id:"tool-search",className:"bar",template:Handlebars.templates.tool_search,events:{click:"focus_and_select","keyup :input":"query_changed"},render:function(){this.$el.append(this.template(this.model.toJSON()));if(!this.model.is_visible()){this.$el.hide()}return this},focus_and_select:function(){this.$el.find(":input").focus().select()},query_changed:function(){this.model.set("query",this.$el.find(":input").val())}});var ToolPanelView=Backbone.View.extend({tagName:"div",className:"toolMenu",initialize:function(){this.collection.tool_search.on("change:results",this.handle_search_results,this)},render:function(){var a=this;var b=new ToolSearchView({model:this.collection.tool_search});b.render();a.$el.append(b.$el);this.collection.each(function(d){if(d instanceof ToolPanelSection){var c=new ToolPanelSectionView({model:d});c.render();a.$el.append(c.$el)}else{if(d instanceof Tool){var e=new ToolLinkView({model:d,className:"toolTitleNoSection"});e.render();a.$el.append(e.$el)}else{if(d instanceof ToolPanelLabel){var f=new ToolPanelLabelView({model:d});f.render();a.$el.append(f.$el)}}}});a.$el.find("a.tool-link").click(function(f){var d=$(this).attr("class").split(/\s+/)[0],c=a.collection.tools.get(d);a.trigger("tool_link_click",f,c)});return this},handle_search_results:function(){var a=this.collection.tool_search.attributes.results;if(a&&a.length===0){$("#search-no-results").show()}else{$("#search-no-results").hide()}}});var ToolFormView=Backbone.View.extend({className:"toolForm",template:Handlebars.templates.tool_form,render:function(){this.$el.children().remove();this.$el.append(this.template(this.model.toJSON()))}});var IntegratedToolMenuAndView=Backbone.View.extend({className:"toolMenuAndView",initialize:function(){this.tool_panel_view=new ToolPanelView({collection:this.collection});this.tool_form_view=new ToolFormView()},render:function(){this.tool_panel_view.render();this.tool_panel_view.$el.css("float","left");this.$el.append(this.tool_panel_view.$el);this.tool_form_view.$el.hide();this.$el.append(this.tool_form_view.$el);var a=this;this.tool_panel_view.on("tool_link_click",function(c,b){c.preventDefault();a.show_tool(b)})},show_tool:function(b){var a=this;b.fetch().done(function(){a.tool_form_view.model=b;a.tool_form_view.render();a.tool_form_view.$el.show();$("#left").width("650px")})}}); \ No newline at end of file diff --git a/static/scripts/packed/mvc/ui.js b/static/scripts/packed/mvc/ui.js new file mode 100644 index 00000000000..f3140ca7d17 --- /dev/null +++ b/static/scripts/packed/mvc/ui.js @@ -0,0 +1 @@ +var GalaxyPaths=Backbone.Model.extend({defaults:{root_path:"",image_path:""}});var IconButton=Backbone.Model.extend({defaults:{title:"",icon_class:"",on_click:null}});var IconButtonCollection=Backbone.Collection.extend({model:IconButton});var IconButtonMenuView=Backbone.View.extend({tagName:"div",render:function(){var a=this;this.collection.each(function(b){$("").attr("href","javascript:void(0)").attr("title",b.attributes.title).addClass("icon-button menu-button").addClass(b.attributes.icon_class).appendTo(a.$el).click(b.attributes.on_click)});return this}});var Grid=Backbone.Collection.extend({});var GridView=Backbone.View.extend({}); \ No newline at end of file diff --git a/static/scripts/packed/mvc/visualization.js b/static/scripts/packed/mvc/visualization.js new file mode 100644 index 00000000000..fc312c26daa --- /dev/null +++ b/static/scripts/packed/mvc/visualization.js @@ -0,0 +1 @@ +var TrackBrowserRouter=Backbone.Router.extend({initialize:function(b){this.view=b.view;this.route(/([\w]+)$/,"change_location");this.route(/([\w]+\:[\d,]+-[\d,]+)$/,"change_location");var a=this;a.view.on("navigate",function(c){a.navigate(c)})},change_location:function(a){this.view.go_to(a)}}); \ No newline at end of file diff --git a/static/scripts/packed/templates/compiled/panel_label.js b/static/scripts/packed/templates/compiled/panel_label.js new file mode 100644 index 00000000000..85f9f943b15 --- /dev/null +++ b/static/scripts/packed/templates/compiled/panel_label.js @@ -0,0 +1 @@ +(function(){var b=Handlebars.template,a=Handlebars.templates=Handlebars.templates||{};a.panel_label=b(function(e,n,d,l,k){d=d||e.helpers;var i="",c,h,o=this,f="function",m=d.helperMissing,g=void 0,j=this.escapeExpression;i+="";h=d.name;c=h||n.name;if(typeof c===f){c=c.call(n,{hash:{}})}else{if(c===g){c=m.call(n,"name",{hash:{}})}}i+=j(c)+"";return i})})(); \ No newline at end of file diff --git a/static/scripts/packed/templates/compiled/panel_section.js b/static/scripts/packed/templates/compiled/panel_section.js new file mode 100644 index 00000000000..65e313324db --- /dev/null +++ b/static/scripts/packed/templates/compiled/panel_section.js @@ -0,0 +1 @@ +(function(){var b=Handlebars.template,a=Handlebars.templates=Handlebars.templates||{};a.panel_section=b(function(e,n,d,l,k){d=d||e.helpers;var i="",c,h,o=this,f="function",m=d.helperMissing,g=void 0,j=this.escapeExpression;i+='\n