opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

merge; add temporary template to commit

Browse Source
This commit is contained in:
Carl Eberhard
2012-08-31 12:12:28 -04:00
parent ac67dc378e 2d589bd376
commit f1c09ee6cf
217 changed files with 4206 additions and 4102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cron/get_ncbi.py
+93
View File
@@ -0,0 +1,93 @@
import urllib, pkg_resources, os
pkg_resources.require( 'elementtree' )
from elementtree import ElementTree, ElementInclude
from xml.parsers.expat import ExpatError as XMLParseErrorThing
import sys
import pkg_resources
class GetListing:
def __init__( self, data ):
self.tree = ElementTree.parse( data )
self.root = self.tree.getroot()
ElementInclude.include(self.root)
def xml_text(self, name=None):
"""Returns the text inside an element"""
root = self.root
if name is not None:
# Try attribute first
val = root.get(name)
if val:
return val
# Then try as element
elem = root.find(name)
else:
elem = root
if elem is not None and elem.text:
text = ''.join(elem.text.splitlines())
return text.strip()
# No luck, return empty string
return ''
def dlcachefile( webenv, querykey, i, results ):
url = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=nuccore&usehistory=y&term=nuccore_assembly[filter]%20AND%20refseq[filter]'
fp = urllib.urlopen( url )
search = GetListing( fp )
fp.close()
webenv = search.xml_text( 'WebEnv' )
querykey = search.xml_text( 'QueryKey' )
url = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=nuccore&WebEnv=%s&query_key=%s&retstart=%d&retmax=%d' % ( webenv, querykey, i, results )
fp = urllib.urlopen( url )
cachefile = os.tmpfile()
for line in fp:
cachefile.write( line )
fp.close()
cachefile.flush()
cachefile.seek(0)
return cachefile
url = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=nuccore&usehistory=y&term=nuccore_assembly[filter]%20AND%20refseq[filter]'
fp = urllib.urlopen( url )
results = GetListing( fp )
fp.close()
webenv = results.xml_text( 'WebEnv' )
querykey = results.xml_text( 'QueryKey' )
counts = int( results.xml_text( 'Count' ) )
results = 10000
found = 0
for i in range(0, counts + results, results):
rets = dict()
cache = dlcachefile( webenv, querykey, i, results )
try:
xmldoc = GetListing( cache )
except (IOError, XMLParseErrorThing):
cache = dlcachefile( webenv, querykey, i, results )
try:
xmldoc = GetListing( cache )
except (IOError, XMLParseErrorThing):
cache.close()
exit()
pass
finally:
cache.close()
entries = xmldoc.root.findall( 'DocSum' )
for entry in entries:
dbkey = None
children = entry.findall('Item')
for item in children:
rets[ item.get('Name') ] = item.text
if not rets['Caption'].startswith('NC_'):
continue
for ret in rets['Extra'].split('|'):
if not ret.startswith('NC_'):
continue
else:
dbkey = ret
break
if dbkey is not None:
print '\t'.join( [ dbkey, rets['Title'] ] )
cron/updatencbi.sh.sample
+42
View File
@@ -0,0 +1,42 @@
#!/bin/sh
#
# Script to update NCBI shared data tables. The idea is to update, but if
# the update fails, not replace current data/tables with error
# messages.
# Edit this line to refer to galaxy's path:
GALAXY=/path/to/galaxy
PYTHONPATH=${GALAXY}/lib
export PYTHONPATH
# setup directories
echo "Creating required directories."
DIRS="
${GALAXY}/tool-data/shared/ncbi
${GALAXY}/tool-data/shared/ncbi/new
"
for dir in $DIRS; do
if [ ! -d $dir ]; then
echo "Creating $dir"
mkdir $dir
else
echo "$dir already exists, continuing."
fi
done
date
echo "Updating NCBI shared data tables."
# Try to build "builds.txt"
echo "Updating builds.txt"
python ${GALAXY}/cron/get_ncbi.py > ${GALAXY}/tool-data/shared/ncbi/new/builds.txt
if [ $? -eq 0 ]
then
diff ${GALAXY}/tool-data/shared/ncbi/new/builds.txt ${GALAXY}/tool-data/shared/ncbi/builds.txt > /dev/null 2>&1
if [ $? -ne 0 ]
then
cp -f ${GALAXY}/tool-data/shared/ncbi/new/builds.txt ${GALAXY}/tool-data/shared/ncbi/builds.txt
fi
else
echo "Failed to update builds.txt" >&2
fi
datatypes_conf.xml.sample
+5 -3
View File
@@ -4,6 +4,7 @@
<datatype extension="ab1" type="galaxy.datatypes.binary:Ab1" mimetype="application/octet-stream" display_in_upload="true"/>
<datatype extension="afg" type="galaxy.datatypes.assembly:Amos" display_in_upload="false"/>
<datatype extension="axt" type="galaxy.datatypes.sequence:Axt" display_in_upload="true"/>
<datatype extension="fli" type="galaxy.datatypes.tabular:FeatureLocationIndex" display_in_upload="false"/>
<datatype extension="bam" type="galaxy.datatypes.binary:Bam" mimetype="application/octet-stream" display_in_upload="true">
<converter file="bam_to_bai.xml" target_datatype="bai"/>
<converter file="bam_to_summary_tree_converter.xml" target_datatype="summary_tree" depends_on="bai"/>
@@ -22,7 +23,9 @@
<display file="genetrack.xml" />
<display file="igb/bed.xml" />
</datatype>
<datatype extension="bedgraph" type="galaxy.datatypes.interval:BedGraph" display_in_upload="true" />
<datatype extension="bedgraph" type="galaxy.datatypes.interval:BedGraph" display_in_upload="true">
<converter file="bedgraph_to_bigwig_converter.xml" target_datatype="bigwig"/>
</datatype>
<datatype extension="bedstrict" type="galaxy.datatypes.interval:BedStrict" />
<datatype extension="bed6" type="galaxy.datatypes.interval:Bed6">
<converter file="bed_to_genetrack_converter.xml" target_datatype="genetrack"/>
@@ -79,6 +82,7 @@
<converter file="gff_to_bed_converter.xml" target_datatype="bed"/>
<converter file="gff_to_interval_index_converter.xml" target_datatype="interval_index"/>
<converter file="gff_to_summary_tree_converter.xml" target_datatype="summary_tree"/>
<converter file="gff_to_fli_converter.xml" target_datatype="fli"/>
<display file="ensembl/ensembl_gff.xml" inherit="True"/>
<!-- <display file="gbrowse/gbrowse_gff.xml" inherit="True" /> -->
</datatype>
@@ -167,7 +171,6 @@
<datatype extension="linecount" type="galaxy.datatypes.data:LineCount" display_in_upload="false"/>
<datatype extension="memexml" type="galaxy.datatypes.xml:MEMEXml" mimetype="application/xml" display_in_upload="true"/>
<datatype extension="cisml" type="galaxy.datatypes.xml:CisML" mimetype="application/xml" display_in_upload="true"/>
<datatype extension="blastxml" type="galaxy.datatypes.xml:BlastXml" mimetype="application/xml" display_in_upload="true"/>
<datatype extension="xml" type="galaxy.datatypes.xml:GenericXml" mimetype="application/xml" display_in_upload="true"/>
<datatype extension="vcf" type="galaxy.datatypes.tabular:Vcf" display_in_upload="true">
<converter file="vcf_to_bgzip_converter.xml" target_datatype="bgzip"/>
@@ -244,7 +247,6 @@
<sniffer type="galaxy.datatypes.binary:TwoBit"/>
<sniffer type="galaxy.datatypes.binary:Bam"/>
<sniffer type="galaxy.datatypes.binary:Sff"/>
<sniffer type="galaxy.datatypes.xml:BlastXml"/>
<sniffer type="galaxy.datatypes.xml:GenericXml"/>
<sniffer type="galaxy.datatypes.sequence:Maf"/>
<sniffer type="galaxy.datatypes.sequence:Lav"/>
display_applications/gbrowse/gbrowse_gff.xml
+13 -5
View File
@@ -1,16 +1,24 @@
<display id="gbrowse_gff" version="1.0.0" name="display at GBrowse">
<!-- Load links from file: one line to one link -->
<dynamic_links from_file="tool-data/shared/gbrowse/gbrowse_build_sites.txt" skip_startswith="#" id="0" name="0">
<dynamic_links from_file="tool-data/shared/gbrowse/gbrowse_build_sites.txt" skip_startswith="#" id="0" name="1">
<!-- Define parameters by column from file, allow splitting on builds -->
<dynamic_param name="site_id" value="0"/>
<dynamic_param name="gbrowse_link" value="1"/>
<dynamic_param name="builds" value="2" split="True" separator="," />
<dynamic_param name="site_name" value="1"/>
<dynamic_param name="site_link" value="2"/>
<dynamic_param name="site_dbkeys" value="3" split="True" separator="," />
<dynamic_param name="site_organisms" value="4" split="True" separator="," />
<!-- Filter out some of the links based upon matching site_id to a Galaxy application configuration parameter and by dataset dbkey -->
<filter>${site_id in $APP.config.gbrowse_display_sites}</filter>
<filter>${dataset.dbkey in $builds}</filter>
<filter>${dataset.dbkey in $site_dbkeys}</filter>
<!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
<url>${gbrowse_link}/?${position}eurl=${gff_file.qp}</url>
<url>${site_link}${site_organism}/?${position}eurl=${gff_file.qp}</url>
<param type="data" name="gff_file" url="galaxy_${DATASET_HASH}.gff" />
<param type="template" name="site_organism" strip="True" >
$site_organisms[ $site_dbkeys.index( $gff_file.dbkey ) ]
</param>
<param type="template" name="position" strip="True" >
#set chrom, start, end = $gff_file.datatype.get_estimated_display_viewport( $gff_file )
#if $chrom is not None:
display_applications/gbrowse/gbrowse_interval_as_bed.xml
+13 -5
View File
@@ -1,16 +1,24 @@
<display id="gbrowse_interval_as_bed" version="1.0.0" name="display at GBrowse">
<!-- Load links from file: one line to one link -->
<dynamic_links from_file="tool-data/shared/gbrowse/gbrowse_build_sites.txt" skip_startswith="#" id="0" name="0">
<dynamic_links from_file="tool-data/shared/gbrowse/gbrowse_build_sites.txt" skip_startswith="#" id="0" name="1">
<!-- Define parameters by column from file, allow splitting on builds -->
<dynamic_param name="site_id" value="0"/>
<dynamic_param name="gbrowse_link" value="1"/>
<dynamic_param name="builds" value="2" split="True" separator="," />
<dynamic_param name="site_name" value="1"/>
<dynamic_param name="site_link" value="2"/>
<dynamic_param name="site_dbkeys" value="3" split="True" separator="," />
<dynamic_param name="site_organisms" value="4" split="True" separator="," />
<!-- Filter out some of the links based upon matching site_id to a Galaxy application configuration parameter and by dataset dbkey -->
<filter>${site_id in $APP.config.gbrowse_display_sites}</filter>
<filter>${dataset.dbkey in $builds}</filter>
<filter>${dataset.dbkey in $site_dbkeys}</filter>
<!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
<url>${gbrowse_link}/?${position}eurl=${bed_file.qp}</url>
<url>${site_link}${site_organism}/?${position}eurl=${bed_file.qp}</url>
<param type="data" name="bed_file" url="galaxy_${DATASET_HASH}.bed" format="bedstrict"/> <!-- Galaxy allows BED files to contain non-standard fields beyond the first 3 columns, gbrowse does not(?): force use of converter which will make strict BED6+ file -->
<param type="template" name="site_organism" strip="True" >
$site_organisms[ $site_dbkeys.index( $bed_file.dbkey ) ]
</param>
<param type="template" name="position" strip="True" >
#set chrom, start, end = $bed_file.datatype.get_estimated_display_viewport( $bed_file )
#if $chrom is not None:
display_applications/gbrowse/gbrowse_wig.xml
+13 -5
View File
@@ -1,16 +1,24 @@
<display id="gbrowse_wig" version="1.0.0" name="display at GBrowse">
<!-- Load links from file: one line to one link -->
<dynamic_links from_file="tool-data/shared/gbrowse/gbrowse_build_sites.txt" skip_startswith="#" id="0" name="0">
<dynamic_links from_file="tool-data/shared/gbrowse/gbrowse_build_sites.txt" skip_startswith="#" id="0" name="1">
<!-- Define parameters by column from file, allow splitting on builds -->
<dynamic_param name="site_id" value="0"/>
<dynamic_param name="gbrowse_link" value="1"/>
<dynamic_param name="builds" value="2" split="True" separator="," />
<dynamic_param name="site_name" value="1"/>
<dynamic_param name="site_link" value="2"/>
<dynamic_param name="site_dbkeys" value="3" split="True" separator="," />
<dynamic_param name="site_organisms" value="4" split="True" separator="," />
<!-- Filter out some of the links based upon matching site_id to a Galaxy application configuration parameter and by dataset dbkey -->
<filter>${site_id in $APP.config.gbrowse_display_sites}</filter>
<filter>${dataset.dbkey in $builds}</filter>
<filter>${dataset.dbkey in $site_dbkeys}</filter>
<!-- We define url and params as normal, but values defined in dynamic_param are available by specified name -->
<url>${gbrowse_link}/?${position}eurl=${wig_file.qp}</url>
<url>${site_link}${site_organism}/?${position}eurl=${wig_file.qp}</url>
<param type="data" name="wig_file" url="galaxy_${DATASET_HASH}.wig" format="wig"/>
<param type="template" name="site_organism" strip="True" >
$site_organisms[ $site_dbkeys.index( $wig_file.dbkey ) ]
</param>
<param type="template" name="position" strip="True" >
#set chrom, start, end = $wig_file.datatype.get_estimated_display_viewport( $wig_file )
#if $chrom is not None:
eggs.ini
+1 -1
View File
@@ -17,7 +17,7 @@ Cheetah = 2.2.2
ctypes = 1.0.2
DRMAA_python = 0.2
MarkupSafe = 0.12
mercurial = 2.1.2
mercurial = 2.2.3
MySQL_python = 1.2.3c1
numpy = 1.6.0
pbs_python = 4.1.0
lib/galaxy/config.py
+16 -5
View File
@@ -132,7 +132,7 @@ class Configuration( object ):
self.log_events = string_as_bool( kwargs.get( 'log_events', 'False' ) )
self.sanitize_all_html = string_as_bool( kwargs.get( 'sanitize_all_html', True ) )
self.ucsc_display_sites = kwargs.get( 'ucsc_display_sites', "main,test,archaea,ucla" ).lower().split(",")
self.gbrowse_display_sites = kwargs.get( 'gbrowse_display_sites', "wormbase,tair,modencode_worm,modencode_fly,sgd_yeast" ).lower().split(",")
self.gbrowse_display_sites = kwargs.get( 'gbrowse_display_sites', "modencode,sgd_yeast,tair,wormbase,wormbase_ws120,wormbase_ws140,wormbase_ws170,wormbase_ws180,wormbase_ws190,wormbase_ws200,wormbase_ws204,wormbase_ws210,wormbase_ws220,wormbase_ws225" ).lower().split(",")
self.genetrack_display_sites = kwargs.get( 'genetrack_display_sites', "main,test" ).lower().split(",")
self.brand = kwargs.get( 'brand', None )
self.support_url = kwargs.get( 'support_url', 'http://wiki.g2.bx.psu.edu/Support' )
@@ -169,10 +169,21 @@ class Configuration( object ):
if self.nginx_upload_store:
self.nginx_upload_store = os.path.abspath( self.nginx_upload_store )
self.object_store = kwargs.get( 'object_store', 'disk' )
self.aws_access_key = kwargs.get( 'aws_access_key', None )
self.aws_secret_key = kwargs.get( 'aws_secret_key', None )
self.s3_bucket = kwargs.get( 's3_bucket', None)
self.use_reduced_redundancy = kwargs.get( 'use_reduced_redundancy', False )
# Handle AWS-specific config options for backward compatibility
if kwargs.get( 'aws_access_key', None) is not None:
self.os_access_key= kwargs.get( 'aws_access_key', None )
self.os_secret_key= kwargs.get( 'aws_secret_key', None )
self.os_bucket_name= kwargs.get( 's3_bucket', None )
self.os_use_reduced_redundancy = kwargs.get( 'use_reduced_redundancy', False )
else:
self.os_access_key = kwargs.get( 'os_access_key', None )
self.os_secret_key = kwargs.get( 'os_secret_key', None )
self.os_bucket_name = kwargs.get( 'os_bucket_name', None )
self.os_use_reduced_redundancy = kwargs.get( 'os_use_reduced_redundancy', False )
self.os_host = kwargs.get( 'os_host', None )
self.os_port = kwargs.get( 'os_port', None )
self.os_is_secure = string_as_bool( kwargs.get( 'os_is_secure', True ) )
self.os_conn_path = kwargs.get( 'os_conn_path', '/' )
self.object_store_cache_size = float(kwargs.get( 'object_store_cache_size', -1 ))
self.distributed_object_store_config_file = kwargs.get( 'distributed_object_store_config_file', None )
# Parse global_conf and save the parser
lib/galaxy/datatypes/converters/bedgraph_to_bigwig_converter.xml
+14
View File
@@ -0,0 +1,14 @@
<tool id="CONVERTER_bedgraph_to_bigwig" name="Convert BedGraph to BigWig" hidden="true">
<!-- Used internally to generate track indexes -->
<command>grep -v "^track" $input | wigToBigWig -clip stdin $chromInfo $output</command>
<inputs>
<page>
<param format="bedgraph" name="input" type="data" label="Choose wiggle"/>
</page>
</inputs>
<outputs>
<data format="bigwig" name="output"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/gff_to_fli.py
+57
View File
@@ -0,0 +1,57 @@
'''
Creates a feature location index for a given GFF file.
'''
import sys
from galaxy import eggs
from galaxy.datatypes.util.gff_util import read_unordered_gtf, convert_gff_coords_to_bed
def main():
# Process arguments.
in_fname = sys.argv[1]
out_fname = sys.argv[2]
# Create dict of name-location pairings.
name_loc_dict = {}
for feature in read_unordered_gtf( open( in_fname, 'r' ) ):
for name in feature.attributes:
val = feature.attributes[ name ]
try:
float( val )
continue
except:
convert_gff_coords_to_bed( feature )
# Value is not a number, so it can be indexed.
if val not in name_loc_dict:
# Value is not in dictionary.
name_loc_dict[ val ] = {
'contig': feature.chrom,
'start': feature.start,
'end': feature.end
}
else:
# Value already in dictionary, so update dictionary.
loc = name_loc_dict[ val ]
if feature.start < loc[ 'start' ]:
loc[ 'start' ] = feature.start
if feature.end > loc[ 'end' ]:
loc[ 'end' ] = feature.end
# Print name, loc in sorted order.
out = open( out_fname, 'w' )
max_len = 0
entries = []
for name in sorted( name_loc_dict.iterkeys() ):
loc = name_loc_dict[ name ]
entry = '%s\t%s' % ( name, '%s:%i-%i' % ( loc[ 'contig' ], loc[ 'start' ], loc[ 'end' ] ) )
if len( entry ) > max_len:
max_len = len( entry )
entries.append( entry )
out.write( str( max_len + 1 ).ljust( max_len ) + '\n' )
for entry in entries:
out.write( entry.ljust( max_len ) + '\n' )
out.close()
if __name__ == '__main__':
main()
lib/galaxy/datatypes/converters/gff_to_fli_converter.xml
+13
View File
@@ -0,0 +1,13 @@
<tool id="CONVERTER_gff_to_fli_0" name="Convert GFF to Feature Location Index">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<!-- Used on the metadata edit page. -->
<command interpreter="python">gff_to_fli.py $input1 $output1</command>
<inputs>
<param format="gff" name="input1" type="data" label="Choose GFF file"/>
</inputs>
<outputs>
<data format="fli" name="output1"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/interval_to_interval_index_converter.py.orig
-79
View File
@@ -1,79 +0,0 @@
#!/usr/bin/env python
"""
Convert from interval file to interval index file. Default input file format is BED (0-based, half-open intervals).
usage: %prog in_file out_file
-G, --gff: input is GFF format, meaning start and end coordinates are 1-based, closed interval
"""
from __future__ import division
import sys, fileinput
from galaxy import eggs
import pkg_resources; pkg_resources.require( "bx-python" )
from galaxy.visualization.tracks.summary import *
from bx.cookbook import doc_optparse
from galaxy.tools.util.gff_util import convert_gff_coords_to_bed
from bx.interval_index_file import Indexes
from galaxy.tools.util.gff_util import parse_gff_attributes
def main():
# Read options, args.
options, args = doc_optparse.parse( __doc__ )
try:
gff_format = bool( options.gff )
input_fname, out_fname = args
except:
doc_optparse.exception()
# Do conversion.
# TODO: take column numbers from command line.
if gff_format:
chr_col, start_col, end_col = ( 0, 3, 4 )
else:
chr_col, start_col, end_col = ( 0, 1, 2 )
index = Indexes()
offset = 0
# Need to keep track of last gene, transcript id for indexing GTF files.
last_gene_id = None
last_transcript_id = None
for line in open(input_fname, "r"):
feature = line.strip().split('\t')
if not feature or feature[0].startswith("track") or feature[0].startswith("#"):
offset += len(line)
continue
chrom = feature[ chr_col ]
chrom_start = int( feature[ start_col ] )
chrom_end = int( feature[ end_col ] )
if gff_format:
chrom_start, chrom_end = convert_gff_coords_to_bed( [chrom_start, chrom_end ] )
# Only add feature if gene_id, transcript_id are different from last
# values.
if len( feature ) == 9:
attributes = parse_gff_attributes( feature[8] )
gene_id = attributes.get( 'gene_id', None )
transcript_id = attributes.get( 'transcript_id', None )
if gene_id and transcript_id and gene_id == last_gene_id and \
transcript_id == last_transcript_id:
# Feature has same gene_id, transcript as last feature, so
# do not add.
offset += len(line)
continue
else:
# gene_id, transcript_id set and are different from last
# values.
last_gene_id = gene_id
last_transcript_id = transcript_id
#print "%s %s %s %s %i %i %i" % (feature[2], last_gene_id, last_transcript_id, chrom, chrom_start, chrom_end, offset)
index.add( chrom, chrom_start, chrom_end, offset )
offset += len(line)
index.write( open(out_fname, "w") )
if __name__ == "__main__":
main()
lib/galaxy/datatypes/converters/wig_to_bigwig_converter.xml
+1 -1
View File
@@ -1,6 +1,6 @@
<tool id="CONVERTER_wig_to_bigwig" name="Convert Wiggle to BigWig" hidden="true">
<!-- Used internally to generate track indexes -->
<command>wigToBigWig $input $chromInfo $output</command>
<command>grep -v "^track" $input | wigToBigWig -clip stdin $chromInfo $output</command>
<inputs>
<page>
<param format="wig" name="input" type="data" label="Choose wiggle"/>
lib/galaxy/datatypes/data.py
+43 -1
View File
@@ -719,7 +719,49 @@ class LineCount( Text ):
pass
class Newick( Text ):
pass
"""New Hampshire/Newick Format"""
file_ext = "nhx"
MetadataElement( name="columns", default=3, desc="Number of columns", readonly=True )
def __init__(self, **kwd):
"""Initialize foobar datatype"""
Text.__init__(self, **kwd)
def init_meta( self, dataset, copy_from=None ):
Text.init_meta( self, dataset, copy_from=copy_from )
def sniff( self, filename ):
""" Returning false as the newick format is too general and cannot be sniffed."""
return False
class Nexus( Text ):
"""Nexus format as used By Paup, Mr Bayes, etc"""
file_ext = "nex"
MetadataElement( name="columns", default=3, desc="Number of columns", readonly=True )
def __init__(self, **kwd):
"""Initialize foobar datatype"""
Text.__init__(self, **kwd)
def init_meta( self, dataset, copy_from=None ):
Text.init_meta( self, dataset, copy_from=copy_from )
def sniff( self, filename ):
"""All Nexus Files Simply puts a '#NEXUS' in its first line"""
f = open(filename, "r")
firstline = f.readline().upper()
f.close()
if "#NEXUS" in firstline:
return True
else:
return False
# ------------- Utility methods --------------
lib/galaxy/datatypes/display_applications/application.py
+5 -1
View File
@@ -98,8 +98,10 @@ class DynamicDisplayApplicationBuilder( object ):
for line in open( filename ):
if not skip_startswith or not line.startswith( skip_startswith ):
line = line.rstrip( '\n\r' )
if not line:
continue
fields = line.split( separator )
if len( fields ) >= max_col:
if len( fields ) > max_col:
new_elem = deepcopy( elem )
new_elem.set( 'id', fields[id_col] )
new_elem.set( 'name', fields[name_col] )
@@ -111,6 +113,8 @@ class DynamicDisplayApplicationBuilder( object ):
dynamic_values[key] = value
#now populate
rval.append( DisplayApplicationLink.from_elem( new_elem, display_application, other_values = dynamic_values ) )
else:
log.warning( 'Invalid dynamic display application link specified in %s: "%s"' % ( filename, line ) )
self.links = rval
def __iter__( self ):
return iter( self.links )
lib/galaxy/datatypes/interval.py
+3 -2
View File
@@ -338,7 +338,7 @@ class BedGraph( Interval ):
file_ext = "bedgraph"
def get_track_type( self ):
return "LineTrack", {"data": "array_tree"}
return "LineTrack", { "data": "bigwig", "index": "bigwig" }
def as_ucsc_display_file( self, dataset, **kwd ):
"""
@@ -1141,8 +1141,9 @@ class Wiggle( Tabular, _RemoteCallMixin ):
resolution = min( resolution, 100000 )
resolution = max( resolution, 1 )
return resolution
def get_track_type( self ):
return "LineTrack", {"data": "bigwig", "index": "bigwig"}
return "LineTrack", { "data": "bigwig", "index": "bigwig" }
class CustomTrack ( Tabular ):
"""UCSC CustomTrack"""
lib/galaxy/datatypes/registry.py
+6 -10
View File
@@ -1,7 +1,7 @@
"""
Provides mapping between extensions and datatypes, mime-types, etc.
"""
import os, sys, tempfile, threading, logging
import os, sys, tempfile, threading, logging, imp
import data, tabular, interval, images, sequence, qualityscore, genetics, xml, coverage, tracks, chrominfo, binary, assembly, ngsindex
import galaxy.util
from galaxy.util.odict import odict
@@ -55,10 +55,9 @@ class Registry( object ):
being installed. Since installation is occurring after the datatypes registry has been initialized, its
contents cannot be overridden by new introduced conflicting data types.
"""
def __import_module( full_path, datatype_module ):
sys.path.insert( 0, full_path )
imported_module = __import__( datatype_module )
sys.path.pop( 0 )
def __import_module( full_path, datatype_module, datatype_class_name ):
open_file_obj, file_name, description = imp.find_module( datatype_module, [ full_path ] )
imported_module = imp.load_module( datatype_class_name, open_file_obj, file_name, description )
return imported_module
if root_dir and config:
handling_proprietary_datatypes = False
@@ -130,12 +129,12 @@ class Registry( object ):
datatype_module = fields[0]
datatype_class_name = fields[1]
datatype_class = None
if proprietary_path and proprietary_datatype_module:
if proprietary_path and proprietary_datatype_module and datatype_class_name:
# We need to change the value of sys.path, so do it in a way that is thread-safe.
lock = threading.Lock()
lock.acquire( True )
try:
imported_module = __import_module( proprietary_path, proprietary_datatype_module )
imported_module = __import_module( proprietary_path, proprietary_datatype_module, datatype_class_name )
if imported_module not in self.imported_modules:
self.imported_modules.append( imported_module )
if hasattr( imported_module, datatype_class_name ):
@@ -276,7 +275,6 @@ class Registry( object ):
'axt' : sequence.Axt(),
'bam' : binary.Bam(),
'bed' : interval.Bed(),
'blastxml' : xml.BlastXml(),
'coverage' : coverage.LastzCoverage(),
'customtrack' : interval.CustomTrack(),
'csfasta' : sequence.csFasta(),
@@ -310,7 +308,6 @@ class Registry( object ):
'axt' : 'text/plain',
'bam' : 'application/octet-stream',
'bed' : 'text/plain',
'blastxml' : 'application/xml',
'customtrack' : 'text/plain',
'csfasta' : 'text/plain',
'eland' : 'application/octet-stream',
@@ -348,7 +345,6 @@ class Registry( object ):
self.sniff_order = [
binary.Bam(),
binary.Sff(),
xml.BlastXml(),
xml.GenericXml(),
sequence.Maf(),
sequence.Lav(),
lib/galaxy/datatypes/sniff.py
+15 -16
View File
@@ -6,6 +6,7 @@ import registry
from galaxy import util
from galaxy.datatypes.checkers import *
from galaxy.datatypes.binary import unsniffable_binary_formats
from encodings import search_function as encodings_search_function
log = logging.getLogger(__name__)
@@ -15,7 +16,7 @@ def get_test_fname(fname):
full_path = os.path.join(path, 'test', fname)
return full_path
def stream_to_open_named_file( stream, fd, filename ):
def stream_to_open_named_file( stream, fd, filename, source_encoding=None, source_error='strict', target_encoding=None, target_error='strict' ):
"""Writes a stream to the provided file descriptor, returns the file's name and bool( is_multi_byte ). Closes file descriptor"""
#signature and behavor is somewhat odd, due to backwards compatibility, but this can/should be done better
CHUNK_SIZE = 1048576
@@ -23,6 +24,10 @@ def stream_to_open_named_file( stream, fd, filename ):
is_compressed = False
is_binary = False
is_multi_byte = False
if not target_encoding or not encodings_search_function( target_encoding ):
target_encoding = util.DEFAULT_ENCODING #utf-8
if not source_encoding:
source_encoding = util.DEFAULT_ENCODING #sys.getdefaultencoding() would mimic old behavior (defaults to ascii)
while 1:
chunk = stream.read( CHUNK_SIZE )
if not chunk:
@@ -42,13 +47,12 @@ def stream_to_open_named_file( stream, fd, filename ):
chars = chunk[:100]
is_multi_byte = util.is_multi_byte( chars )
if not is_multi_byte:
for char in chars:
if ord( char ) > 128:
is_binary = True
break
is_binary = util.is_binary( chunk )
data_checked = True
if not is_compressed and not is_binary:
os.write( fd, chunk.encode( "utf-8" ) )
if not isinstance( chunk, unicode ):
chunk = chunk.decode( source_encoding, source_error )
os.write( fd, chunk.encode( target_encoding, target_error ) )
else:
# Compressed files must be encoded after they are uncompressed in the upload utility,
# while binary files should not be encoded at all.
@@ -56,10 +60,10 @@ def stream_to_open_named_file( stream, fd, filename ):
os.close( fd )
return filename, is_multi_byte
def stream_to_file( stream, suffix='', prefix='', dir=None, text=False ):
def stream_to_file( stream, suffix='', prefix='', dir=None, text=False, **kwd ):
"""Writes a stream to a temporary file, returns the temporary file's name"""
fd, temp_name = tempfile.mkstemp( suffix=suffix, prefix=prefix, dir=dir, text=text )
return stream_to_open_named_file( stream, fd, temp_name )
return stream_to_open_named_file( stream, fd, temp_name, **kwd )
def check_newlines( fname, bytes_to_read=52428800 ):
"""
@@ -305,14 +309,9 @@ def guess_ext( fname, sniff_order=None, is_multi_byte=False ):
else:
for hdr in headers:
for char in hdr:
if len( char ) > 1:
for c in char:
if ord( c ) > 128:
is_binary = True
break
elif ord( char ) > 128:
is_binary = True
break
#old behavior had 'char' possibly having length > 1,
#need to determine when/if this occurs
is_binary = util.is_binary( char )
if is_binary:
break
if is_binary:
lib/galaxy/datatypes/tabular.py
+10 -2
View File
@@ -264,10 +264,10 @@ class Tabular( data.Text ):
def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, chunk=None):
#TODO Prevent failure when displaying extremely long > 50kb lines.
if to_ext or not preview:
return self._serve_raw(trans, dataset, to_ext)
if chunk:
return self.get_chunk(trans, dataset, chunk)
if to_ext or not preview:
return self._serve_raw(trans, dataset, to_ext)
else:
column_names = 'null'
if dataset.metadata.column_names:
@@ -638,3 +638,11 @@ class Eland( Tabular ):
dataset.metadata.reads = reads.keys()
class FeatureLocationIndex( Tabular ):
"""
An index that stores feature locations in tabular format.
"""
file_ext='fli'
MetadataElement( name="columns", default=2, desc="Number of columns", readonly=True, visible=False )
MetadataElement( name="column_types", default=['str', 'str'], param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False, no_value=[] )
lib/galaxy/datatypes/test/tblastn_four_human_vs_rhodopsin.xml
-722
View File
@@ -1,722 +0,0 @@
<?xml version="1.0"?>
<!DOCTYPE BlastOutput PUBLIC "-//NCBI//NCBI BlastOutput/EN" "NCBI_BlastOutput.dtd">
<BlastOutput>
<BlastOutput_program>tblastn</BlastOutput_program>
<BlastOutput_version>TBLASTN 2.2.25+</BlastOutput_version>
<BlastOutput_reference>Stephen F. Altschul, Thomas L. Madden, Alejandro A. Sch&amp;auml;ffer, Jinghui Zhang, Zheng Zhang, Webb Miller, and David J. Lipman (1997), &quot;Gapped BLAST and PSI-BLAST: a new generation of protein database search programs&quot;, Nucleic Acids Res. 25:3389-3402.</BlastOutput_reference>
<BlastOutput_db></BlastOutput_db>
<BlastOutput_query-ID>Query_1</BlastOutput_query-ID>
<BlastOutput_query-def>sp|Q9BS26|ERP44_HUMAN Endoplasmic reticulum resident protein 44 OS=Homo sapiens GN=ERP44 PE=1 SV=1</BlastOutput_query-def>
<BlastOutput_query-len>406</BlastOutput_query-len>
<BlastOutput_param>
<Parameters>
<Parameters_matrix>BLOSUM80</Parameters_matrix>
<Parameters_expect>1e-10</Parameters_expect>
<Parameters_gap-open>10</Parameters_gap-open>
<Parameters_gap-extend>1</Parameters_gap-extend>
<Parameters_filter>F</Parameters_filter>
</Parameters>
</BlastOutput_param>
<BlastOutput_iterations>
<Iteration>
<Iteration_iter-num>1</Iteration_iter-num>
<Iteration_query-ID>Query_1</Iteration_query-ID>
<Iteration_query-def>sp|Q9BS26|ERP44_HUMAN Endoplasmic reticulum resident protein 44 OS=Homo sapiens GN=ERP44 PE=1 SV=1</Iteration_query-def>
<Iteration_query-len>406</Iteration_query-len>
<Iteration_hits></Iteration_hits>
<Iteration_stat>
<Statistics>
<Statistics_db-num>0</Statistics_db-num>
<Statistics_db-len>0</Statistics_db-len>
<Statistics_hsp-len>19</Statistics_hsp-len>
<Statistics_eff-space>127710</Statistics_eff-space>
<Statistics_kappa>0.071</Statistics_kappa>
<Statistics_lambda>0.299</Statistics_lambda>
<Statistics_entropy>0.27</Statistics_entropy>
</Statistics>
</Iteration_stat>
<Iteration_message>No hits found</Iteration_message>
</Iteration>
<Iteration>
<Iteration_iter-num>2</Iteration_iter-num>
<Iteration_query-ID>Query_1</Iteration_query-ID>
<Iteration_query-def>sp|Q9BS26|ERP44_HUMAN Endoplasmic reticulum resident protein 44 OS=Homo sapiens GN=ERP44 PE=1 SV=1</Iteration_query-def>
<Iteration_query-len>406</Iteration_query-len>
<Iteration_hits></Iteration_hits>
<Iteration_stat>
<Statistics>
<Statistics_db-num>0</Statistics_db-num>
<Statistics_db-len>0</Statistics_db-len>
<Statistics_hsp-len>19</Statistics_hsp-len>
<Statistics_eff-space>127710</Statistics_eff-space>
<Statistics_kappa>0.071</Statistics_kappa>
<Statistics_lambda>0.299</Statistics_lambda>
<Statistics_entropy>0.27</Statistics_entropy>
</Statistics>
</Iteration_stat>
<Iteration_message>No hits found</Iteration_message>
</Iteration>
<Iteration>
<Iteration_iter-num>3</Iteration_iter-num>
<Iteration_query-ID>Query_1</Iteration_query-ID>
<Iteration_query-def>sp|Q9BS26|ERP44_HUMAN Endoplasmic reticulum resident protein 44 OS=Homo sapiens GN=ERP44 PE=1 SV=1</Iteration_query-def>
<Iteration_query-len>406</Iteration_query-len>
<Iteration_hits></Iteration_hits>
<Iteration_stat>
<Statistics>
<Statistics_db-num>0</Statistics_db-num>
<Statistics_db-len>0</Statistics_db-len>
<Statistics_hsp-len>19</Statistics_hsp-len>
<Statistics_eff-space>127710</Statistics_eff-space>
<Statistics_kappa>0.071</Statistics_kappa>
<Statistics_lambda>0.299</Statistics_lambda>
<Statistics_entropy>0.27</Statistics_entropy>
</Statistics>
</Iteration_stat>
<Iteration_message>No hits found</Iteration_message>
</Iteration>
<Iteration>
<Iteration_iter-num>4</Iteration_iter-num>
<Iteration_query-ID>Query_1</Iteration_query-ID>
<Iteration_query-def>sp|Q9BS26|ERP44_HUMAN Endoplasmic reticulum resident protein 44 OS=Homo sapiens GN=ERP44 PE=1 SV=1</Iteration_query-def>
<Iteration_query-len>406</Iteration_query-len>
<Iteration_hits></Iteration_hits>
<Iteration_stat>
<Statistics>
<Statistics_db-num>0</Statistics_db-num>
<Statistics_db-len>0</Statistics_db-len>
<Statistics_hsp-len>19</Statistics_hsp-len>
<Statistics_eff-space>127710</Statistics_eff-space>
<Statistics_kappa>0.071</Statistics_kappa>
<Statistics_lambda>0.299</Statistics_lambda>
<Statistics_entropy>0.27</Statistics_entropy>
</Statistics>
</Iteration_stat>
<Iteration_message>No hits found</Iteration_message>
</Iteration>
<Iteration>
<Iteration_iter-num>5</Iteration_iter-num>
<Iteration_query-ID>Query_1</Iteration_query-ID>
<Iteration_query-def>sp|Q9BS26|ERP44_HUMAN Endoplasmic reticulum resident protein 44 OS=Homo sapiens GN=ERP44 PE=1 SV=1</Iteration_query-def>
<Iteration_query-len>406</Iteration_query-len>
<Iteration_hits></Iteration_hits>
<Iteration_stat>
<Statistics>
<Statistics_db-num>0</Statistics_db-num>
<Statistics_db-len>0</Statistics_db-len>
<Statistics_hsp-len>19</Statistics_hsp-len>
<Statistics_eff-space>127710</Statistics_eff-space>
<Statistics_kappa>0.071</Statistics_kappa>
<Statistics_lambda>0.299</Statistics_lambda>
<Statistics_entropy>0.27</Statistics_entropy>
</Statistics>
</Iteration_stat>
<Iteration_message>No hits found</Iteration_message>
</Iteration>
<Iteration>
<Iteration_iter-num>6</Iteration_iter-num>
<Iteration_query-ID>Query_1</Iteration_query-ID>
<Iteration_query-def>sp|Q9BS26|ERP44_HUMAN Endoplasmic reticulum resident protein 44 OS=Homo sapiens GN=ERP44 PE=1 SV=1</Iteration_query-def>
<Iteration_query-len>406</Iteration_query-len>
<Iteration_hits></Iteration_hits>
<Iteration_stat>
<Statistics>
<Statistics_db-num>0</Statistics_db-num>
<Statistics_db-len>0</Statistics_db-len>
<Statistics_hsp-len>19</Statistics_hsp-len>
<Statistics_eff-space>127710</Statistics_eff-space>
<Statistics_kappa>0.071</Statistics_kappa>
<Statistics_lambda>0.299</Statistics_lambda>
<Statistics_entropy>0.27</Statistics_entropy>
</Statistics>
</Iteration_stat>
<Iteration_message>No hits found</Iteration_message>
</Iteration>
<Iteration>
<Iteration_iter-num>7</Iteration_iter-num>
<Iteration_query-ID>Query_2</Iteration_query-ID>
<Iteration_query-def>sp|Q9NSY1|BMP2K_HUMAN BMP-2-inducible protein kinase OS=Homo sapiens GN=BMP2K PE=1 SV=2</Iteration_query-def>
<Iteration_query-len>1161</Iteration_query-len>
<Iteration_hits></Iteration_hits>
<Iteration_stat>
<Statistics>
<Statistics_db-num>0</Statistics_db-num>
<Statistics_db-len>0</Statistics_db-len>
<Statistics_hsp-len>23</Statistics_hsp-len>
<Statistics_eff-space>370988</Statistics_eff-space>
<Statistics_kappa>0.071</Statistics_kappa>
<Statistics_lambda>0.299</Statistics_lambda>
<Statistics_entropy>0.27</Statistics_entropy>
</Statistics>
</Iteration_stat>
<Iteration_message>No hits found</Iteration_message>
</Iteration>
<Iteration>
<Iteration_iter-num>8</Iteration_iter-num>
<Iteration_query-ID>Query_2</Iteration_query-ID>
<Iteration_query-def>sp|Q9NSY1|BMP2K_HUMAN BMP-2-inducible protein kinase OS=Homo sapiens GN=BMP2K PE=1 SV=2</Iteration_query-def>
<Iteration_query-len>1161</Iteration_query-len>
<Iteration_hits></Iteration_hits>
<Iteration_stat>
<Statistics>
<Statistics_db-num>0</Statistics_db-num>
<Statistics_db-len>0</Statistics_db-len>
<Statistics_hsp-len>23</Statistics_hsp-len>
<Statistics_eff-space>370988</Statistics_eff-space>
<Statistics_kappa>0.071</Statistics_kappa>
<Statistics_lambda>0.299</Statistics_lambda>
<Statistics_entropy>0.27</Statistics_entropy>
</Statistics>
</Iteration_stat>
<Iteration_message>No hits found</Iteration_message>
</Iteration>
<Iteration>
<Iteration_iter-num>9</Iteration_iter-num>
<Iteration_query-ID>Query_2</Iteration_query-ID>
<Iteration_query-def>sp|Q9NSY1|BMP2K_HUMAN BMP-2-inducible protein kinase OS=Homo sapiens GN=BMP2K PE=1 SV=2</Iteration_query-def>
<Iteration_query-len>1161</Iteration_query-len>
<Iteration_hits></Iteration_hits>
<Iteration_stat>
<Statistics>
<Statistics_db-num>0</Statistics_db-num>
<Statistics_db-len>0</Statistics_db-len>
<Statistics_hsp-len>23</Statistics_hsp-len>
<Statistics_eff-space>370988</Statistics_eff-space>
<Statistics_kappa>0.071</Statistics_kappa>
<Statistics_lambda>0.299</Statistics_lambda>
<Statistics_entropy>0.27</Statistics_entropy>
</Statistics>
</Iteration_stat>
<Iteration_message>No hits found</Iteration_message>
</Iteration>
<Iteration>
<Iteration_iter-num>10</Iteration_iter-num>
<Iteration_query-ID>Query_2</Iteration_query-ID>
<Iteration_query-def>sp|Q9NSY1|BMP2K_HUMAN BMP-2-inducible protein kinase OS=Homo sapiens GN=BMP2K PE=1 SV=2</Iteration_query-def>
<Iteration_query-len>1161</Iteration_query-len>
<Iteration_hits></Iteration_hits>
<Iteration_stat>
<Statistics>
<Statistics_db-num>0</Statistics_db-num>
<Statistics_db-len>0</Statistics_db-len>
<Statistics_hsp-len>23</Statistics_hsp-len>
<Statistics_eff-space>370988</Statistics_eff-space>
<Statistics_kappa>0.071</Statistics_kappa>
<Statistics_lambda>0.299</Statistics_lambda>
<Statistics_entropy>0.27</Statistics_entropy>
</Statistics>
</Iteration_stat>
<Iteration_message>No hits found</Iteration_message>
</Iteration>
<Iteration>
<Iteration_iter-num>11</Iteration_iter-num>
<Iteration_query-ID>Query_2</Iteration_query-ID>
<Iteration_query-def>sp|Q9NSY1|BMP2K_HUMAN BMP-2-inducible protein kinase OS=Homo sapiens GN=BMP2K PE=1 SV=2</Iteration_query-def>
<Iteration_query-len>1161</Iteration_query-len>
<Iteration_hits></Iteration_hits>
<Iteration_stat>
<Statistics>
<Statistics_db-num>0</Statistics_db-num>
<Statistics_db-len>0</Statistics_db-len>
<Statistics_hsp-len>23</Statistics_hsp-len>
<Statistics_eff-space>370988</Statistics_eff-space>
<Statistics_kappa>0.071</Statistics_kappa>
<Statistics_lambda>0.299</Statistics_lambda>
<Statistics_entropy>0.27</Statistics_entropy>
</Statistics>
</Iteration_stat>
<Iteration_message>No hits found</Iteration_message>
</Iteration>
<Iteration>
<Iteration_iter-num>12</Iteration_iter-num>
<Iteration_query-ID>Query_2</Iteration_query-ID>
<Iteration_query-def>sp|Q9NSY1|BMP2K_HUMAN BMP-2-inducible protein kinase OS=Homo sapiens GN=BMP2K PE=1 SV=2</Iteration_query-def>
<Iteration_query-len>1161</Iteration_query-len>
<Iteration_hits></Iteration_hits>
<Iteration_stat>
<Statistics>
<Statistics_db-num>0</Statistics_db-num>
<Statistics_db-len>0</Statistics_db-len>
<Statistics_hsp-len>23</Statistics_hsp-len>
<Statistics_eff-space>370988</Statistics_eff-space>
<Statistics_kappa>0.071</Statistics_kappa>
<Statistics_lambda>0.299</Statistics_lambda>
<Statistics_entropy>0.27</Statistics_entropy>
</Statistics>
</Iteration_stat>
<Iteration_message>No hits found</Iteration_message>
</Iteration>
<Iteration>
<Iteration_iter-num>13</Iteration_iter-num>
<Iteration_query-ID>Query_3</Iteration_query-ID>
<Iteration_query-def>sp|P06213|INSR_HUMAN Insulin receptor OS=Homo sapiens GN=INSR PE=1 SV=4</Iteration_query-def>
<Iteration_query-len>1382</Iteration_query-len>
<Iteration_hits></Iteration_hits>
<Iteration_stat>
<Statistics>
<Statistics_db-num>0</Statistics_db-num>
<Statistics_db-len>0</Statistics_db-len>
<Statistics_hsp-len>24</Statistics_hsp-len>
<Statistics_eff-space>441350</Statistics_eff-space>
<Statistics_kappa>0.071</Statistics_kappa>
<Statistics_lambda>0.299</Statistics_lambda>
<Statistics_entropy>0.27</Statistics_entropy>
</Statistics>
</Iteration_stat>
<Iteration_message>No hits found</Iteration_message>
</Iteration>
<Iteration>
<Iteration_iter-num>14</Iteration_iter-num>
<Iteration_query-ID>Query_3</Iteration_query-ID>
<Iteration_query-def>sp|P06213|INSR_HUMAN Insulin receptor OS=Homo sapiens GN=INSR PE=1 SV=4</Iteration_query-def>
<Iteration_query-len>1382</Iteration_query-len>
<Iteration_hits></Iteration_hits>
<Iteration_stat>
<Statistics>
<Statistics_db-num>0</Statistics_db-num>
<Statistics_db-len>0</Statistics_db-len>
<Statistics_hsp-len>24</Statistics_hsp-len>
<Statistics_eff-space>441350</Statistics_eff-space>
<Statistics_kappa>0.071</Statistics_kappa>
<Statistics_lambda>0.299</Statistics_lambda>
<Statistics_entropy>0.27</Statistics_entropy>
</Statistics>
</Iteration_stat>
<Iteration_message>No hits found</Iteration_message>
</Iteration>
<Iteration>
<Iteration_iter-num>15</Iteration_iter-num>
<Iteration_query-ID>Query_3</Iteration_query-ID>
<Iteration_query-def>sp|P06213|INSR_HUMAN Insulin receptor OS=Homo sapiens GN=INSR PE=1 SV=4</Iteration_query-def>
<Iteration_query-len>1382</Iteration_query-len>
<Iteration_hits></Iteration_hits>
<Iteration_stat>
<Statistics>
<Statistics_db-num>0</Statistics_db-num>
<Statistics_db-len>0</Statistics_db-len>
<Statistics_hsp-len>24</Statistics_hsp-len>
<Statistics_eff-space>441350</Statistics_eff-space>
<Statistics_kappa>0.071</Statistics_kappa>
<Statistics_lambda>0.299</Statistics_lambda>
<Statistics_entropy>0.27</Statistics_entropy>
</Statistics>
</Iteration_stat>
<Iteration_message>No hits found</Iteration_message>
</Iteration>
<Iteration>
<Iteration_iter-num>16</Iteration_iter-num>
<Iteration_query-ID>Query_3</Iteration_query-ID>
<Iteration_query-def>sp|P06213|INSR_HUMAN Insulin receptor OS=Homo sapiens GN=INSR PE=1 SV=4</Iteration_query-def>
<Iteration_query-len>1382</Iteration_query-len>
<Iteration_hits></Iteration_hits>
<Iteration_stat>
<Statistics>
<Statistics_db-num>0</Statistics_db-num>
<Statistics_db-len>0</Statistics_db-len>
<Statistics_hsp-len>24</Statistics_hsp-len>
<Statistics_eff-space>441350</Statistics_eff-space>
<Statistics_kappa>0.071</Statistics_kappa>
<Statistics_lambda>0.299</Statistics_lambda>
<Statistics_entropy>0.27</Statistics_entropy>
</Statistics>
</Iteration_stat>
<Iteration_message>No hits found</Iteration_message>
</Iteration>
<Iteration>
<Iteration_iter-num>17</Iteration_iter-num>
<Iteration_query-ID>Query_3</Iteration_query-ID>
<Iteration_query-def>sp|P06213|INSR_HUMAN Insulin receptor OS=Homo sapiens GN=INSR PE=1 SV=4</Iteration_query-def>
<Iteration_query-len>1382</Iteration_query-len>
<Iteration_hits></Iteration_hits>
<Iteration_stat>
<Statistics>
<Statistics_db-num>0</Statistics_db-num>
<Statistics_db-len>0</Statistics_db-len>
<Statistics_hsp-len>24</Statistics_hsp-len>
<Statistics_eff-space>441350</Statistics_eff-space>
<Statistics_kappa>0.071</Statistics_kappa>
<Statistics_lambda>0.299</Statistics_lambda>
<Statistics_entropy>0.27</Statistics_entropy>
</Statistics>
</Iteration_stat>
<Iteration_message>No hits found</Iteration_message>
</Iteration>
<Iteration>
<Iteration_iter-num>18</Iteration_iter-num>
<Iteration_query-ID>Query_3</Iteration_query-ID>
<Iteration_query-def>sp|P06213|INSR_HUMAN Insulin receptor OS=Homo sapiens GN=INSR PE=1 SV=4</Iteration_query-def>
<Iteration_query-len>1382</Iteration_query-len>
<Iteration_hits></Iteration_hits>
<Iteration_stat>
<Statistics>
<Statistics_db-num>0</Statistics_db-num>
<Statistics_db-len>0</Statistics_db-len>
<Statistics_hsp-len>24</Statistics_hsp-len>
<Statistics_eff-space>441350</Statistics_eff-space>
<Statistics_kappa>0.071</Statistics_kappa>
<Statistics_lambda>0.299</Statistics_lambda>
<Statistics_entropy>0.27</Statistics_entropy>
</Statistics>
</Iteration_stat>
<Iteration_message>No hits found</Iteration_message>
</Iteration>
<Iteration>
<Iteration_iter-num>19</Iteration_iter-num>
<Iteration_query-ID>Query_4</Iteration_query-ID>
<Iteration_query-def>sp|P08100|OPSD_HUMAN Rhodopsin OS=Homo sapiens GN=RHO PE=1 SV=1</Iteration_query-def>
<Iteration_query-len>348</Iteration_query-len>
<Iteration_hits>
<Hit>
<Hit_num>1</Hit_num>
<Hit_id>Subject_1</Hit_id>
<Hit_def>gi|57163782|ref|NM_001009242.1| Felis catus rhodopsin (RHO), mRNA</Hit_def>
<Hit_accession>Subject_1</Hit_accession>
<Hit_len>1047</Hit_len>
<Hit_hsps>
<Hsp>
<Hsp_num>1</Hsp_num>
<Hsp_bit-score>732.392902459534</Hsp_bit-score>
<Hsp_score>1689</Hsp_score>
<Hsp_evalue>0</Hsp_evalue>
<Hsp_query-from>1</Hsp_query-from>
<Hsp_query-to>348</Hsp_query-to>
<Hsp_hit-from>1</Hsp_hit-from>
<Hsp_hit-to>1044</Hsp_hit-to>
<Hsp_query-frame>0</Hsp_query-frame>
<Hsp_hit-frame>1</Hsp_hit-frame>
<Hsp_identity>336</Hsp_identity>
<Hsp_positive>343</Hsp_positive>
<Hsp_gaps>0</Hsp_gaps>
<Hsp_align-len>348</Hsp_align-len>
<Hsp_qseq>MNGTEGPNFYVPFSNATGVVRSPFEYPQYYLAEPWQFSMLAAYMFLLIVLGFPINFLTLYVTVQHKKLRTPLNYILLNLAVADLFMVLGGFTSTLYTSLHGYFVFGPTGCNLEGFFATLGGEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMGVAFTWVMALACAAPPLAGWSRYIPEGLQCSCGIDYYTLKPEVNNESFVIYMFVVHFTIPMIIIFFCYGQLVFTVKEAAAQQQESATTQKAEKEVTRMVIIMVIAFLICWVPYASVAFYIFTHQGSNFGPIFMTIPAFFAKSAAIYNPVIYIMMNKQFRNCMLTTICCGKNPLGDDEASATVSKTETSQVAPA</Hsp_qseq>
<Hsp_hseq>MNGTEGPNFYVPFSNKTGVVRSPFEYPQYYLAEPWQFSMLAAYMFLLIVLGFPINFLTLYVTVQHKKLRTPLNYILLNLAVADLFMVFGGFTTTLYTSLHGYFVFGPTGCNLEGFFATLGGEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMGVAFTWVMALACAAPPLVGWSRYIPEGMQCSCGIDYYTLKPEVNNESFVIYMFVVHFTIPMIVIFFCYGQLVFTVKEAAAQQQESATTQKAEKEVTRMVIIMVIAFLICWVPYASVAFYIFTHQGSNFGPIFMTLPAFFAKSSSIYNPVIYIMMNKQFRNCMLTTLCCGKNPLGDDEASTTGSKTETSQVAPA</Hsp_hseq>
<Hsp_midline>MNGTEGPNFYVPFSN TGVVRSPFEYPQYYLAEPWQFSMLAAYMFLLIVLGFPINFLTLYVTVQHKKLRTPLNYILLNLAVADLFMV GGFT+TLYTSLHGYFVFGPTGCNLEGFFATLGGEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMGVAFTWVMALACAAPPL GWSRYIPEG+QCSCGIDYYTLKPEVNNESFVIYMFVVHFTIPMI+IFFCYGQLVFTVKEAAAQQQESATTQKAEKEVTRMVIIMVIAFLICWVPYASVAFYIFTHQGSNFGPIFMT+PAFFAKS++IYNPVIYIMMNKQFRNCMLTT+CCGKNPLGDDEAS T SKTETSQVAPA</Hsp_midline>
</Hsp>
</Hit_hsps>
</Hit>
</Iteration_hits>
<Iteration_stat>
<Statistics>
<Statistics_db-num>0</Statistics_db-num>
<Statistics_db-len>0</Statistics_db-len>
<Statistics_hsp-len>18</Statistics_hsp-len>
<Statistics_eff-space>109230</Statistics_eff-space>
<Statistics_kappa>0.071</Statistics_kappa>
<Statistics_lambda>0.299</Statistics_lambda>
<Statistics_entropy>0.27</Statistics_entropy>
</Statistics>
</Iteration_stat>
</Iteration>
<Iteration>
<Iteration_iter-num>20</Iteration_iter-num>
<Iteration_query-ID>Query_4</Iteration_query-ID>
<Iteration_query-def>sp|P08100|OPSD_HUMAN Rhodopsin OS=Homo sapiens GN=RHO PE=1 SV=1</Iteration_query-def>
<Iteration_query-len>348</Iteration_query-len>
<Iteration_hits>
<Hit>
<Hit_num>1</Hit_num>
<Hit_id>Subject_2</Hit_id>
<Hit_def>gi|2734705|gb|U59921.1|BBU59921 Bufo bufo rhodopsin mRNA, complete cds</Hit_def>
<Hit_accession>Subject_2</Hit_accession>
<Hit_len>1574</Hit_len>
<Hit_hsps>
<Hsp>
<Hsp_num>1</Hsp_num>
<Hsp_bit-score>646.119739014374</Hsp_bit-score>
<Hsp_score>1489</Hsp_score>
<Hsp_evalue>0</Hsp_evalue>
<Hsp_query-from>1</Hsp_query-from>
<Hsp_query-to>341</Hsp_query-to>
<Hsp_hit-from>42</Hsp_hit-from>
<Hsp_hit-to>1067</Hsp_hit-to>
<Hsp_query-frame>0</Hsp_query-frame>
<Hsp_hit-frame>3</Hsp_hit-frame>
<Hsp_identity>290</Hsp_identity>
<Hsp_positive>320</Hsp_positive>
<Hsp_gaps>1</Hsp_gaps>
<Hsp_align-len>342</Hsp_align-len>
<Hsp_qseq>MNGTEGPNFYVPFSNATGVVRSPFEYPQYYLAEPWQFSMLAAYMFLLIVLGFPINFLTLYVTVQHKKLRTPLNYILLNLAVADLFMVLGGFTSTLYTSLHGYFVFGPTGCNLEGFFATLGGEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMGVAFTWVMALACAAPPLAGWSRYIPEGLQCSCGIDYYTLKPEVNNESFVIYMFVVHFTIPMIIIFFCYGQLVFTVKEAAAQQQESATTQKAEKEVTRMVIIMVIAFLICWVPYASVAFYIFTHQGSNFGPIFMTIPAFFAKSAAIYNPVIYIMMNKQFRNCMLTTICCGKNPLGDDEA-SATVSKTE</Hsp_qseq>
<Hsp_hseq>MNGTEGPNFYIPMSNKTGVVRSPFEYPQYYLAEPWQYSILCAYMFLLILLGFPINFMTLYVTIQHKKLRTPLNYILLNLAFANHFMVLCGFTVTMYSSMNGYFILGATGCYVEGFFATLGGEIALWSLVVLAIERYVVVCKPMSNFRFSENHAVMGVAFTWIMALSCAVPPLLGWSRYIPEGMQCSCGVDYYTLKPEVNNESFVIYMFVVHFTIPLIIIFFCYGRLVCTVKEAAAQQQESATTQKAEKEVTRMVIIMVVFFLICWVPYASVAFFIFSNQGSEFGPIFMTVPAFFAKSSSIYNPVIYIMLNKQFRNCMITTLCCGKNPFGEDDASSAATSKTE</Hsp_hseq>
<Hsp_midline>MNGTEGPNFY+P SN TGVVRSPFEYPQYYLAEPWQ+S+L AYMFLLI+LGFPINF+TLYVT+QHKKLRTPLNYILLNLA A+ FMVL GFT T+Y+S+ GYF+ G TGC +EGFFATLGGEIALWSLVVLAIERYVVVCKPMSNFRF ENHA+MGVAFTW+MAL+CA PPL GWSRYIPEG+QCSCG+DYYTLKPEVNNESFVIYMFVVHFTIP+IIIFFCYG+LV TVKEAAAQQQESATTQKAEKEVTRMVIIMV+ FLICWVPYASVAF+IF+ QGS FGPIFMT+PAFFAKS++IYNPVIYIM+NKQFRNCM+TT+CCGKNP G+D+A SA SKTE</Hsp_midline>
</Hsp>
</Hit_hsps>
</Hit>
</Iteration_hits>
<Iteration_stat>
<Statistics>
<Statistics_db-num>0</Statistics_db-num>
<Statistics_db-len>0</Statistics_db-len>
<Statistics_hsp-len>18</Statistics_hsp-len>
<Statistics_eff-space>109230</Statistics_eff-space>
<Statistics_kappa>0.071</Statistics_kappa>
<Statistics_lambda>0.299</Statistics_lambda>
<Statistics_entropy>0.27</Statistics_entropy>
</Statistics>
</Iteration_stat>
</Iteration>
<Iteration>
<Iteration_iter-num>21</Iteration_iter-num>
<Iteration_query-ID>Query_4</Iteration_query-ID>
<Iteration_query-def>sp|P08100|OPSD_HUMAN Rhodopsin OS=Homo sapiens GN=RHO PE=1 SV=1</Iteration_query-def>
<Iteration_query-len>348</Iteration_query-len>
<Iteration_hits>
<Hit>
<Hit_num>1</Hit_num>
<Hit_id>Subject_3</Hit_id>
<Hit_def>gi|283855845|gb|GQ290303.1| Cynopterus brachyotis voucher 20020434 rhodopsin (RHO) gene, exons 1 through 5 and partial cds</Hit_def>
<Hit_accession>Subject_3</Hit_accession>
<Hit_len>4301</Hit_len>
<Hit_hsps>
<Hsp>
<Hsp_num>1</Hsp_num>
<Hsp_bit-score>151.343146656381</Hsp_bit-score>
<Hsp_score>342</Hsp_score>
<Hsp_evalue>1.39566684546685e-72</Hsp_evalue>
<Hsp_query-from>239</Hsp_query-from>
<Hsp_query-to>312</Hsp_query-to>
<Hsp_hit-from>3147</Hsp_hit-from>
<Hsp_hit-to>3368</Hsp_hit-to>
<Hsp_query-frame>0</Hsp_query-frame>
<Hsp_hit-frame>3</Hsp_hit-frame>
<Hsp_identity>69</Hsp_identity>
<Hsp_positive>73</Hsp_positive>
<Hsp_gaps>0</Hsp_gaps>
<Hsp_align-len>74</Hsp_align-len>
<Hsp_qseq>ESATTQKAEKEVTRMVIIMVIAFLICWVPYASVAFYIFTHQGSNFGPIFMTIPAFFAKSAAIYNPVIYIMMNKQ</Hsp_qseq>
<Hsp_hseq>ESATTQKAEKEVTRMVIIMVIAFLICWLPYAGVAFYIFTHQGSNFGPIFMTLPAFFAKSSSIYNPVIYIMMNKQ</Hsp_hseq>
<Hsp_midline>ESATTQKAEKEVTRMVIIMVIAFLICW+PYA VAFYIFTHQGSNFGPIFMT+PAFFAKS++IYNPVIYIMMNKQ</Hsp_midline>
</Hsp>
<Hsp>
<Hsp_num>2</Hsp_num>
<Hsp_bit-score>126.323929257285</Hsp_bit-score>
<Hsp_score>284</Hsp_score>
<Hsp_evalue>1.39566684546685e-72</Hsp_evalue>
<Hsp_query-from>177</Hsp_query-from>
<Hsp_query-to>235</Hsp_query-to>
<Hsp_hit-from>2855</Hsp_hit-from>
<Hsp_hit-to>3031</Hsp_hit-to>
<Hsp_query-frame>0</Hsp_query-frame>
<Hsp_hit-frame>2</Hsp_hit-frame>
<Hsp_identity>54</Hsp_identity>
<Hsp_positive>57</Hsp_positive>
<Hsp_gaps>0</Hsp_gaps>
<Hsp_align-len>59</Hsp_align-len>
<Hsp_qseq>RYIPEGLQCSCGIDYYTLKPEVNNESFVIYMFVVHFTIPMIIIFFCYGQLVFTVKEAAA</Hsp_qseq>
<Hsp_hseq>RYIPEGMQCSCGIDYYTLKPEVNNESFVIYMFVVHFTIPMIVIFFCYGQLVFTVKEVRS</Hsp_hseq>
<Hsp_midline>RYIPEG+QCSCGIDYYTLKPEVNNESFVIYMFVVHFTIPMI+IFFCYGQLVFTVKE +</Hsp_midline>
</Hsp>
<Hsp>
<Hsp_num>3</Hsp_num>
<Hsp_bit-score>229.420359574251</Hsp_bit-score>
<Hsp_score>523</Hsp_score>
<Hsp_evalue>9.84654801241353e-65</Hsp_evalue>
<Hsp_query-from>11</Hsp_query-from>
<Hsp_query-to>121</Hsp_query-to>
<Hsp_hit-from>1</Hsp_hit-from>
<Hsp_hit-to>333</Hsp_hit-to>
<Hsp_query-frame>0</Hsp_query-frame>
<Hsp_hit-frame>1</Hsp_hit-frame>
<Hsp_identity>107</Hsp_identity>
<Hsp_positive>109</Hsp_positive>
<Hsp_gaps>0</Hsp_gaps>
<Hsp_align-len>111</Hsp_align-len>
<Hsp_qseq>VPFSNATGVVRSPFEYPQYYLAEPWQFSMLAAYMFLLIVLGFPINFLTLYVTVQHKKLRTPLNYILLNLAVADLFMVLGGFTSTLYTSLHGYFVFGPTGCNLEGFFATLGG</Hsp_qseq>
<Hsp_hseq>VPFSNKTGVVRSPFEHPQYYLAEPWQFSMLAAYMFLLIVLGFPINFLTLYVTVQHKKLRTPLNYILLNLAVADLFMVFGGFTTTLYTSLHGYFVFGPTGCNLEGFFATLGG</Hsp_hseq>
<Hsp_midline>VPFSN TGVVRSPFE+PQYYLAEPWQFSMLAAYMFLLIVLGFPINFLTLYVTVQHKKLRTPLNYILLNLAVADLFMV GGFT+TLYTSLHGYFVFGPTGCNLEGFFATLGG</Hsp_midline>
</Hsp>
<Hsp>
<Hsp_num>4</Hsp_num>
<Hsp_bit-score>122.873002719478</Hsp_bit-score>
<Hsp_score>276</Hsp_score>
<Hsp_evalue>1.40732096096596e-32</Hsp_evalue>
<Hsp_query-from>119</Hsp_query-from>
<Hsp_query-to>177</Hsp_query-to>
<Hsp_hit-from>1404</Hsp_hit-from>
<Hsp_hit-to>1580</Hsp_hit-to>
<Hsp_query-frame>0</Hsp_query-frame>
<Hsp_hit-frame>3</Hsp_hit-frame>
<Hsp_identity>55</Hsp_identity>
<Hsp_positive>56</Hsp_positive>
<Hsp_gaps>0</Hsp_gaps>
<Hsp_align-len>59</Hsp_align-len>
<Hsp_qseq>LGGEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMGVAFTWVMALACAAPPLAGWSR</Hsp_qseq>
<Hsp_hseq>LAGEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMGLALTWVMALACAAPPLVGWSR</Hsp_hseq>
<Hsp_midline>L GEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMG+A TWVMALACAAPPL GWSR</Hsp_midline>
</Hsp>
<Hsp>
<Hsp_num>5</Hsp_num>
<Hsp_bit-score>57.7367643183824</Hsp_bit-score>
<Hsp_score>125</Hsp_score>
<Hsp_evalue>5.60065526485586e-13</Hsp_evalue>
<Hsp_query-from>312</Hsp_query-from>
<Hsp_query-to>337</Hsp_query-to>
<Hsp_hit-from>4222</Hsp_hit-from>
<Hsp_hit-to>4299</Hsp_hit-to>
<Hsp_query-frame>0</Hsp_query-frame>
<Hsp_hit-frame>1</Hsp_hit-frame>
<Hsp_identity>23</Hsp_identity>
<Hsp_positive>24</Hsp_positive>
<Hsp_gaps>0</Hsp_gaps>
<Hsp_align-len>26</Hsp_align-len>
<Hsp_qseq>QFRNCMLTTICCGKNPLGDDEASATV</Hsp_qseq>
<Hsp_hseq>QFRNCMLTTLCCGKNPLGDDEASTTA</Hsp_hseq>
<Hsp_midline>QFRNCMLTT+CCGKNPLGDDEAS T </Hsp_midline>
</Hsp>
</Hit_hsps>
</Hit>
</Iteration_hits>
<Iteration_stat>
<Statistics>
<Statistics_db-num>0</Statistics_db-num>
<Statistics_db-len>0</Statistics_db-len>
<Statistics_hsp-len>18</Statistics_hsp-len>
<Statistics_eff-space>109230</Statistics_eff-space>
<Statistics_kappa>0.071</Statistics_kappa>
<Statistics_lambda>0.299</Statistics_lambda>
<Statistics_entropy>0.27</Statistics_entropy>
</Statistics>
</Iteration_stat>
</Iteration>
<Iteration>
<Iteration_iter-num>22</Iteration_iter-num>
<Iteration_query-ID>Query_4</Iteration_query-ID>
<Iteration_query-def>sp|P08100|OPSD_HUMAN Rhodopsin OS=Homo sapiens GN=RHO PE=1 SV=1</Iteration_query-def>
<Iteration_query-len>348</Iteration_query-len>
<Iteration_hits>
<Hit>
<Hit_num>1</Hit_num>
<Hit_id>Subject_4</Hit_id>
<Hit_def>gi|283855822|gb|GQ290312.1| Myotis ricketti voucher GQX10 rhodopsin (RHO) mRNA, partial cds</Hit_def>
<Hit_accession>Subject_4</Hit_accession>
<Hit_len>983</Hit_len>
<Hit_hsps>
<Hsp>
<Hsp_num>1</Hsp_num>
<Hsp_bit-score>658.197981896696</Hsp_bit-score>
<Hsp_score>1517</Hsp_score>
<Hsp_evalue>0</Hsp_evalue>
<Hsp_query-from>11</Hsp_query-from>
<Hsp_query-to>336</Hsp_query-to>
<Hsp_hit-from>1</Hsp_hit-from>
<Hsp_hit-to>978</Hsp_hit-to>
<Hsp_query-frame>0</Hsp_query-frame>
<Hsp_hit-frame>1</Hsp_hit-frame>
<Hsp_identity>310</Hsp_identity>
<Hsp_positive>322</Hsp_positive>
<Hsp_gaps>0</Hsp_gaps>
<Hsp_align-len>326</Hsp_align-len>
<Hsp_qseq>VPFSNATGVVRSPFEYPQYYLAEPWQFSMLAAYMFLLIVLGFPINFLTLYVTVQHKKLRTPLNYILLNLAVADLFMVLGGFTSTLYTSLHGYFVFGPTGCNLEGFFATLGGEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMGVAFTWVMALACAAPPLAGWSRYIPEGLQCSCGIDYYTLKPEVNNESFVIYMFVVHFTIPMIIIFFCYGQLVFTVKEAAAQQQESATTQKAEKEVTRMVIIMVIAFLICWVPYASVAFYIFTHQGSNFGPIFMTIPAFFAKSAAIYNPVIYIMMNKQFRNCMLTTICCGKNPLGDDEASAT</Hsp_qseq>
<Hsp_hseq>VPFSNKTGVVRSPFEYPQYYLAEPWQFSMLAAYMFLLIVLGFPINFLTLYVTVQHKKLRTPLNYILLNLAVANLFMVFGGFTTTLYTSMHGYFVFGATGCNLEGFFATLGGEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMGLAFTWVMALACAAPPLAGWSRYIPEGMQCSCGIDYYTLKPEVNNESFVIYMFVVHFTIPMIVIFFCYGQLVFTVKEAAAQQQESATTQKAEKEVTRMVIIMVVAFLICWLPYASVAFYIFTHQGSNFGPVFMTIPAFFAKSSSIYNPVIYIMMNKQFRNCMLTTLCCGKNPLGDDEASTT</Hsp_hseq>
<Hsp_midline>VPFSN TGVVRSPFEYPQYYLAEPWQFSMLAAYMFLLIVLGFPINFLTLYVTVQHKKLRTPLNYILLNLAVA+LFMV GGFT+TLYTS+HGYFVFG TGCNLEGFFATLGGEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMG+AFTWVMALACAAPPLAGWSRYIPEG+QCSCGIDYYTLKPEVNNESFVIYMFVVHFTIPMI+IFFCYGQLVFTVKEAAAQQQESATTQKAEKEVTRMVIIMV+AFLICW+PYASVAFYIFTHQGSNFGP+FMTIPAFFAKS++IYNPVIYIMMNKQFRNCMLTT+CCGKNPLGDDEAS T</Hsp_midline>
</Hsp>
</Hit_hsps>
</Hit>
</Iteration_hits>
<Iteration_stat>
<Statistics>
<Statistics_db-num>0</Statistics_db-num>
<Statistics_db-len>0</Statistics_db-len>
<Statistics_hsp-len>18</Statistics_hsp-len>
<Statistics_eff-space>109230</Statistics_eff-space>
<Statistics_kappa>0.071</Statistics_kappa>
<Statistics_lambda>0.299</Statistics_lambda>
<Statistics_entropy>0.27</Statistics_entropy>
</Statistics>
</Iteration_stat>
</Iteration>
<Iteration>
<Iteration_iter-num>23</Iteration_iter-num>
<Iteration_query-ID>Query_4</Iteration_query-ID>
<Iteration_query-def>sp|P08100|OPSD_HUMAN Rhodopsin OS=Homo sapiens GN=RHO PE=1 SV=1</Iteration_query-def>
<Iteration_query-len>348</Iteration_query-len>
<Iteration_hits>
<Hit>
<Hit_num>1</Hit_num>
<Hit_id>Subject_5</Hit_id>
<Hit_def>gi|18148870|dbj|AB062417.1| Synthetic construct Bos taurus gene for rhodopsin, complete cds</Hit_def>
<Hit_accession>Subject_5</Hit_accession>
<Hit_len>1047</Hit_len>
<Hit_hsps>
<Hsp>
<Hsp_num>1</Hsp_num>
<Hsp_bit-score>711.255977415469</Hsp_bit-score>
<Hsp_score>1640</Hsp_score>
<Hsp_evalue>0</Hsp_evalue>
<Hsp_query-from>1</Hsp_query-from>
<Hsp_query-to>348</Hsp_query-to>
<Hsp_hit-from>1</Hsp_hit-from>
<Hsp_hit-to>1044</Hsp_hit-to>
<Hsp_query-frame>0</Hsp_query-frame>
<Hsp_hit-frame>1</Hsp_hit-frame>
<Hsp_identity>325</Hsp_identity>
<Hsp_positive>337</Hsp_positive>
<Hsp_gaps>0</Hsp_gaps>
<Hsp_align-len>348</Hsp_align-len>
<Hsp_qseq>MNGTEGPNFYVPFSNATGVVRSPFEYPQYYLAEPWQFSMLAAYMFLLIVLGFPINFLTLYVTVQHKKLRTPLNYILLNLAVADLFMVLGGFTSTLYTSLHGYFVFGPTGCNLEGFFATLGGEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMGVAFTWVMALACAAPPLAGWSRYIPEGLQCSCGIDYYTLKPEVNNESFVIYMFVVHFTIPMIIIFFCYGQLVFTVKEAAAQQQESATTQKAEKEVTRMVIIMVIAFLICWVPYASVAFYIFTHQGSNFGPIFMTIPAFFAKSAAIYNPVIYIMMNKQFRNCMLTTICCGKNPLGDDEASATVSKTETSQVAPA</Hsp_qseq>
<Hsp_hseq>MNGTEGPNFYVPFSNKTGVVRSPFEAPQYYLAEPWQFSMLAAYMFLLIMLGFPINFLTLYVTVQHKKLRTPLNYILLNLAVADLFMVFGGFTTTLYTSLHGYFVFGPTGCNLEGFFATLGGEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMGVAFTWVMALACAAPPLVGWSRYIPEGMQCSCGIDYYTPHEETNNESFVIYMFVVHFIIPLIVIFFCYGQLVFTVKEAAAQQQESATTQKAEKEVTRMVIIMVIAFLICWLPYAGVAFYIFTHQGSDFGPIFMTIPAFFAKTSAVYNPVIYIMMNKQFRNCMVTTLCCGKNPLGDDEASTTVSKTETSQVAPA</Hsp_hseq>
<Hsp_midline>MNGTEGPNFYVPFSN TGVVRSPFE PQYYLAEPWQFSMLAAYMFLLI+LGFPINFLTLYVTVQHKKLRTPLNYILLNLAVADLFMV GGFT+TLYTSLHGYFVFGPTGCNLEGFFATLGGEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMGVAFTWVMALACAAPPL GWSRYIPEG+QCSCGIDYYT E NNESFVIYMFVVHF IP+I+IFFCYGQLVFTVKEAAAQQQESATTQKAEKEVTRMVIIMVIAFLICW+PYA VAFYIFTHQGS+FGPIFMTIPAFFAK++A+YNPVIYIMMNKQFRNCM+TT+CCGKNPLGDDEAS TVSKTETSQVAPA</Hsp_midline>
</Hsp>
</Hit_hsps>
</Hit>
</Iteration_hits>
<Iteration_stat>
<Statistics>
<Statistics_db-num>0</Statistics_db-num>
<Statistics_db-len>0</Statistics_db-len>
<Statistics_hsp-len>18</Statistics_hsp-len>
<Statistics_eff-space>109230</Statistics_eff-space>
<Statistics_kappa>0.071</Statistics_kappa>
<Statistics_lambda>0.299</Statistics_lambda>
<Statistics_entropy>0.27</Statistics_entropy>
</Statistics>
</Iteration_stat>
</Iteration>
<Iteration>
<Iteration_iter-num>24</Iteration_iter-num>
<Iteration_query-ID>Query_4</Iteration_query-ID>
<Iteration_query-def>sp|P08100|OPSD_HUMAN Rhodopsin OS=Homo sapiens GN=RHO PE=1 SV=1</Iteration_query-def>
<Iteration_query-len>348</Iteration_query-len>
<Iteration_hits>
<Hit>
<Hit_num>1</Hit_num>
<Hit_id>Subject_6</Hit_id>
<Hit_def>gi|12583664|dbj|AB043817.1| Conger myriaster conf gene for fresh water form rod opsin, complete cds</Hit_def>
<Hit_accession>Subject_6</Hit_accession>
<Hit_len>1344</Hit_len>
<Hit_hsps>
<Hsp>
<Hsp_num>1</Hsp_num>
<Hsp_bit-score>626.708277239213</Hsp_bit-score>
<Hsp_score>1444</Hsp_score>
<Hsp_evalue>0</Hsp_evalue>
<Hsp_query-from>1</Hsp_query-from>
<Hsp_query-to>341</Hsp_query-to>
<Hsp_hit-from>23</Hsp_hit-from>
<Hsp_hit-to>1048</Hsp_hit-to>
<Hsp_query-frame>0</Hsp_query-frame>
<Hsp_hit-frame>2</Hsp_hit-frame>
<Hsp_identity>281</Hsp_identity>
<Hsp_positive>311</Hsp_positive>
<Hsp_gaps>1</Hsp_gaps>
<Hsp_align-len>342</Hsp_align-len>
<Hsp_qseq>MNGTEGPNFYVPFSNATGVVRSPFEYPQYYLAEPWQFSMLAAYMFLLIVLGFPINFLTLYVTVQHKKLRTPLNYILLNLAVADLFMVLGGFTSTLYTSLHGYFVFGPTGCNLEGFFATLGGEIALWSLVVLAIERYVVVCKPMSNFRFGENHAIMGVAFTWVMALACAAPPLAGWSRYIPEGLQCSCGIDYYTLKPEVNNESFVIYMFVVHFTIPMIIIFFCYGQLVFTVKEAAAQQQESATTQKAEKEVTRMVIIMVIAFLICWVPYASVAFYIFTHQGSNFGPIFMTIPAFFAKSAAIYNPVIYIMMNKQFRNCMLTTICCGKNPL-GDDEASATVSKTE</Hsp_qseq>
<Hsp_hseq>MNGTEGPNFYIPMSNATGVVRSPFEYPQYYLAEPWAFSALSAYMFFLIIAGFPINFLTLYVTIEHKKLRTPLNYILLNLAVADLFMVFGGFTTTMYTSMHGYFVFGPTGCNIEGFFATLGGEIALWCLVVLAIERWMVVCKPVTNFRFGESHAIMGVMVTWTMALACALPPLFGWSRYIPEGLQCSCGIDYYTRAPGINNESFVIYMFTCHFSIPLAVISFCYGRLVCTVKEAAAQQQESETTQRAEREVTRMVVIMVISFLVCWVPYASVAWYIFTHQGSTFGPIFMTIPSFFAKSSALYNPMIYICMNKQFRHCMITTLCCGKNPFEEEDGASATSSKTE</Hsp_hseq>
<Hsp_midline>MNGTEGPNFY+P SNATGVVRSPFEYPQYYLAEPW FS L+AYMF LI+ GFPINFLTLYVT++HKKLRTPLNYILLNLAVADLFMV GGFT+T+YTS+HGYFVFGPTGCN+EGFFATLGGEIALW LVVLAIER++VVCKP++NFRFGE HAIMGV TW MALACA PPL GWSRYIPEGLQCSCGIDYYT P +NNESFVIYMF HF+IP+ +I FCYG+LV TVKEAAAQQQES TTQ+AE+EVTRMV+IMVI+FL+CWVPYASVA YIFTHQGS FGPIFMTIP+FFAKS+A+YNP+IYI MNKQFR CM+TT+CCGKNP +D ASAT SKTE</Hsp_midline>
</Hsp>
</Hit_hsps>
</Hit>
</Iteration_hits>
<Iteration_stat>
<Statistics>
<Statistics_db-num>0</Statistics_db-num>
<Statistics_db-len>0</Statistics_db-len>
<Statistics_hsp-len>18</Statistics_hsp-len>
<Statistics_eff-space>109230</Statistics_eff-space>
<Statistics_kappa>0.071</Statistics_kappa>
<Statistics_lambda>0.299</Statistics_lambda>
<Statistics_entropy>0.27</Statistics_entropy>
</Statistics>
</Iteration_stat>
</Iteration>
</BlastOutput_iterations>
</BlastOutput>
lib/galaxy/datatypes/xml.py
+21 -120
View File
@@ -27,9 +27,6 @@ class GenericXml( data.Text ):
>>> fname = get_test_fname( 'megablast_xml_parser_test1.blastxml' )
>>> GenericXml().sniff( fname )
True
>>> fname = get_test_fname( 'tblastn_four_human_vs_rhodopsin.xml' )
>>> BlastXml().sniff( fname )
True
>>> fname = get_test_fname( 'interval.interval' )
>>> GenericXml().sniff( fname )
False
@@ -50,123 +47,6 @@ class GenericXml( data.Text ):
data.Text.merge(split_files, output_file)
merge = staticmethod(merge)
class BlastXml( GenericXml ):
"""NCBI Blast XML Output data"""
file_ext = "blastxml"
def set_peek( self, dataset, is_multi_byte=False ):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek( dataset.file_name, is_multi_byte=is_multi_byte )
dataset.blurb = 'NCBI Blast XML data'
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disk'
def sniff( self, filename ):
"""
Determines whether the file is blastxml
>>> fname = get_test_fname( 'megablast_xml_parser_test1.blastxml' )
>>> BlastXml().sniff( fname )
True
>>> fname = get_test_fname( 'tblastn_four_human_vs_rhodopsin.xml' )
>>> BlastXml().sniff( fname )
True
>>> fname = get_test_fname( 'interval.interval' )
>>> BlastXml().sniff( fname )
False
"""
#TODO - Use a context manager on Python 2.5+ to close handle
handle = open(filename)
line = handle.readline()
if line.strip() != '<?xml version="1.0"?>':
handle.close()
return False
line = handle.readline()
if line.strip() not in ['<!DOCTYPE BlastOutput PUBLIC "-//NCBI//NCBI BlastOutput/EN" "http://www.ncbi.nlm.nih.gov/dtd/NCBI_BlastOutput.dtd">',
'<!DOCTYPE BlastOutput PUBLIC "-//NCBI//NCBI BlastOutput/EN" "NCBI_BlastOutput.dtd">']:
handle.close()
return False
line = handle.readline()
if line.strip() != '<BlastOutput>':
handle.close()
return False
handle.close()
return True
def merge(split_files, output_file):
"""Merging multiple XML files is non-trivial and must be done in subclasses."""
if len(split_files) == 1:
#For one file only, use base class method (move/copy)
return data.Text.merge(split_files, output_file)
out = open(output_file, "w")
h = None
for f in split_files:
h = open(f)
body = False
header = h.readline()
if not header:
out.close()
h.close()
raise ValueError("BLAST XML file %s was empty" % f)
if header.strip() != '<?xml version="1.0"?>':
out.write(header) #for diagnosis
out.close()
h.close()
raise ValueError("%s is not an XML file!" % f)
line = h.readline()
header += line
if line.strip() not in ['<!DOCTYPE BlastOutput PUBLIC "-//NCBI//NCBI BlastOutput/EN" "http://www.ncbi.nlm.nih.gov/dtd/NCBI_BlastOutput.dtd">',
'<!DOCTYPE BlastOutput PUBLIC "-//NCBI//NCBI BlastOutput/EN" "NCBI_BlastOutput.dtd">']:
out.write(header) #for diagnosis
out.close()
h.close()
raise ValueError("%s is not a BLAST XML file!" % f)
while True:
line = h.readline()
if not line:
out.write(header) #for diagnosis
out.close()
h.close()
raise ValueError("BLAST XML file %s ended prematurely" % f)
header += line
if "<Iteration>" in line:
break
if len(header) > 10000:
#Something has gone wrong, don't load too much into memory!
#Write what we have to the merged file for diagnostics
out.write(header)
out.close()
h.close()
raise ValueError("BLAST XML file %s has too long a header!" % f)
if "<BlastOutput>" not in header:
out.close()
h.close()
raise ValueError("%s is not a BLAST XML file:\n%s\n..." % (f, header))
if f == split_files[0]:
out.write(header)
old_header = header
elif old_header[:300] != header[:300]:
#Enough to check <BlastOutput_program> and <BlastOutput_version> match
out.close()
h.close()
raise ValueError("BLAST XML headers don't match for %s and %s - have:\n%s\n...\n\nAnd:\n%s\n...\n" \
% (split_files[0], f, old_header[:300], header[:300]))
else:
out.write(" <Iteration>\n")
for line in h:
if "</BlastOutput_iterations>" in line:
break
#TODO - Increment <Iteration_iter-num> and if required automatic query names
#like <Iteration_query-ID>Query_3</Iteration_query-ID> to be increasing?
out.write(line)
h.close()
out.write(" </BlastOutput_iterations>\n")
out.write("</BlastOutput>\n")
out.close()
merge = staticmethod(merge)
class MEMEXml( GenericXml ):
"""MEME XML Output data"""
file_ext = "memexml"
@@ -196,3 +76,24 @@ class CisML( GenericXml ):
dataset.blurb = 'file purged from disk'
def sniff( self, filename ):
return False
class Phyloxml( GenericXml ):
"""Format for defining phyloxml data http://www.phyloxml.org/"""
file_ext = "phyloxml"
def set_peek( self, dataset, is_multi_byte=False ):
"""Set the peek and blurb text"""
if not dataset.dataset.purged:
dataset.peek = data.get_file_peek( dataset.file_name, is_multi_byte=is_multi_byte )
dataset.blurb = 'Phyloxml data'
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disk'
def sniff( self, filename ):
""""Checking for keyword - 'phyloxml' always in lowercase in the first few lines"""
f = open(filename, "r")
firstlines = "".join(f.readlines(5))
f.close()
if "phyloxml" in firstlines:
return True
return False
lib/galaxy/jobs/__init__.py
+16 -16
View File
@@ -471,7 +471,7 @@ class JobWrapper( object ):
job.user.total_disk_usage += bytes
# fix permissions
for path in [ dp.real_path for dp in self.get_output_fnames() ]:
for path in [ dp.real_path for dp in self.get_mutable_output_fnames() ]:
util.umask_fix_perms( path, self.app.config.umask, 0666, self.app.config.gid )
self.sa_session.flush()
log.debug( 'job %d ended' % self.job_id )
@@ -490,7 +490,6 @@ class JobWrapper( object ):
if stderr contains anything, then False is returned.
Note that the job id is just for messages.
"""
err_msg = ""
# By default, the tool succeeded. This covers the case where the code
# has a bug but the tool was ok, and it lets a workflow continue.
success = True
@@ -507,7 +506,7 @@ class JobWrapper( object ):
# Check the exit code ranges in the order in which
# they were specified. Each exit_code is a StdioExitCode
# that includes an applicable range. If the exit code was in
# that range, then apply the error level and add in a message.
# that range, then apply the error level and add a message.
# If we've reached a fatal error rule, then stop.
max_error_level = galaxy.tools.StdioErrorLevel.NO_ERROR
for stdio_exit_code in self.tool.stdio_exit_codes:
@@ -515,20 +514,16 @@ class JobWrapper( object ):
tool_exit_code <= stdio_exit_code.range_end ):
# Tack on a generic description of the code
# plus a specific code description. For example,
# this might append "Job 42: Warning: Out of Memory\n".
# TODO: Find somewhere to stick the err_msg -
# possibly to the source (stderr/stdout), possibly
# in a new db column.
# this might prepend "Job 42: Warning: Out of Memory\n".
code_desc = stdio_exit_code.desc
if ( None == code_desc ):
code_desc = ""
tool_msg = ( "Job %s: %s: Exit code %d: %s" % (
job.get_id_tag(),
galaxy.tools.StdioErrorLevel.desc( tool_exit_code ),
tool_msg = ( "%s: Exit code %d: %s" % (
galaxy.tools.StdioErrorLevel.desc( stdio_exit_code.error_level ),
tool_exit_code,
code_desc ) )
log.info( tool_msg )
stderr = err_msg + stderr
log.info( "Job %s: %s" % (job.get_id_tag(), tool_msg) )
stderr = tool_msg + "\n" + stderr
max_error_level = max( max_error_level,
stdio_exit_code.error_level )
if ( max_error_level >=
@@ -571,7 +566,6 @@ class JobWrapper( object ):
re.IGNORECASE )
if ( regex_match ):
rexmsg = self.regex_err_msg( regex_match, regex)
# DELETEME
log.info( "Job %s: %s"
% ( job.get_id_tag(), rexmsg ) )
stderr = rexmsg + "\n" + stderr
@@ -685,6 +679,11 @@ class JobWrapper( object ):
self.compute_outputs()
return self.output_paths
def get_mutable_output_fnames( self ):
if self.output_paths is None:
self.compute_outputs()
return filter( lambda dsp: dsp.mutable, self.output_paths )
def get_output_hdas_and_fnames( self ):
if self.output_hdas_and_paths is None:
self.compute_outputs()
@@ -692,10 +691,11 @@ class JobWrapper( object ):
def compute_outputs( self ) :
class DatasetPath( object ):
def __init__( self, dataset_id, real_path, false_path = None ):
def __init__( self, dataset_id, real_path, false_path = None, mutable = True ):
self.dataset_id = dataset_id
self.real_path = real_path
self.false_path = false_path
self.mutable = mutable
def __str__( self ):
if self.false_path is None:
return self.real_path
@@ -712,13 +712,13 @@ class JobWrapper( object ):
self.output_hdas_and_paths = {}
for name, hda in [ ( da.name, da.dataset ) for da in job.output_datasets + job.output_library_datasets ]:
false_path = os.path.abspath( os.path.join( self.working_directory, "galaxy_dataset_%d.dat" % hda.dataset.id ) )
dsp = DatasetPath( hda.dataset.id, hda.dataset.file_name, false_path )
dsp = DatasetPath( hda.dataset.id, hda.dataset.file_name, false_path, mutable = hda.dataset.external_filename is None )
self.output_paths.append( dsp )
self.output_hdas_and_paths[name] = hda, dsp
if special:
false_path = os.path.abspath( os.path.join( self.working_directory, "galaxy_dataset_%d.dat" % special.dataset.id ) )
else:
results = [ ( da.name, da.dataset, DatasetPath( da.dataset.dataset.id, da.dataset.file_name ) ) for da in job.output_datasets + job.output_library_datasets ]
results = [ ( da.name, da.dataset, DatasetPath( da.dataset.dataset.id, da.dataset.file_name, mutable = da.dataset.dataset.external_filename is None ) ) for da in job.output_datasets + job.output_library_datasets ]
self.output_paths = [t[2] for t in results]
self.output_hdas_and_paths = dict([(t[0], t[1:]) for t in results])
if special:
lib/galaxy/jobs/deferred/genome_transfer.py
+10 -9
View File
@@ -115,15 +115,16 @@ class GenomeTransferPlugin( DataTransfer ):
files = tar.getmembers()
for filename in files:
z = tar.extractfile(filename)
try:
chunk = z.read( CHUNK_SIZE )
except IOError:
os.close( fd )
log.error( 'Problem decompressing compressed data' )
exit()
if not chunk:
break
os.write( fd, chunk )
while 1:
try:
chunk = z.read( CHUNK_SIZE )
except IOError:
os.close( fd )
log.error( 'Problem decompressing compressed data' )
exit()
if not chunk:
break
os.write( fd, chunk )
os.write( fd, '\n' )
os.close( fd )
tar.close()
lib/galaxy/jobs/handler.py
+1 -1
View File
@@ -360,7 +360,7 @@ class DefaultJobDispatcher( object ):
def __init__( self, app ):
self.app = app
self.job_runners = {}
start_job_runners = ["local", "lwr", "dynamic"]
start_job_runners = ["local", "lwr"]
if app.config.start_job_runners is not None:
start_job_runners.extend( [ x.strip() for x in util.listify( app.config.start_job_runners ) ] )
if app.config.use_tasked_jobs:
lib/galaxy/jobs/mapper.py
+1 -1
View File
@@ -111,7 +111,7 @@ class JobRunnerMapper( object ):
expand_function = self.__get_expand_function( expand_function_name )
return self.__invoke_expand_function( expand_function )
else:
raise Exception( "Unhandled dynamic job runner type specified - %s" % calculation_type )
raise Exception( "Unhandled dynamic job runner type specified - %s" % expand_type )
def __cache_job_runner_url( self, params ):
raw_job_runner_url = self.job_wrapper.tool.get_job_runner_url( params )
lib/galaxy/model/item_attrs.py
+1 -1
View File
@@ -95,7 +95,7 @@ class UsesAnnotations:
""" Returns a user's annotation string for an item. """
annotation_obj = self.get_item_annotation_obj( db_session, user, item )
if annotation_obj:
return annotation_obj.annotation
return galaxy.util.unicodify( annotation_obj.annotation )
return None
def get_item_annotation_obj( self, db_session, user, item ):
lib/galaxy/objectstore/__init__.py
+51 -23
View File
@@ -25,6 +25,7 @@ from sqlalchemy.orm import object_session
if sys.version_info >= (2, 6):
import multiprocessing
from galaxy.objectstore.s3_multipart_upload import multipart_upload
import boto
from boto.s3.key import Key
from boto.s3.connection import S3Connection
from boto.exception import S3ResponseError
@@ -377,9 +378,9 @@ class S3ObjectStore(ObjectStore):
super(S3ObjectStore, self).__init__()
self.config = config
self.staging_path = self.config.file_path
self.s3_conn = S3Connection()
self.bucket = self._get_bucket(self.config.s3_bucket)
self.use_rr = self.config.use_reduced_redundancy
self.s3_conn = get_OS_connection(self.config)
self.bucket = self._get_bucket(self.config.os_bucket_name)
self.use_rr = self.config.os_use_reduced_redundancy
self.cache_size = self.config.object_store_cache_size
self.transfer_progress = 0
# Clean cache only if value is set in universe_wsgi.ini
@@ -468,7 +469,7 @@ class S3ObjectStore(ObjectStore):
for i in range(5):
try:
bucket = self.s3_conn.get_bucket(bucket_name)
log.debug("Using S3 object store; got bucket '%s'" % bucket.name)
log.debug("Using cloud object store with bucket '%s'" % bucket.name)
return bucket
except S3ResponseError:
log.debug("Could not get bucket '%s', attempt %s/5" % (bucket_name, i+1))
@@ -607,13 +608,13 @@ class S3ObjectStore(ObjectStore):
log.error("Problem downloading key '%s' from S3 bucket '%s': %s" % (rel_path, self.bucket.name, ex))
return False
def _push_to_s3(self, rel_path, source_file=None, from_string=None):
def _push_to_os(self, rel_path, source_file=None, from_string=None):
"""
Push the file pointed to by `rel_path` to S3 naming the key `rel_path`.
If `source_file` is provided, push that file instead while still using
`rel_path` as the key name.
If `from_string` is provided, set contents of the file to the value of
the string
Push the file pointed to by ``rel_path`` to the object store naming the key
``rel_path``. If ``source_file`` is provided, push that file instead while
still using ``rel_path`` as the key name.
If ``from_string`` is provided, set contents of the file to the value of
the string.
"""
try:
source_file = source_file if source_file else self._get_cache_path(rel_path)
@@ -630,7 +631,7 @@ class S3ObjectStore(ObjectStore):
# print "Pushing cache file '%s' of size %s bytes to key '%s'" % (source_file, os.path.getsize(source_file), rel_path)
# print "+ Push started at '%s'" % start_time
mb_size = os.path.getsize(source_file) / 1e6
if mb_size < 60:
if mb_size < 60 or self.config.object_store == 'swift':
self.transfer_progress = 0 # Reset transfer progress counter
key.set_contents_from_filename(source_file, reduced_redundancy=self.use_rr,
cb=self._transfer_cb, num_cb=10)
@@ -648,12 +649,17 @@ class S3ObjectStore(ObjectStore):
return False
def file_ready(self, obj, **kwargs):
""" A helper method that checks if a file corresponding to a dataset
is ready and available to be used. Return True if so, False otherwise."""
"""
A helper method that checks if a file corresponding to a dataset is
ready and available to be used. Return ``True`` if so, ``False`` otherwise.
"""
rel_path = self._construct_path(obj, **kwargs)
# Make sure the size in cache is available in its entirety
if self._in_cache(rel_path) and os.path.getsize(self._get_cache_path(rel_path)) == self._get_size_in_s3(rel_path):
return True
if self._in_cache(rel_path):
if os.path.getsize(self._get_cache_path(rel_path)) == self._get_size_in_s3(rel_path):
return True
log.debug("Waiting for dataset {0} to transfer from OS: {1}/{2}".format(rel_path,
os.path.getsize(self._get_cache_path(rel_path)), self._get_size_in_s3(rel_path)))
return False
def exists(self, obj, **kwargs):
@@ -674,7 +680,7 @@ class S3ObjectStore(ObjectStore):
return False
# TODO: Sync should probably not be done here. Add this to an async upload stack?
if in_cache and not in_s3:
self._push_to_s3(rel_path, source_file=self._get_cache_path(rel_path))
self._push_to_os(rel_path, source_file=self._get_cache_path(rel_path))
return True
elif in_s3:
return True
@@ -707,12 +713,12 @@ class S3ObjectStore(ObjectStore):
# flat namespace), do so for consistency with the regular file system
# S3 folders are marked by having trailing '/' so add it now
# s3_dir = '%s/' % rel_path
# self._push_to_s3(s3_dir, from_string='')
# self._push_to_os(s3_dir, from_string='')
# If instructed, create the dataset in cache & in S3
if not dir_only:
rel_path = os.path.join(rel_path, alt_name if alt_name else "dataset_%s.dat" % obj.id)
open(os.path.join(self.staging_path, rel_path), 'w').close()
self._push_to_s3(rel_path, from_string='')
self._push_to_os(rel_path, from_string='')
def empty(self, obj, **kwargs):
if self.exists(obj, **kwargs):
@@ -826,7 +832,7 @@ class S3ObjectStore(ObjectStore):
else:
source_file = self._get_cache_path(rel_path)
# Update the file on S3
self._push_to_s3(rel_path, source_file)
self._push_to_os(rel_path, source_file)
else:
raise ObjectNotFound()
@@ -843,7 +849,6 @@ class S3ObjectStore(ObjectStore):
def get_store_usage_percent(self):
return 0.0
class DistributedObjectStore(ObjectStore):
"""
ObjectStore that defers to a list of backends, for getting objects the
@@ -1009,14 +1014,14 @@ def build_object_store_from_config(config):
store = config.object_store
if store == 'disk':
return DiskObjectStore(config=config)
elif store == 's3':
os.environ['AWS_ACCESS_KEY_ID'] = config.aws_access_key
os.environ['AWS_SECRET_ACCESS_KEY'] = config.aws_secret_key
elif store == 's3' or store == 'swift':
return S3ObjectStore(config=config)
elif store == 'distributed':
return DistributedObjectStore(config=config)
elif store == 'hierarchical':
return HierarchicalObjectStore()
else:
log.error("Unrecognized object store definition: {0}".format(store))
def convert_bytes(bytes):
""" A helper function used for pretty printing disk usage """
@@ -1039,3 +1044,26 @@ def convert_bytes(bytes):
else:
size = '%.2fb' % bytes
return size
def get_OS_connection(config):
"""
Get a connection object for a cloud Object Store specified in the config.
Currently, this is a ``boto`` connection object.
"""
log.debug("Getting a connection object for '{0}' object store".format(config.object_store))
a_key = config.os_access_key
s_key = config.os_secret_key
if config.object_store == 's3':
return S3Connection(a_key, s_key)
else:
# Establish the connection now
calling_format = boto.s3.connection.OrdinaryCallingFormat()
s3_conn = boto.connect_s3(aws_access_key_id=a_key,
aws_secret_access_key=s_key,
is_secure=config.os_is_secure,
host=config.os_host,
port=int(config.os_port),
calling_format=calling_format,
path=config.os_conn_path)
return s3_conn
lib/galaxy/tool_shed/migrate/versions/0004_tools.py
+14
View File
@@ -0,0 +1,14 @@
"""
The NCBI BLAST+ tools have been eliminated from the distribution. The tools and
datatypes are are now available in repositories named ncbi_blast_plus and
blast_datatypes, respectively, from the main Galaxy tool shed at
http://toolshed.g2.bx.psu.edu will be installed into your local Galaxy instance
at the location discussed above by running the following command.
"""
import sys
def upgrade():
print __doc__
def downgrade():
pass
lib/galaxy/tools/__init__.py
+11 -8
View File
@@ -187,7 +187,9 @@ class ToolBox( object ):
section.elems[ section_key ] = workflow
log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) )
elif section_key.startswith( 'label_' ):
section.elems[ section_key ] = section_val
if section_val:
section.elems[ section_key ] = section_val
log.debug( "Loaded label: %s" % ( section_val.text ) )
self.tool_panel[ key ] = section
def load_integrated_tool_panel_keys( self ):
"""
@@ -215,12 +217,12 @@ class ToolBox( object ):
section.elems[ key ] = None
elif section_elem.tag == 'label':
key = 'label_%s' % section_elem.get( 'id' )
section.elems[ key ] = ToolSectionLabel( section_elem )
section.elems[ key ] = None
key = 'section_%s' % elem.get( 'id' )
self.integrated_tool_panel[ key ] = section
elif elem.tag == 'label':
key = 'label_%s' % elem.get( 'id' )
self.integrated_tool_panel[ key ] = ToolSectionLabel( elem )
self.integrated_tool_panel[ key ] = None
def write_integrated_tool_panel_config_file( self ):
"""
Write the current in-memory version of the integrated_tool_panel.xml file to disk. Since Galaxy administrators
@@ -254,10 +256,11 @@ class ToolBox( object ):
if section_item:
os.write( fd, ' <workflow id="%s" />\n' % section_item.id )
elif section_key.startswith( 'label_' ):
label_id = section_item.id or ''
label_text = section_item.text or ''
label_version = section_item.version or ''
os.write( fd, ' <label id="%s" text="%s" version="%s" />\n' % ( label_id, label_text, label_version ) )
if section_item:
label_id = section_item.id or ''
label_text = section_item.text or ''
label_version = section_item.version or ''
os.write( fd, ' <label id="%s" text="%s" version="%s" />\n' % ( label_id, label_text, label_version ) )
os.write( fd, ' </section>\n' )
os.write( fd, '</toolbox>\n' )
os.close( fd )
@@ -2631,7 +2634,7 @@ class Tool:
if for_link:
# Create tool link.
if not self.tool_type.startswith( 'data_source' ):
link = url_for( controller='tool_runner', tool_id=self.id )
link = url_for( '/tool_runner', tool_id=self.id )
else:
link = url_for( self.action, **self.get_static_param_values( trans ) )
lib/galaxy/tools/genome_index/__init__.py
+16 -5
View File
@@ -18,13 +18,16 @@ def load_genome_index_tools( toolbox ):
<tool id="__GENOME_INDEX__" name="Index Genome" version="0.1" tool_type="genome_index">
<type class="GenomeIndexTool" module="galaxy.tools"/>
<action module="galaxy.tools.actions.index_genome" class="GenomeIndexToolAction"/>
<command>$__GENOME_INDEX_COMMAND__ $output_file $output_file.files_path $__app__.config.rsync_url "$__app__.config.tool_data_path"</command>
<command>$__GENOME_INDEX_COMMAND__ $output_file $output_file.files_path "$__app__.config.rsync_url" "$__app__.config.tool_data_path"</command>
<inputs>
<param name="__GENOME_INDEX_COMMAND__" type="hidden"/>
</inputs>
<outputs>
<data format="txt" name="output_file"/>
</outputs>
<stdio>
<exit_code range="1:" err_level="fatal" />
</stdio>
</tool>
"""
@@ -64,6 +67,18 @@ class GenomeIndexToolWrapper( object ):
if gitd:
fp = open( gitd.dataset.get_file_name(), 'r' )
deferred = sa_session.query( model.DeferredJob ).filter_by( id=gitd.deferred_job_id ).first()
try:
logloc = json.load( fp )
except ValueError:
deferred.state = app.model.DeferredJob.states.ERROR
sa_session.add( deferred )
sa_session.flush()
log.debug( 'Indexing job failed, setting deferred job state to error.' )
return False
finally:
fp.close()
destination = None
tdtman = ToolDataTableManager( app.config.tool_data_path )
xmltree = tdtman.load_from_config_file( app.config.tool_data_table_config_path, app.config.tool_data_path )
@@ -72,16 +87,12 @@ class GenomeIndexToolWrapper( object ):
location = node.findall('file')[0].get('path')
self.locations[table] = os.path.abspath( location )
locbase = os.path.abspath( os.path.split( self.locations['all_fasta'] )[0] )
deferred = sa_session.query( model.DeferredJob ).filter_by( id=gitd.deferred_job_id ).first()
params = deferred.params
dbkey = params[ 'dbkey' ]
basepath = os.path.join( os.path.abspath( app.config.genome_data_path ), dbkey )
intname = params[ 'intname' ]
indexer = gitd.indexer
workingdir = os.path.abspath( gitd.dataset.extra_files_path )
fp = open( gitd.dataset.get_file_name(), 'r' )
logloc = json.load( fp )
fp.close()
location = []
indexdata = gitd.dataset.extra_files_path
if indexer == '2bit':
lib/galaxy/tools/genome_index/index_genome.py
+9 -5
View File
@@ -40,19 +40,21 @@ class ManagedIndexer():
self.genome = os.path.splitext( self.fafile )[0]
with WithChDir( self.basedir ):
if indexer not in self.indexers:
raise KeyError, 'The requested indexing function does not exist'
sys.stderr.write( 'The requested indexing function does not exist' )
exit(127)
else:
with WithChDir( self.workingdir ):
self._log( 'Running indexer %s.' % indexer )
result = getattr( self, self.indexers[ indexer ] )()
if result in [ None, False ]:
self._log( 'Error running indexer %s, %s' % ( indexer, result ) )
sys.stderr.write( 'Error running indexer %s, %s' % ( indexer, result ) )
self._flush_files()
return True
exit(1)
else:
self._log( self.locations )
self._log( 'Indexer %s completed successfully.' % indexer )
self._flush_files()
exit(0)
def _check_link( self ):
self._log( 'Checking symlink to %s' % self.fafile )
@@ -309,5 +311,7 @@ if __name__ == "__main__":
# Create archive.
idxobj = ManagedIndexer( outfile, infile, working_dir, rsync_url, tooldata )
idxobj.run_indexer( indexer )
returncode = idxobj.run_indexer( indexer )
if not returncode:
exit(1)
exit(0)
lib/galaxy/util/__init__.py
+42 -2
View File
@@ -34,6 +34,9 @@ _lock = threading.RLock()
gzip_magic = '\037\213'
bz2_magic = 'BZh'
DEFAULT_ENCODING = 'utf-8'
NULL_CHAR = '\000'
BINARY_CHARS = [ NULL_CHAR ]
from inflection import Inflector, English
inflector = Inflector(English)
@@ -57,6 +60,32 @@ def is_multi_byte( chars ):
return True
return False
def is_binary( value, binary_chars=None ):
"""
File is binary if it contains a null-byte by default (e.g. behavior of grep, etc.).
This may fail for utf-16 files, but so would ASCII encoding.
>>> is_binary( string.printable )
False
>>> is_binary( '\\xce\\x94' )
False
>>> is_binary( '\\000' )
True
"""
if binary_chars is None:
binary_chars = BINARY_CHARS
for binary_char in binary_chars:
if binary_char in value:
return True
return False
def get_charset_from_http_headers( headers, default=None ):
rval = headers.get('content-type', None )
if rval and 'charset=' in rval:
rval = rval.split('charset=')[-1].split(';')[0].strip()
if rval:
return rval
return default
def synchronized(func):
"""This wrapper will serialize access to 'func' to a single thread. Use it as a decorator."""
def caller(*params, **kparams):
@@ -333,6 +362,17 @@ def roundify(amount, sfs = 2):
else:
return amount[0:sfs] + '0'*(len(amount) - sfs)
def unicodify( value, encoding=DEFAULT_ENCODING, error='replace', default=None ):
"""
Returns a unicode string or None
"""
if isinstance( value, unicode ):
return value
try:
return unicode( value, encoding, error )
except:
return default
def object_to_string( obj ):
return binascii.hexlify( pickle.dumps( obj, 2 ) )
@@ -502,7 +542,7 @@ def stringify_dictionary_keys( in_dict ):
def recursively_stringify_dictionary_keys( d ):
if isinstance(d, dict):
return dict([(k.encode('utf-8'), recursively_stringify_dictionary_keys(v)) for k,v in d.iteritems()])
return dict([(k.encode( DEFAULT_ENCODING ), recursively_stringify_dictionary_keys(v)) for k,v in d.iteritems()])
elif isinstance(d, list):
return [recursively_stringify_dictionary_keys(x) for x in d]
else:
@@ -622,7 +662,7 @@ def send_mail( frm, to, subject, body, config ):
Sends an email.
"""
to = listify( to )
msg = MIMEText( body )
msg = MIMEText( body.encode( 'ascii', 'replace' ) )
msg[ 'To' ] = ', '.join( to )
msg[ 'From' ] = frm
msg[ 'Subject' ] = subject
lib/galaxy/util/shed_util.py
+44 -29
View File
@@ -334,7 +334,7 @@ def clone_repository( repository_clone_url, repository_file_dir, ctx_rev ):
noupdate=False,
rev=util.listify( str( ctx_rev ) ) )
def copy_sample_file( app, filename, dest_path=None ):
"""Copy xxx.loc.sample to dest_path/xxx.loc.sample and dest_path/xxx.loc. The default value for dest_path is ~/tool-data."""
"""Copy xxx.sample to dest_path/xxx.sample and dest_path/xxx. The default value for dest_path is ~/tool-data."""
if dest_path is None:
dest_path = os.path.abspath( app.config.tool_data_path )
sample_file_name = strip_path( filename )
@@ -454,7 +454,7 @@ def create_tool_dependency_objects( app, tool_shed_repository, relative_install_
def generate_clone_url( trans, repository ):
"""Generate the URL for cloning a repository."""
tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository )
return '%s/repos/%s/%s' % ( tool_shed_url, repository.owner, repository.name )
return url_join( tool_shed_url, 'repos', repository.owner, repository.name )
def generate_datatypes_metadata( datatypes_config, metadata_dict ):
"""Update the received metadata_dict with information from the parsed datatypes_config."""
tree = ElementTree.parse( datatypes_config )
@@ -533,7 +533,7 @@ def can_generate_tool_dependency_metadata( root, metadata_dict ):
if req_name==tool_dependency_name and req_version==tool_dependency_version and req_type==tool_dependency_type:
can_generate_dependency_metadata = True
break
if not can_generate_dependency_metadata:
if requirements and not can_generate_dependency_metadata:
# We've discovered at least 1 combination of name, version and type that is not defined in the <requirement>
# tag for any tool in the repository.
break
@@ -573,13 +573,13 @@ def generate_metadata_for_changeset_revision( app, repository_clone_url, relativ
if datatypes_config:
metadata_dict = generate_datatypes_metadata( datatypes_config, metadata_dict )
# Get the relative path to all sample files included in the repository for storage in the repository's metadata.
sample_files = get_sample_files_from_disk( repository_files_dir=files_dir,
relative_install_dir=relative_install_dir,
resetting_all_metadata_on_repository=resetting_all_metadata_on_repository )
if sample_files:
metadata_dict[ 'sample_files' ] = sample_files
sample_file_metadata_paths, sample_file_copy_paths = get_sample_files_from_disk( repository_files_dir=files_dir,
relative_install_dir=relative_install_dir,
resetting_all_metadata_on_repository=resetting_all_metadata_on_repository )
if sample_file_metadata_paths:
metadata_dict[ 'sample_files' ] = sample_file_metadata_paths
# Copy all sample files included in the repository to a single directory location so we can load tools that depend on them.
for sample_file in sample_files:
for sample_file in sample_file_copy_paths:
copy_sample_file( app, sample_file, dest_path=work_dir )
# If the list of sample files includes a tool_data_table_conf.xml.sample file, laad it's table elements into memory.
relative_path, filename = os.path.split( sample_file )
@@ -608,21 +608,9 @@ def generate_metadata_for_changeset_revision( app, repository_clone_url, relativ
print "Error parsing %s", full_path, ", exception: ", str( e )
is_tool = False
if is_tool:
try:
tool = app.toolbox.load_tool( full_path )
except KeyError, e:
tool = None
invalid_tool_configs.append( name )
error_message = 'This file requires an entry for "%s" in the tool_data_table_conf.xml file. Upload a file ' % str( e )
error_message += 'named tool_data_table_conf.xml.sample to the repository that includes the required entry to correct '
error_message += 'this error. '
invalid_file_tups.append( ( name, error_message ) )
except Exception, e:
tool = None
invalid_tool_configs.append( name )
invalid_file_tups.append( ( name, str( e ) ) )
tool, valid, error_message = load_tool_from_config( app, full_path )
if tool is not None:
invalid_files_and_errors_tups = check_tool_input_params( app, files_dir, name, tool, sample_files, webapp=webapp )
invalid_files_and_errors_tups = check_tool_input_params( app, files_dir, name, tool, sample_file_metadata_paths, webapp=webapp )
can_set_metadata = True
for tup in invalid_files_and_errors_tups:
if name in tup:
@@ -993,7 +981,7 @@ def get_converter_and_display_paths( registration_elem, relative_install_dir ):
break
return converter_path, display_path
def get_ctx_rev( tool_shed_url, name, owner, changeset_revision ):
url = '%s/repository/get_ctx_rev?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % ( tool_shed_url, name, owner, changeset_revision )
url = url_join( tool_shed_url, 'repository/get_ctx_rev?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % ( name, owner, changeset_revision ) )
response = urllib2.urlopen( url )
ctx_rev = response.read()
response.close()
@@ -1077,7 +1065,8 @@ def get_sample_files_from_disk( repository_files_dir, relative_install_dir=None,
if resetting_all_metadata_on_repository:
# Keep track of the location where the repository is temporarily cloned so that we can strip it when setting metadata.
work_dir = repository_files_dir
sample_files = []
sample_file_metadata_paths = []
sample_file_copy_paths = []
for root, dirs, files in os.walk( repository_files_dir ):
if root.find( '.hg' ) < 0:
for name in files:
@@ -1088,10 +1077,15 @@ def get_sample_files_from_disk( repository_files_dir, relative_install_dir=None,
if stripped_path_to_sample_file.startswith( '/' ):
stripped_path_to_sample_file = stripped_path_to_sample_file[ 1: ]
relative_path_to_sample_file = os.path.join( relative_install_dir, stripped_path_to_sample_file )
if os.path.exists( relative_path_to_sample_file ):
sample_file_copy_paths.append( relative_path_to_sample_file )
else:
sample_file_copy_paths.append( full_path_to_sample_file )
else:
relative_path_to_sample_file = os.path.join( root, name )
sample_files.append( relative_path_to_sample_file )
return sample_files
sample_file_copy_paths.append( relative_path_to_sample_file )
sample_file_metadata_paths.append( relative_path_to_sample_file )
return sample_file_metadata_paths, sample_file_copy_paths
def get_shed_tool_conf_dict( app, shed_tool_conf ):
"""
Return the in-memory version of the shed_tool_conf file, which is stored in the config_elems entry
@@ -1221,8 +1215,8 @@ def get_tool_version_association( app, parent_tool_version, tool_version ):
def get_update_to_changeset_revision_and_ctx_rev( trans, repository ):
"""Return the changeset revision hash to which the repository can be updated."""
tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository )
url = '%s/repository/get_changeset_revision_and_ctx_rev?name=%s&owner=%s&changeset_revision=%s' % \
( tool_shed_url, repository.name, repository.owner, repository.installed_changeset_revision )
url = url_join( tool_shed_url, 'repository/get_changeset_revision_and_ctx_rev?name=%s&owner=%s&changeset_revision=%s' % \
( repository.name, repository.owner, repository.installed_changeset_revision ) )
try:
response = urllib2.urlopen( url )
encoded_update_dict = response.read()
@@ -1404,6 +1398,22 @@ def load_installed_datatypes( app, repository, relative_install_dir, deactivate=
def load_installed_display_applications( app, installed_repository_dict, deactivate=False ):
# Load or deactivate proprietary datatype display applications
app.datatypes_registry.load_display_applications( installed_repository_dict=installed_repository_dict, deactivate=deactivate )
def load_tool_from_config( app, full_path ):
try:
tool = app.toolbox.load_tool( full_path )
valid = True
error_message = None
except KeyError, e:
tool = None
valid = False
error_message = 'This file requires an entry for "%s" in the tool_data_table_conf.xml file. Upload a file ' % str( e )
error_message += 'named tool_data_table_conf.xml.sample to the repository that includes the required entry to correct '
error_message += 'this error. '
except Exception, e:
tool = None
valid = False
error_message = str( e )
return tool, valid, error_message
def open_repository_files_folder( trans, folder_path ):
try:
files_list = get_repository_files( trans, folder_path )
@@ -1645,3 +1655,8 @@ def update_tool_shed_repository_status( app, tool_shed_repository, status ):
tool_shed_repository.status = status
sa_session.add( tool_shed_repository )
sa_session.flush()
def url_join( *args ):
parts = []
for arg in args:
parts.append( arg.strip( '/' ) )
return '/'.join( parts )
lib/galaxy/visualization/phyloviz/__init__.py
+1
View File
@@ -0,0 +1 @@
__author__ = 'Tomithy'
lib/galaxy/visualization/phyloviz/baseparser.py
+125
View File
@@ -0,0 +1,125 @@
import json
class Node(object):
"""Node class of PhyloTree, which represents a CLAUDE in a phylogenetic tree"""
def __init__(self, nodeName, **kwargs):
"""Creates a node and adds in the typical annotations"""
self.name, self.id = nodeName, kwargs.get("id", 0)
self.depth = kwargs.get("depth", 0)
self.children = []
self.isInternal = kwargs.get("isInternal", 0)
self.length, self.bootstrap = kwargs.get("length", 0), kwargs.get("bootstrap", None)
self.events = kwargs.get("events", "")
# clean up boot strap values
if self.bootstrap == -1:
self.bootstrap = None
def addChildNode(self, child):
"""Adds a child node to the current node"""
if isinstance(child, Node):
self.children.append(child)
else:
self.children += child
def __str__(self):
return self.name + " id:" + str(self.id) + ", depth: " + str(self.depth)
def toJson(self):
"""Converts the data in the node to a dict representation of json"""
thisJson = {
"name" : self.name,
"id" : self.id,
"depth" : self.depth,
"dist" : self.length
}
thisJson = self.addChildrenToJson(thisJson)
thisJson = self.addMiscToJson(thisJson)
return thisJson
def addChildrenToJson(self, jsonDict):
"""Needs a special method to addChildren, such that the key does not appear in the Jsondict when the children is empty
this requirement is due to the layout algorithm used by d3 layout for hiding subtree """
if len(self.children) > 0:
children = [ node.toJson() for node in self.children]
jsonDict["children"] = children
return jsonDict
def addMiscToJson(self, jsonDict):
"""Adds other misc attributes to json if they are present"""
if not self.events == "":
jsonDict["events"] = self.events
if not self.bootstrap == None:
jsonDict["bootstrap"] = self.bootstrap
return jsonDict
class PhyloTree(object):
"""Standardized python based class to represent the phylogenetic tree parsed from different
phylogenetic file formats."""
def __init__(self):
self.root, self.rootAttr = None, {}
self.nodes = {}
self.title = None
self.id = 1
def addAttributesToRoot(self, attrDict):
"""Adds attributes to root, but first we put it in a temp store and bind it with root when .toJson is called"""
for key, value in attrDict.items():
self.rootAttr[key] = value
def makeNode(self, nodeName, **kwargs):
"""Called to make a node within PhyloTree, arbitrary kwargs can be passed to annotate nodes
Tracks the number of nodes via internally incremented id"""
kwargs["id"] = self.id
self.id += 1
return Node(nodeName, **kwargs)
def addRoot(self, root):
"""Creates a root for phyloTree"""
assert isinstance(root, Node)
root.parent = None
self.root = root
def generateJsonableDict(self):
"""Changes itself into a dictonary by recurssively calling the tojson on all its nodes. Think of it
as a dict in an array of dict in an array of dict and so on..."""
jsonTree = ""
if self.root:
assert isinstance(self.root, Node)
jsonTree = self.root.toJson()
for key, value in self.rootAttr.items():
# transfer temporary stored attr to root
jsonTree[key] = value
else:
raise Exception("Root is not assigned!")
return jsonTree
class Base_Parser(object):
"""Base parsers contain all the methods to handle phylogeny tree creation and
converting the data to json that all parsers should have"""
def __init__(self):
self.phyloTrees = []
def parseFile(self, filePath):
"""Base method that all phylogeny file parser should have"""
raise Exception("Base method for phylogeny file parsers is not implemented")
def toJson(self, jsonDict):
"""Convenience method to get a json string from a python json dict"""
return json.dumps(jsonDict)
def _writeJsonToFile(self, filepath, json):
"""Writes the file out to the system"""
f = open(filepath, "w")
f.writelines(json)
f.close()
lib/galaxy/visualization/phyloviz/newickparser.py
+185
View File
@@ -0,0 +1,185 @@
from baseparser import Base_Parser, PhyloTree
import re
class Newick_Parser(Base_Parser):
"""For parsing trees stored in the newick format (.nhx)
It is necessarily more complex because this parser is later extended by Nexus for parsing newick as well.."""
def __init__(self):
super(Newick_Parser, self).__init__()
def parseFile(self, filePath):
"""Parses a newick file to obtain the string inside. Returns: jsonableDict"""
with open(filePath, "r") as newickFile:
newickString = newickFile.read()
newickString = newickString.replace("\n", "").replace("\r", "")
return [self.parseData(newickString)], "Success"
def parseData(self, newickString):
"""To be called on a newickString directly to parse it. Returns: jsonableDict"""
return self._parseNewickToJson(newickString)
def _parseNewickToJson(self, newickString, treeName=None, nameMap=None):
"""parses a newick representation of a tree into a PhyloTree data structure,
which can be easily converted to json"""
self.phyloTree = PhyloTree()
newickString = self.cleanNewickString(newickString)
if nameMap:
newickString = self._mapName(newickString, nameMap)
self.phyloTree.root = self.parseNode(newickString, 0)
if nameMap:
self.phyloTree.addAttributesToRoot({"treeName": treeName})
return self.phyloTree.generateJsonableDict()
def cleanNewickString(self, rawNewick):
"""removing semi colon, and illegal json characters (\,',") and white spaces"""
return re.sub(r'\s|;|\"|\'|\\', '', rawNewick)
def _makeNodesFromString(self, string, depth):
"""elements separated by comma could be empty"""
if string.find("(") != -1:
raise Exception("Tree is not well form, location: " + string)
childrenString = string.split(",")
childrenNodes = []
for childString in childrenString:
if len(childString) == 0:
continue
nodeInfo = childString.split(":")
name, length, bootstrap = "", None, -1
if len(nodeInfo) == 2: # has length info
length = nodeInfo[1]
# checking for bootstap values
name = nodeInfo[0]
try: # Nexus may bootstrap in names position
name = float(name)
if 0<= name <= 1:
bootstrap = name
elif 1 <= name <= 100:
bootstrap = name / 100
name = ""
except ValueError:
name = nodeInfo[0]
else:
name = nodeInfo[0] # string only contains name
node = self.phyloTree.makeNode(name, length=length, depth=depth, bootstrap= bootstrap)
childrenNodes += [node]
return childrenNodes
def _mapName(self, newickString, nameMap):
"""
Necessary to replace names of terms inside nexus representation
Also, its here because Mailaud's doesnt deal with id_strings outside of quotes(" ")
"""
newString = ""
start = 0
end = 0
for i in xrange(len(newickString)):
if newickString[i] == "(" or newickString[i] == ",":
if re.match(r"[,(]", newickString[i+1:]):
continue
else:
end = i + 1
# i now refers to the starting position of the term to be replaced,
# we will next find j which is the ending pos of the term
for j in xrange(i+1, len(newickString)):
enclosingSymbol = newickString[j] # the immediate symbol after a common or left bracket which denotes the end of a term
if enclosingSymbol == ")" or enclosingSymbol == ":" or enclosingSymbol == ",":
termToReplace = newickString[end:j]
newString += newickString[start : end] + nameMap[termToReplace] #+ "'" "'" +
start = j
break
newString += newickString[start:]
return newString
def parseNode(self, string, depth):
""" Recursive method for parsing newick string, works by stripping down the string into substring
of newick contained with brackers, which is used to call itself.
Eg ... ( A, B, (D, E)C, F, G ) ...
We will make the preceeding nodes first A, B, then the internal node C, its children D, E,
and finally the succeeding nodes F, G"""
# Base case where there is only an empty string
if string == "":
return
# Base case there its only an internal claude
if string.find("(") == -1:
return self._makeNodesFromString(string, depth)
nodes, children = [], [] # nodes refer to the nodes on this level, children refers to the child of the
start = 0
lenOfPreceedingInternalNodeString = 0
bracketStack = []
for j in xrange(len(string)):
if string[j] == "(": #finding the positions of all the open brackets
bracketStack.append(j)
continue
if string[j] == ")": #finding the positions of all the closed brackets to extract claude
i = bracketStack.pop()
if len(bracketStack) == 0: # is child of current node
InternalNode = None
#First flat call to make nodes of the same depth but from the preceeding string.
startSubstring = string[start + lenOfPreceedingInternalNodeString: i]
preceedingNodes = self._makeNodesFromString(startSubstring, depth)
nodes += preceedingNodes
# Then We will try to see if the substring has any internal nodes first, make it then make nodes preceeding it and succeeding it.
if j + 1 < len(string):
stringRightOfBracket = string[j+1:] # Eg. '(b:0.4,a:0.3)c:0.3, stringRightOfBracket = c:0.3
match = re.search(r"[\)\,\(]", stringRightOfBracket)
if match:
indexOfNextSymbol = match.start()
stringRepOfInternalNode = stringRightOfBracket[:indexOfNextSymbol]
internalNodes = self._makeNodesFromString( stringRepOfInternalNode, depth)
if len(internalNodes) > 0:
InternalNode = internalNodes[0]
lenOfPreceedingInternalNodeString = len(stringRepOfInternalNode)
else: # sometimes the node can be the last element of a string
InternalNode = self._makeNodesFromString(string[j+1:], depth)[0]
lenOfPreceedingInternalNodeString = len(string) - j
if InternalNode == None: #creating a generic node if it is unnamed
InternalNode = self.phyloTree.makeNode( "", depth=depth, isInternal=True ) #"internal-" + str(depth)
lenOfPreceedingInternalNodeString = 0
# recussive call to make the internal claude
childSubString = string[ i + 1 : j ]
InternalNode.addChildNode(self.parseNode(childSubString, depth + 1))
nodes.append(InternalNode) # we append the internal node later to preserve order
start = j + 1
continue
if depth == 0: # if its the root node, we do nothing about it and return
return nodes[0]
# Adding last most set of children
endString = string[start:]
if string[start-1] == ")": # if the symbol belongs to an internal node which is created previously, then we remove it from the string left to parse
match = re.search(r"[\)\,\(]", endString)
if match:
endOfNodeName = start + match.start() + 1
endString = string[endOfNodeName:]
nodes += self._makeNodesFromString(endString, depth)
return nodes
lib/galaxy/visualization/phyloviz/nexusparser.py
+107
View File
@@ -0,0 +1,107 @@
from newickparser import Newick_Parser
import re
MAX_READLINES = 200000
class Nexus_Parser(Newick_Parser):
def __init__(self):
super(Nexus_Parser, self).__init__()
def parseFile(self, filePath):
"""passes a file and extracts its Nexus content."""
return self.parseNexus(filePath)
def parseNexus(self, filename):
""" Nexus data is stored in blocks between a line starting with begin and another line starting with end;
Commends inside square brackets are to be ignored,
For more information: http://wiki.christophchamp.com/index.php/NEXUS_file_format
Nexus can store multiple trees
"""
with open( filename, "rt") as nex_file:
nexlines = nex_file.readlines()
rowCount = 0
inTreeBlock = False # sentinel to check if we are in a tree block
intranslateBlock = False # sentinel to check if we are in the translate region of the tree. Stores synonyms of the labellings
self.inCommentBlock = False
self.nameMapping = None # stores mapping representation used in nexus format
treeNames = []
for line in nexlines:
line = line.replace(";\n", "")
lline = line.lower()
if rowCount > MAX_READLINES or (not nex_file) :
break
rowCount +=1
# We are only interested in the tree block.
if "begin" in lline and "tree" in lline and not inTreeBlock:
inTreeBlock = True
continue
if inTreeBlock and "end" in lline[:3]:
inTreeBlock, currPhyloTree = False, None
continue
if inTreeBlock:
if "title" in lline: # Adding title to the tree
titleLoc = lline.find("title")
title = line[titleLoc + 5:].replace(" ", "")
continue
if "translate" in lline:
intranslateBlock = True
self.nameMapping = {}
continue
if intranslateBlock:
mappingLine = self.splitLinebyWhitespaces(line)
key, value = mappingLine[1], mappingLine[2].replace(",", "").replace("'","") #replacing illegal json characters
self.nameMapping[key] = value
# Extracting newick Trees
if "tree" in lline:
intranslateBlock = False
treeLineCols = self.splitLinebyWhitespaces(line)
treeName, newick = treeLineCols[2], treeLineCols[-1]
if newick == "": # Empty lines can be found in tree blocks
continue
currPhyloTree = self._parseNewickToJson(newick, treeName, nameMap=self.nameMapping)
self.phyloTrees.append(currPhyloTree)
treeIndex = len(self.phyloTrees) - 1
treeNames.append( (treeName, treeIndex) ) # appending name of tree, and its index
continue
return self.phyloTrees, treeNames
def splitLinebyWhitespaces(self, line):
"""replace tabs and write spaces to a single write space, so we can properly split it."""
return re.split(r"\s+", line)
def checkComments(self, line):
"""Check to see if the line/lines is a comment."""
if not self.inCommentBlock:
if "[" in line:
if "]" not in line:
self.inCommentBlock = True
else:
return "Nextline" # need to move on to the nextline after getting out of comment
else :
if "]" in line:
if line.rfind("[") > line.rfind("]"):
pass # a comment block is closed but another is open.
else:
self.inCommentBlock = False
return "Nextline" # need to move on to the nextline after getting out of comment
return ""
lib/galaxy/visualization/phyloviz/phyloviz_dataprovider.py
+35
View File
@@ -0,0 +1,35 @@
from newickparser import Newick_Parser
from nexusparser import Nexus_Parser
from phyloxmlparser import Phyloxml_Parser
class Phyloviz_DataProvider(object):
def __init__(self):
pass
def parseFile(self, filepath, fileExt):
"""returns [trees], meta
Trees are actually an array of JsonDicts. It's usually one tree, except in the case of Nexus
"""
jsonDicts, meta = [], {}
try:
if fileExt == "nhx": # parses newick files
newickParser = Newick_Parser()
jsonDicts, parseMsg = newickParser.parseFile(filepath)
elif fileExt == "phyloxml": # parses phyloXML files
phyloxmlParser = Phyloxml_Parser()
jsonDicts, parseMsg = phyloxmlParser.parseFile(filepath)
elif fileExt == "nex": # parses nexus files
nexusParser = Nexus_Parser()
jsonDicts, parseMsg = nexusParser.parseFile(filepath)
meta["trees"] = parseMsg
else:
raise Exception("File type is not supported")
meta["msg"] = parseMsg
except Exception:
jsonDicts, meta["msg"] = [], "Parse failed"
return jsonDicts, meta
lib/galaxy/visualization/phyloviz/phyloxmlparser.py
+134
View File
@@ -0,0 +1,134 @@
from baseparser import Base_Parser, PhyloTree, Node
from xml.etree import ElementTree
class Phyloxml_Parser(Base_Parser):
"""Parses a phyloxml file into a json file that will be passed to PhyloViz for display"""
def __init__(self):
super(Phyloxml_Parser, self).__init__()
self.phyloTree = PhyloTree()
self.tagsOfInterest = {
"clade": "",
"name" : "name",
"branch_length" : "length",
"confidence" : "bootstrap",
"events" : "events"
}
def parseFile(self, filePath):
"""passes a file and extracts its Phylogeny Tree content."""
phyloXmlFile = open(filePath, "r")
xmlTree = ElementTree.parse(phyloXmlFile)
xmlRoot = xmlTree.getroot()[0]
self.nameSpaceIndex = xmlRoot.tag.rfind("}") + 1 # used later by the clean tag method to remove the name space in every element.tag
phyloRoot = None
for child in xmlRoot:
childTag = self.cleanTag(child.tag)
if childTag == "clade":
phyloRoot = child
elif childTag == "name":
self.phyloTree.title = child.text
self.phyloTree.root = self.parseNode(phyloRoot, 0)
jsonDict = self.phyloTree.generateJsonableDict()
return [jsonDict], "Success"
def parseNode(self, node, depth):
"""Parses any node within a phyloxml tree and looks out for claude, which signals the creation of
nodes - internal OR leaf"""
assert isinstance(node, etree._Element)
tag = self.cleanTag(node.tag)
if not tag == "clade":
return None
hasInnerClade = False
# peeking once for parent and once for child to check if the node is internal
for child in node:
childTag = self.cleanTag(child.tag)
if childTag == "clade":
hasInnerClade = True
break
if hasInnerClade: # this node is an internal node
currentNode = self._makeInternalNode(node, depth= depth)
for child in node:
child = self.parseNode(child, depth + 1)
if isinstance(child, Node):
currentNode.addChildNode(child)
else: # this node is a leaf node
currentNode = self._makeLeafNode(node, depth=depth+1)
return currentNode
def _makeLeafNode(self, leafNode, depth = 0 ):
"""Makes leaf nodes by calling Phylotree methods"""
node = {}
for child in leafNode:
childTag = self.cleanTag(child.tag)
if childTag in self.tagsOfInterest:
key = self.tagsOfInterest[childTag] # need to map phyloxml terms to ours
node[key] = child.text
node["depth"] = depth
return self.phyloTree.makeNode(self._getNodeName(leafNode), **node)
def _getNodeName(self, node, depth=-1):
"""Gets the name of a claude. It handles the case where a taxonomy node is involved"""
def getTagFromTaxonomyNode(node):
"""Returns the name of a taxonomy node. A taxonomy node have to be treated differently as the name
is embedded one level deeper"""
phyloxmlTaxoNames = {
"common_name" : "",
"scientific_name" : "",
"code" : ""
}
for child in node:
childTag = self.cleanTag(child.tag)
if childTag in phyloxmlTaxoNames:
return child.text
return ""
nodeName = ""
for child in node:
childTag = self.cleanTag(child.tag)
if childTag == "name" :
nodeName = child.text
break
elif childTag == "taxonomy":
nodeName = getTagFromTaxonomyNode(child)
break
return nodeName
def _makeInternalNode(self, internalNode, depth=0):
""" Makes an internal node from an element object that is guranteed to be a parent node.
Gets the value of interests like events and appends it to a custom node object that will be passed to PhyloTree to make nodes
"""
node = {}
for child in internalNode:
childTag = self.cleanTag(child.tag)
if childTag == "clade":
continue
elif childTag in self.tagsOfInterest:
if childTag == "events": # events is nested 1 more level deeper than others
key, text = "events", self.cleanTag(child[0].tag)
else:
key = self.tagsOfInterest[childTag]
text = child.text
node[key] = text
return self.phyloTree.makeNode(self._getNodeName(internalNode, depth), **node)
def cleanTag(self, tagString):
return tagString[self.nameSpaceIndex:]
lib/galaxy/visualization/tracks/data_providers.py
+136 -64
View File
@@ -2,7 +2,7 @@
Data providers for tracks visualizations.
"""
import sys
import os, sys
from math import ceil, log
import pkg_resources
pkg_resources.require( "bx-python" )
@@ -59,6 +59,51 @@ def _convert_between_ucsc_and_ensemble_naming( chrom ):
def _chrom_naming_matches( chrom1, chrom2 ):
return ( chrom1.startswith( 'chr' ) and chrom2.startswith( 'chr' ) ) or ( not chrom1.startswith( 'chr' ) and not chrom2.startswith( 'chr' ) )
class FeatureLocationIndexDataProvider( object ):
'''
'''
def __init__( self, converted_dataset ):
self.converted_dataset = converted_dataset
def get_data( self, query ):
# Init.
textloc_file = open( self.converted_dataset.file_name, 'r' )
line_len = int( textloc_file.readline() )
file_len = os.path.getsize( self.converted_dataset.file_name )
# Find query in file using binary search.
low = 0
high = file_len / line_len
while low < high:
mid = ( low + high ) // 2
position = mid * line_len
textloc_file.seek( position )
# Compare line with query and update low, high.
line = textloc_file.readline()
print '--', mid, line
if line < query:
low = mid + 1
else:
high = mid
position = low * line_len
# At right point in file, generate hits.
result = [ ]
while True:
line = textloc_file.readline()
if not line.startswith( query ):
break
if line[ -1: ] == '\n':
line = line[ :-1 ]
result.append( line.split() )
textloc_file.close()
return result
class TracksDataProvider( object ):
""" Base class for tracks data providers. """
@@ -439,6 +484,11 @@ class BedDataProvider( TracksDataProvider ):
# Score (filter data)
if length >= 5 and filter_cols and filter_cols[0] == "Score":
# If dataset doesn't have name/strand/thick start/thick end/blocks,
# add placeholders. There should be 8 entries if all attributes
# are present.
payload.extend( [ None for i in range( 8 - len( payload ) ) ] )
try:
payload.append( float( feature[4] ) )
except:
@@ -575,7 +625,8 @@ class VcfDataProvider( TracksDataProvider ):
end = start + len( new_seq )
# Pack line.
payload = [ hash( line ),
payload = [
hash( line ),
start,
end,
# ID:
@@ -584,7 +635,8 @@ class VcfDataProvider( TracksDataProvider ):
# TODO? VCF does not have strand, so default to positive.
"+",
new_seq,
float( feature[5] ) ]
None if feature[5] == '.' else float( feature[5] )
]
rval.append(payload)
return { 'data': rval, 'message': message }
@@ -923,79 +975,99 @@ class BBIDataProvider( TracksDataProvider ):
# which we use converted_dataset
f, bbi = self._get_dataset()
# If the stats kwarg was provide, we compute overall summary data for the
# range defined by start and end but no reduced data. This is currently
# used by client to determine the default range.
# If stats requested, compute overall summary data for the range
# start:endbut no reduced data. This is currently used by client
# to determine the default range.
if 'stats' in kwargs:
summary = bbi.summarize( chrom, start, end, 1 )
f.close()
if summary is None:
return None
else:
min = 0
max = 0
mean = 0
sd = 0
if summary is not None:
# Does the summary contain any defined values?
valid_count = summary.valid_count[0]
if summary.valid_count < 1:
return None
if summary.valid_count > 0:
# Compute $\mu \pm 2\sigma$ to provide an estimate for upper and lower
# bounds that contain ~95% of the data.
mean = summary.sum_data[0] / valid_count
var = summary.sum_squares[0] - mean
if valid_count > 1:
var /= valid_count - 1
sd = numpy.sqrt( var )
min = summary.min_val[0]
max = summary.max_val[0]
# Compute $\mu \pm 2\sigma$ to provide an estimate for upper and lower
# bounds that contain ~95% of the data.
mean = summary.sum_data[0] / valid_count
var = summary.sum_squares[0] - mean
if valid_count > 1:
var /= valid_count - 1
sd = numpy.sqrt( var )
return dict( data=dict( min=min, max=max, mean=mean, sd=sd ) )
return dict( data=dict( min=summary.min_val[0], max=summary.max_val[0], mean=mean, sd=sd ) )
# Sample from region using approximately this many samples.
N = 1000
# The following seems not to work very well, for example it will only return one
# data point if the tile is 1280px wide. Not sure what the intent is.
def summarize_region( bbi, chrom, start, end, num_points ):
'''
Returns results from summarizing a region using num_points.
NOTE: num_points cannot be greater than end - start or BBI
will return None for all positions.s
'''
result = []
# The first zoom level for BBI files is 640. If too much is requested, it will look at each block instead
# of summaries. The calculation done is: zoom <> (end-start)/num_points/2.
# Thus, the optimal number of points is (end-start)/num_points/2 = 640
# num_points = (end-start) / 1280
#num_points = (end-start) / 1280
#if num_points < 1:
# num_points = end - start
#else:
# num_points = min(num_points, 500)
# For now, we'll do 1000 data points by default. However, the summaries
# don't seem to work when a summary pixel corresponds to less than one
# datapoint, so we prevent that.
# FIXME: need to choose the number of points to maximize coverage of the area.
# It appears that BBI calculates points using intervals of
# floor( num_points / end - start )
# In some cases, this prevents sampling near the end of the interval,
# especially when (a) the total interval is small ( < 20-30Kb) and (b) the
# computed interval size has a large fraction, e.g. 14.7 or 35.8
num_points = min( 1000, end - start )
# HACK to address the FIXME above; should generalize.
if end - start <= 2000:
num_points = end - start
summary = bbi.summarize( chrom, start, end, num_points )
f.close()
result = []
if summary:
#mean = summary.sum_data / summary.valid_count
# Get summary; this samples at intervals of length
# (end - start)/num_points -- i.e. drops any fractional component
# of interval length.
summary = bbi.summarize( chrom, start, end, num_points )
if summary:
#mean = summary.sum_data / summary.valid_count
## Standard deviation by bin, not yet used
## var = summary.sum_squares - mean
## var /= minimum( valid_count - 1, 1 )
## sd = sqrt( var )
## Standard deviation by bin, not yet used
## var = summary.sum_squares - mean
## var /= minimum( valid_count - 1, 1 )
## sd = sqrt( var )
pos = start
step_size = (end - start) / num_points
pos = start
step_size = (end - start) / num_points
for i in range( num_points ):
result.append( (pos, float_nan( summary.sum_data[i] / summary.valid_count[i] ) ) )
pos += step_size
for i in range( num_points ):
result.append( (pos, float_nan( summary.sum_data[i] / summary.valid_count[i] ) ) )
pos += step_size
return result
# Approach is different depending on region size.
if end - start < N:
# Get values for individual bases in region, including start and end.
# To do this, need to increase end to next base and request number of points.
num_points = end - start + 1
end += 1
else:
#
# The goal is to sample the region between start and end uniformly
# using ~N data points. The challenge is that the size of sampled
# intervals rarely is full bases, so sampling using N points will
# leave the end of the region unsampled due to remainders for each
# interval. To recitify this, a new N is calculated based on the
# step size that covers as much of the region as possible.
#
# However, this still leaves some of the region unsampled. This
# could be addressed by repeatedly sampling remainder using a
# smaller and smaller step_size, but that would require iteratively
# going to BBI, which could be time consuming.
#
# Start with N samples.
num_points = N
step_size = ( end - start ) / num_points
# Add additional points to sample in the remainder not covered by
# the initial N samples.
remainder_start = start + step_size * num_points
additional_points = ( end - remainder_start ) / step_size
num_points += additional_points
result = summarize_region( bbi, chrom, start, end, num_points )
# Cleanup and return.
f.close()
return { 'data': result }
class BigBedDataProvider( BBIDataProvider ):
lib/galaxy/web/api/histories.py
+1
View File
@@ -109,6 +109,7 @@ class HistoriesController( BaseAPIController, UsesHistoryMixin ):
trans.sa_session.add( new_history )
trans.sa_session.flush()
item = new_history.get_api_value(view='element', value_mapper={'id':trans.security.encode_id})
item['url'] = url_for( 'history', id=item['id'] )
return item
@web.expose_api
lib/galaxy/web/api/libraries.py
+1 -1
View File
@@ -98,7 +98,7 @@ class LibrariesController( BaseAPIController ):
rval['url'] = url_for( 'library', id=encoded_id )
rval['name'] = name
rval['id'] = encoded_id
return [ rval ]
return rval
@web.expose_api
def delete( self, trans, id, **kwd ):
lib/galaxy/web/api/permissions.py
+2 -2
View File
@@ -40,10 +40,10 @@ class PermissionsController( BaseAPIController ):
role_params = params.get( k + '_in', [] )
in_roles = [ trans.sa_session.query( trans.app.model.Role ).get( trans.security.decode_id( x ) ) for x in util.listify( role_params ) ]
permissions[ trans.app.security_agent.get_action( v.action ) ] = in_roles
trans.app.security_agent.set_all_library_permissions( library, permissions )
trans.app.security_agent.set_all_library_permissions( trans, library, permissions )
trans.sa_session.refresh( library )
# Copy the permissions to the root folder
trans.app.security_agent.copy_library_permissions( library, library.root_folder )
trans.app.security_agent.copy_library_permissions( trans, library, library.root_folder )
message = "Permissions updated for library '%s'." % library.name
item = library.get_api_value( view='element' )
lib/galaxy/web/api/roles.py
+3 -3
View File
@@ -17,7 +17,7 @@ class RoleAPIController( BaseAPIController ):
"""
rval = []
for role in trans.sa_session.query( trans.app.model.Role ).filter( trans.app.model.Role.table.c.deleted == False ):
if trans.app.security_agent.ok_to_display( trans.user, role ):
if trans.user_is_admin() or trans.app.security_agent.ok_to_display( trans.user, role ):
item = role.get_api_value( value_mapper={ 'id': trans.security.encode_id } )
encoded_id = trans.security.encode_id( role.id )
item['url'] = url_for( 'role', id=encoded_id )
@@ -32,7 +32,7 @@ class RoleAPIController( BaseAPIController ):
"""
role_id = id
try:
role_id = trans.security.decode_id( role_id )
decoded_role_id = trans.security.decode_id( role_id )
except TypeError:
trans.response.status = 400
return "Malformed role id ( %s ) specified, unable to decode." % str( role_id )
@@ -40,7 +40,7 @@ class RoleAPIController( BaseAPIController ):
role = trans.sa_session.query( trans.app.model.Role ).get( decoded_role_id )
except:
role = None
if not role or not trans.app.security_agent.ok_to_display( trans.user, role ):
if not role or not (trans.user_is_admin() or trans.app.security_agent.ok_to_display( trans.user, role )):
trans.response.status = 400
return "Invalid role id ( %s ) specified." % str( role_id )
item = role.get_api_value( view='element', value_mapper={ 'id': trans.security.encode_id } )
lib/galaxy/web/api/workflows.py
+331 -1
View File
@@ -10,10 +10,12 @@ from galaxy.tools.parameters import visit_input_values, DataToolParameter
from galaxy.web.base.controller import BaseAPIController, url_for
from galaxy.workflow.modules import module_factory
from galaxy.jobs.actions.post import ActionBox
from galaxy.model.item_attrs import UsesAnnotations
from galaxy.web.controllers.workflow import attach_ordered_steps
log = logging.getLogger(__name__)
class WorkflowsAPIController(BaseAPIController):
class WorkflowsAPIController(BaseAPIController, UsesAnnotations):
@web.expose_api
def index(self, trans, **kwd):
"""
@@ -73,6 +75,16 @@ class WorkflowsAPIController(BaseAPIController):
# Eventually, allow regular tool parameters to be inserted and modified at runtime.
# p = step.get_required_parameters()
item['inputs'] = inputs
steps = {}
for step in latest_workflow.steps:
steps[step.id] = {'id': step.id,
'type': step.type,
'tool_id': step.tool_id,
'input_steps': {}}
for conn in step.input_connections:
steps[step.id]['input_steps'][conn.input_name] = {'source_step': conn.output_step_id,
'step_output': conn.output_name}
item['steps'] = steps
return item
@web.expose_api
@@ -84,6 +96,17 @@ class WorkflowsAPIController(BaseAPIController):
However, we will import them if installed_repository_file is specified
"""
# ------------------------------------------------------------------------------- #
### RPARK: dictionary containing which workflows to change and edit ###
param_map = {};
if (payload.has_key('parameters') ):
param_map = payload['parameters'];
# ------------------------------------------------------------------------------- #
if 'workflow_id' not in payload:
# create new
if 'installed_repository_file' in payload:
@@ -168,6 +191,17 @@ class WorkflowsAPIController(BaseAPIController):
# are not persisted so we need to do it every time)
step.module.add_dummy_datasets( connections=step.input_connections )
step.state = step.module.state
####################################################
####################################################
# RPARK: IF TOOL_NAME IN PARAMETER MAP #
if step.tool_id in param_map:
change_param = param_map[step.tool_id]['param'];
change_value = param_map[step.tool_id]['value'];
step.state.inputs[change_param] = change_value;
####################################################
####################################################
if step.tool_errors:
trans.response.status = 400
return "Workflow cannot be run because of validation errors in some steps: %s" % step_errors
@@ -220,3 +254,299 @@ class WorkflowsAPIController(BaseAPIController):
trans.sa_session.flush()
return rval
# ---------------------------------------------------------------------------------------------- #
# ---------------------------------------------------------------------------------------------- #
# ---- RPARK EDITS ---- #
# ---------------------------------------------------------------------------------------------- #
# ---------------------------------------------------------------------------------------------- #
@web.expose_api
#@web.json
def workflow_dict( self, trans, workflow_id, **kwd ):
"""
GET /api/workflows/{encoded_workflow_id}/download
Returns a selected workflow as a json dictionary.
"""
try:
stored_workflow = trans.sa_session.query(self.app.model.StoredWorkflow).get(trans.security.decode_id(workflow_id))
except Exception,e:
return ("Workflow with ID='%s' can not be found\n Exception: %s") % (workflow_id, str( e ))
# check to see if user has permissions to selected workflow
if stored_workflow.user != trans.user and not trans.user_is_admin():
if trans.sa_session.query(trans.app.model.StoredWorkflowUserShareAssociation).filter_by(user=trans.user, stored_workflow=stored_workflow).count() == 0:
trans.response.status = 400
return("Workflow is not owned by or shared with current user")
ret_dict = self._workflow_to_dict( trans, stored_workflow );
return ret_dict
@web.expose_api
def delete( self, trans, id, **kwd ):
"""
DELETE /api/workflows/{encoded_workflow_id}
Deletes a specified workflow
Author: rpark
copied from galaxy.web.controllers.workflows.py (delete)
"""
workflow_id = id;
try:
stored_workflow = trans.sa_session.query(self.app.model.StoredWorkflow).get(trans.security.decode_id(workflow_id))
except Exception,e:
return ("Workflow with ID='%s' can not be found\n Exception: %s") % (workflow_id, str( e ))
# check to see if user has permissions to selected workflow
if stored_workflow.user != trans.user and not trans.user_is_admin():
if trans.sa_session.query(trans.app.model.StoredWorkflowUserShareAssociation).filter_by(user=trans.user, stored_workflow=stored_workflow).count() == 0:
trans.response.status = 400
return("Workflow is not owned by or shared with current user")
#Mark a workflow as deleted
stored_workflow.deleted = True
trans.sa_session.flush()
# TODO: Unsure of response message to let api know that a workflow was successfully deleted
#return 'OK'
return ( "Workflow '%s' successfully deleted" % stored_workflow.name )
@web.expose_api
def import_new_workflow(self, trans, payload, **kwd):
"""
POST /api/workflows/upload
Importing dynamic workflows from the api. Return newly generated workflow id.
Author: rpark
# currently assumes payload['workflow'] is a json representation of a workflow to be inserted into the database
"""
data = payload['workflow'];
workflow, missing_tool_tups = self._workflow_from_dict( trans, data, source="API" )
# galaxy workflow newly created id
workflow_id = workflow.id;
# api encoded, id
encoded_id = trans.security.encode_id(workflow_id);
# return list
rval= [];
item = workflow.get_api_value(value_mapper={'id':trans.security.encode_id})
item['url'] = url_for('workflow', id=encoded_id)
rval.append(item);
return item;
def _workflow_from_dict( self, trans, data, source=None ):
"""
RPARK: copied from galaxy.web.controllers.workflows.py
Creates a workflow from a dict. Created workflow is stored in the database and returned.
"""
# Put parameters in workflow mode
trans.workflow_building_mode = True
# Create new workflow from incoming dict
workflow = model.Workflow()
# If there's a source, put it in the workflow name.
if source:
name = "%s (imported from %s)" % ( data['name'], source )
else:
name = data['name']
workflow.name = name
# Assume no errors until we find a step that has some
workflow.has_errors = False
# Create each step
steps = []
# The editor will provide ids for each step that we don't need to save,
# but do need to use to make connections
steps_by_external_id = {}
# Keep track of tools required by the workflow that are not available in
# the local Galaxy instance. Each tuple in the list of missing_tool_tups
# will be ( tool_id, tool_name, tool_version ).
missing_tool_tups = []
# First pass to build step objects and populate basic values
for key, step_dict in data[ 'steps' ].iteritems():
# Create the model class for the step
step = model.WorkflowStep()
steps.append( step )
steps_by_external_id[ step_dict['id' ] ] = step
# FIXME: Position should be handled inside module
step.position = step_dict['position']
module = module_factory.from_dict( trans, step_dict, secure=False )
if module.type == 'tool' and module.tool is None:
# A required tool is not available in the local Galaxy instance.
missing_tool_tup = ( step_dict[ 'tool_id' ], step_dict[ 'name' ], step_dict[ 'tool_version' ] )
if missing_tool_tup not in missing_tool_tups:
missing_tool_tups.append( missing_tool_tup )
module.save_to_step( step )
if step.tool_errors:
workflow.has_errors = True
# Stick this in the step temporarily
step.temp_input_connections = step_dict['input_connections']
# Save step annotation.
annotation = step_dict[ 'annotation' ]
#if annotation:
#annotation = sanitize_html( annotation, 'utf-8', 'text/html' )
# ------------------------------------------ #
# RPARK REMOVING: user annotation b/c of API
#self.add_item_annotation( trans.sa_session, trans.get_user(), step, annotation )
# ------------------------------------------ #
# Unpack and add post-job actions.
post_job_actions = step_dict.get( 'post_job_actions', {} )
for name, pja_dict in post_job_actions.items():
pja = PostJobAction( pja_dict[ 'action_type' ],
step, pja_dict[ 'output_name' ],
pja_dict[ 'action_arguments' ] )
# Second pass to deal with connections between steps
for step in steps:
# Input connections
for input_name, conn_dict in step.temp_input_connections.iteritems():
if conn_dict:
conn = model.WorkflowStepConnection()
conn.input_step = step
conn.input_name = input_name
conn.output_name = conn_dict['output_name']
conn.output_step = steps_by_external_id[ conn_dict['id'] ]
del step.temp_input_connections
# Order the steps if possible
attach_ordered_steps( workflow, steps )
# Connect up
stored = model.StoredWorkflow()
stored.name = workflow.name
workflow.stored_workflow = stored
stored.latest_workflow = workflow
stored.user = trans.user
# Persist
trans.sa_session.add( stored )
trans.sa_session.flush()
return stored, missing_tool_tups
def _workflow_to_dict( self, trans, stored ):
"""
RPARK: copied from galaxy.web.controllers.workflows.py
Converts a workflow to a dict of attributes suitable for exporting.
"""
workflow = stored.latest_workflow
### ----------------------------------- ###
## RPARK EDIT ##
workflow_annotation = self.get_item_annotation_obj( trans.sa_session, trans.user, stored )
annotation_str = ""
if workflow_annotation:
annotation_str = workflow_annotation.annotation
### ----------------------------------- ###
# Pack workflow data into a dictionary and return
data = {}
data['a_galaxy_workflow'] = 'true' # Placeholder for identifying galaxy workflow
data['format-version'] = "0.1"
data['name'] = workflow.name
### ----------------------------------- ###
## RPARK EDIT ##
data['annotation'] = annotation_str
### ----------------------------------- ###
data['steps'] = {}
# For each step, rebuild the form and encode the state
for step in workflow.steps:
# Load from database representation
module = module_factory.from_workflow_step( trans, step )
### ----------------------------------- ###
## RPARK EDIT ##
# Get user annotation.
step_annotation = self.get_item_annotation_obj(trans.sa_session, trans.user, step )
annotation_str = ""
if step_annotation:
annotation_str = step_annotation.annotation
### ----------------------------------- ###
# Step info
step_dict = {
'id': step.order_index,
'type': module.type,
'tool_id': module.get_tool_id(),
'tool_version' : step.tool_version,
'name': module.get_name(),
'tool_state': module.get_state( secure=False ),
'tool_errors': module.get_errors(),
## 'data_inputs': module.get_data_inputs(),
## 'data_outputs': module.get_data_outputs(),
### ----------------------------------- ###
## RPARK EDIT ##
'annotation' : annotation_str
### ----------------------------------- ###
}
# Add post-job actions to step dict.
if module.type == 'tool':
pja_dict = {}
for pja in step.post_job_actions:
pja_dict[pja.action_type+pja.output_name] = dict( action_type = pja.action_type,
output_name = pja.output_name,
action_arguments = pja.action_arguments )
step_dict[ 'post_job_actions' ] = pja_dict
# Data inputs
step_dict['inputs'] = []
if module.type == "data_input":
# Get input dataset name; default to 'Input Dataset'
name = module.state.get( 'name', 'Input Dataset')
step_dict['inputs'].append( { "name" : name, "description" : annotation_str } )
else:
# Step is a tool and may have runtime inputs.
for name, val in module.state.inputs.items():
input_type = type( val )
if input_type == RuntimeValue:
step_dict['inputs'].append( { "name" : name, "description" : "runtime parameter for tool %s" % module.get_name() } )
elif input_type == dict:
# Input type is described by a dict, e.g. indexed parameters.
for partname, partval in val.items():
if type( partval ) == RuntimeValue:
step_dict['inputs'].append( { "name" : name, "description" : "runtime parameter for tool %s" % module.get_name() } )
# User outputs
step_dict['user_outputs'] = []
"""
module_outputs = module.get_data_outputs()
step_outputs = trans.sa_session.query( WorkflowOutput ).filter( step=step )
for output in step_outputs:
name = output.output_name
annotation = ""
for module_output in module_outputs:
if module_output.get( 'name', None ) == name:
output_type = module_output.get( 'extension', '' )
break
data['outputs'][name] = { 'name' : name, 'annotation' : annotation, 'type' : output_type }
"""
# All step outputs
step_dict['outputs'] = []
if type( module ) is ToolModule:
for output in module.get_data_outputs():
step_dict['outputs'].append( { 'name' : output['name'], 'type' : output['extensions'][0] } )
# Connections
input_connections = step.input_connections
if step.type is None or step.type == 'tool':
# Determine full (prefixed) names of valid input datasets
data_input_names = {}
def callback( input, value, prefixed_name, prefixed_label ):
if isinstance( input, DataToolParameter ):
data_input_names[ prefixed_name ] = True
visit_input_values( module.tool.inputs, module.state.inputs, callback )
# Filter
# FIXME: this removes connection without displaying a message currently!
input_connections = [ conn for conn in input_connections if conn.input_name in data_input_names ]
# Encode input connections as dictionary
input_conn_dict = {}
for conn in input_connections:
input_conn_dict[ conn.input_name ] = \
dict( id=conn.output_step.order_index, output_name=conn.output_name )
step_dict['input_connections'] = input_conn_dict
# Position
step_dict['position'] = step.position
# Add to return value
data['steps'][step.order_index] = step_dict
return data
lib/galaxy/web/buildapp.py
+5
View File
@@ -151,6 +151,11 @@ def app_factory( global_conf, **kwargs ):
webapp.api_mapper.resource_with_deleted( 'history', 'histories', path_prefix='/api' )
#webapp.api_mapper.connect( 'run_workflow', '/api/workflow/{workflow_id}/library/{library_id}', controller='workflows', action='run', workflow_id=None, library_id=None, conditions=dict(method=["GET"]) )
# "POST /api/workflows/import" => ``workflows.import_workflow()``.
# Defines a named route "import_workflow".
webapp.api_mapper.connect("import_workflow", "/api/workflows/upload", controller="workflows", action="import_new_workflow", conditions=dict(method=["POST"]))
webapp.api_mapper.connect("workflow_dict", '/api/workflows/download/{workflow_id}', controller='workflows', action='workflow_dict', conditions=dict(method=['GET']))
webapp.finalize_config()
# Wrap the webapp in some useful middleware
if kwargs.get( 'middleware', True ):
lib/galaxy/web/controllers/admin_toolshed.py
+23 -16
View File
@@ -374,7 +374,7 @@ class AdminToolshed( AdminGalaxy ):
def browse_tool_shed( self, trans, **kwd ):
tool_shed_url = kwd[ 'tool_shed_url' ]
galaxy_url = url_for( '/', qualified=True )
url = '%srepository/browse_valid_categories?galaxy_url=%s&webapp=galaxy' % ( tool_shed_url, galaxy_url )
url = url_join( tool_shed_url, 'repository/browse_valid_categories?galaxy_url=%s&webapp=galaxy' % ( galaxy_url ) )
return trans.response.send_redirect( url )
@web.expose
@web.require_admin
@@ -392,8 +392,9 @@ class AdminToolshed( AdminGalaxy ):
# Send a request to the relevant tool shed to see if there are any updates.
repository = get_repository( trans, kwd[ 'id' ] )
tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository )
url = '%s/repository/check_for_updates?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \
( tool_shed_url, url_for( '/', qualified=True ), repository.name, repository.owner, repository.changeset_revision )
url = url_join( tool_shed_url,
'repository/check_for_updates?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \
( url_for( '/', qualified=True ), repository.name, repository.owner, repository.changeset_revision ) )
return trans.response.send_redirect( url )
@web.expose
@web.require_admin
@@ -467,14 +468,14 @@ class AdminToolshed( AdminGalaxy ):
def find_tools_in_tool_shed( self, trans, **kwd ):
tool_shed_url = kwd[ 'tool_shed_url' ]
galaxy_url = url_for( '/', qualified=True )
url = '%srepository/find_tools?galaxy_url=%s&webapp=galaxy' % ( tool_shed_url, galaxy_url )
url = url_join( tool_shed_url, 'repository/find_tools?galaxy_url=%s&webapp=galaxy' % galaxy_url )
return trans.response.send_redirect( url )
@web.expose
@web.require_admin
def find_workflows_in_tool_shed( self, trans, **kwd ):
tool_shed_url = kwd[ 'tool_shed_url' ]
galaxy_url = url_for( '/', qualified=True )
url = '%srepository/find_workflows?galaxy_url=%s&webapp=galaxy' % ( tool_shed_url, galaxy_url )
url = url_join( tool_shed_url, 'repository/find_workflows?galaxy_url=%s&webapp=galaxy' % galaxy_url )
return trans.response.send_redirect( url )
def generate_tool_path( self, repository_clone_url, changeset_revision ):
"""
@@ -489,7 +490,7 @@ class AdminToolshed( AdminGalaxy ):
tool_shed_url = items[ 0 ]
repo_path = items[ 1 ]
tool_shed_url = clean_tool_shed_url( tool_shed_url )
return '%s/repos%s/%s' % ( tool_shed_url, repo_path, changeset_revision )
return url_join( tool_shed_url, 'repos', repo_path, changeset_revision )
@web.json
@web.require_admin
def get_file_contents( self, trans, file_path ):
@@ -634,8 +635,9 @@ class AdminToolshed( AdminGalaxy ):
tool_shed_repository,
trans.model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS )
tool_shed_url = get_url_from_repository_tool_shed( trans.app, tool_shed_repository )
url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \
( tool_shed_url, tool_shed_repository.name, tool_shed_repository.owner, tool_shed_repository.changeset_revision )
url = url_join( tool_shed_url,
'/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \
( tool_shed_repository.name, tool_shed_repository.owner, tool_shed_repository.changeset_revision ) )
response = urllib2.urlopen( url )
text = response.read()
response.close()
@@ -954,7 +956,9 @@ class AdminToolshed( AdminGalaxy ):
repository_ids = kwd.get( 'repository_ids', None )
changeset_revisions = kwd.get( 'changeset_revisions', None )
# Get the information necessary to install each repository.
url = '%srepository/get_repository_information?repository_ids=%s&changeset_revisions=%s&webapp=galaxy' % ( tool_shed_url, repository_ids, changeset_revisions )
url = url_join( tool_shed_url,
'repository/get_repository_information?repository_ids=%s&changeset_revisions=%s&webapp=galaxy' % \
( repository_ids, changeset_revisions ) )
response = urllib2.urlopen( url )
raw_text = response.read()
response.close()
@@ -1097,8 +1101,9 @@ class AdminToolshed( AdminGalaxy ):
name = repo_info_dict.keys()[ 0 ]
repo_info_tuple = repo_info_dict[ name ]
description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, tool_dependencies = repo_info_tuple
url = '%srepository/get_readme?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \
( tool_shed_url, name, repository_owner, changeset_revision )
url = url_join( tool_shed_url,
'repository/get_readme?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \
( name, repository_owner, changeset_revision ) )
response = urllib2.urlopen( url )
raw_text = response.read()
response.close()
@@ -1273,8 +1278,9 @@ class AdminToolshed( AdminGalaxy ):
tool_shed = get_tool_shed_from_clone_url( repository_clone_url )
# Get all previous change set revisions from the tool shed for the repository back to, but excluding, the previous valid changeset
# revision to see if it was previously installed using one of them.
url = '%s/repository/previous_changeset_revisions?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \
( tool_shed_url, url_for( '/', qualified=True ), repository_name, repository_owner, changeset_revision )
url = url_join( tool_shed_url,
'repository/previous_changeset_revisions?galaxy_url=%s&name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \
( url_for( '/', qualified=True ), repository_name, repository_owner, changeset_revision ) )
response = urllib2.urlopen( url )
text = response.read()
response.close()
@@ -1350,8 +1356,9 @@ class AdminToolshed( AdminGalaxy ):
# Get the tool_versions from the tool shed for each tool in the installed change set.
repository = get_repository( trans, kwd[ 'id' ] )
tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository )
url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \
( tool_shed_url, repository.name, repository.owner, repository.changeset_revision )
url = url_join( tool_shed_url,
'repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s&webapp=galaxy' % \
( repository.name, repository.owner, repository.changeset_revision ) )
response = urllib2.urlopen( url )
text = response.read()
response.close()
@@ -1522,7 +1529,7 @@ class AdminToolshed( AdminGalaxy ):
def __generate_clone_url( self, trans, repository ):
"""Generate the URL for cloning a repository."""
tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository )
return '%s/repos/%s/%s' % ( tool_shed_url, repository.owner, repository.name )
return url_join( tool_shed_url, 'repos', repository.owner, repository.name )
## ---- Utility methods -------------------------------------------------------
lib/galaxy/web/controllers/data_admin.py
+14 -8
View File
@@ -75,7 +75,7 @@ class DataAdmin( BaseUIController ):
@web.require_admin
def add_genome( self, trans, **kwd ):
if trans.app.config.get_bool( 'enable_beta_job_managers', False ) == False:
return trans.fill_template( '/admin/data_admin/betajob.mako' )
return trans.fill_template( '/admin/data_admin/generic_error.mako', message='This feature requires that enable_beta_job_managers be set to True in your Galaxy configuration.' )
dbkeys = trans.ucsc_builds
ensemblkeys = trans.ensembl_builds
ncbikeys = trans.ncbi_builds
@@ -137,9 +137,10 @@ class DataAdmin( BaseUIController ):
dbkey = build.split( ': ' )[0]
longname = build.split( ': ' )[-1]
url = 'http://togows.dbcls.jp/entry/ncbi-nucleotide/%s.fasta' % dbkey
elif source == 'Broad':
dbkey = params.get('broad_dbkey', '')[0]
url = 'ftp://ftp.broadinstitute.org/pub/seq/references/%s.fasta' % dbkey
elif source == 'URL':
dbkey = params.get( 'url_dbkey', '' )
url = params.get( 'url', None )
longname = params.get( 'longname', None )
elif source == 'UCSC':
longname = None
for build in trans.ucsc_builds:
@@ -147,7 +148,8 @@ class DataAdmin( BaseUIController ):
dbkey = build[0]
longname = build[1]
break
assert dbkey is not '?', 'That build was not found'
if dbkey == '?':
return trans.fill_template( '/admin/data_admin/generic_error.mako', message='An invalid build was specified.' )
ftp = ftplib.FTP('hgdownload.cse.ucsc.edu')
ftp.login('anonymous', trans.get_user().email)
checker = []
@@ -188,7 +190,8 @@ class DataAdmin( BaseUIController ):
dbkeys=trans.ucsc_builds )
elif source == 'Ensembl':
dbkey = params.get( 'ensembl_dbkey', None )
assert dbkey is not '?', 'That build was not found'
if dbkey == '?':
return trans.fill_template( '/admin/data_admin/generic_error.mako', message='An invalid build was specified.' )
for build in trans.ensembl_builds:
if build[ 'dbkey' ] == dbkey:
dbkey = build[ 'dbkey' ]
@@ -198,7 +201,9 @@ class DataAdmin( BaseUIController ):
break
url = 'ftp://ftp.ensembl.org/pub/release-%s/fasta/%s/dna/%s.%s.%s.dna.toplevel.fa.gz' % ( release, pathname.lower(), pathname, dbkey, release )
else:
raise ValueError, 'Somehow an invalid data source was specified.'
return trans.fill_template( '/admin/data_admin/generic_error.mako', message='An invalid data source was specified.' )
if url is None:
return trans.fill_template( '/admin/data_admin/generic_error.mako', message='Unable to generate a valid URL with the specified parameters.' )
params = dict( protocol='http', name=dbkey, datatype='fasta', url=url, user=trans.user.id )
jobid = trans.app.job_manager.deferred_job_queue.plugins['GenomeTransferPlugin'].create_job( trans, url, dbkey, longname, indexers )
chainjob = []
@@ -245,7 +250,8 @@ class DataAdmin( BaseUIController ):
sa = trans.app.model.context.current
if jobtype == 'liftover':
job = sa.query( model.TransferJob ).filter_by( id=jobid ).first()
joblabel = 'Download liftOver'
liftover = trans.app.job_manager.deferred_job_queue.plugins['LiftOverTransferPlugin'].get_job_status( jobid )
joblabel = 'Download liftOver (%s to %s)' % ( liftover.params[ 'from_genome' ], liftover.params[ 'to_genome' ] )
elif jobtype == 'transfer':
job = sa.query( model.TransferJob ).filter_by( id=jobid ).first()
joblabel = 'Download Genome'
lib/galaxy/web/controllers/dataset.py
+9 -6
View File
@@ -203,12 +203,12 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistoryMixin, Use
job_id=job.id,
job_tool_id=job.tool_id,
job_command_line=job.command_line,
job_stderr=job.stderr,
job_stdout=job.stdout,
job_info=job.info,
job_traceback=job.traceback,
job_stderr=util.unicodify( job.stderr ),
job_stdout=util.unicodify( job.stdout ),
job_info=util.unicodify( job.info ),
job_traceback=util.unicodify( job.traceback ),
email=email,
message=message )
message=util.unicodify( message ) )
frm = to_address
# Check email a bit
email = email.strip()
@@ -644,7 +644,10 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistoryMixin, Use
dataset = self.get_dataset( trans, id, False, True )
if not dataset:
web.httpexceptions.HTTPNotFound()
return self.get_item_annotation_str( trans.sa_session, trans.user, dataset )
annotation = self.get_item_annotation_str( trans.sa_session, trans.user, dataset )
if annotation and isinstance( annotation, unicode ):
annotation = annotation.encode( 'ascii', 'replace' ) #paste needs ascii here
return annotation
@web.expose
def display_at( self, trans, dataset_id, filename=None, **kwd ):
lib/galaxy/web/controllers/library_common.py
+1
View File
@@ -1752,6 +1752,7 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin ):
if action == 'zip':
# Can't use mkstemp - the file must not exist first
tmpd = tempfile.mkdtemp()
util.umask_fix_perms( tmpd, trans.app.config.umask, 0777, self.app.config.gid )
tmpf = os.path.join( tmpd, 'library_download.' + action )
if ziptype == '64' and trans.app.config.upstream_gzip:
archive = zipfile.ZipFile( tmpf, 'w', zipfile.ZIP_STORED, True )
lib/galaxy/web/controllers/phyloviz.py
+97
View File
@@ -0,0 +1,97 @@
import pkg_resources
pkg_resources.require( "bx-python" )
from galaxy.util.json import to_json_string, from_json_string
from galaxy.web.base.controller import *
from galaxy.visualization.phyloviz.phyloviz_dataprovider import Phyloviz_DataProvider
class PhyloVizController( BaseUIController, UsesVisualizationMixin, UsesHistoryDatasetAssociationMixin, SharableMixin ):
"""
Controller for phyloViz browser interface.
"""
def __init__(self, app ):
BaseUIController.__init__( self, app )
@web.expose
@web.require_login()
def index( self, trans, dataset_id = None, **kwargs ):
"""
The index method is called using phyloviz/ with a dataset id passed in.
The relevant data set is then retrieved via get_json_from_datasetId which interfaces with the parser
The json representation of the phylogenetic tree along with the config is then written in the .mako template and passed back to the user
"""
json, config = self.get_json_from_datasetId(trans, dataset_id)
config["saved_visualization"] = False
return trans.fill_template( "visualization/phyloviz.mako", data = json, config=config)
@web.expose
def visualization(self, trans, id):
"""
Called using a viz_id (id) to retrieved stored visualization data (in json format) and all the viz_config
"""
viz = self.get_visualization(trans, id)
config = self.get_visualization_config(trans, viz)
config["saved_visualization"] = True
data = config["root"]
return trans.fill_template( "visualization/phyloviz.mako", data = data, config=config)
@web.expose
@web.json
def load_visualization_json(self, trans, viz_id):
"""
Though not used in current implementation, this provides user with a convenient method to retrieve the viz_data & viz_config via json.
"""
viz = self.get_visualization(trans, viz_id)
viz_config = self.get_visualization_config(trans, viz)
viz_config["saved_visualization"] = True
return {
"data" : viz_config["root"],
"config" : viz_config
}
@web.expose
@web.json
def getJsonData(self, trans, dataset_id, treeIndex=0):
"""
Method to retrieve data asynchronously via json format. Retriving from here rather than
making a direct datasets/ call allows for some processing and event capturing
"""
treeIndex = int(treeIndex)
json, config = self.get_json_from_datasetId(trans, dataset_id, treeIndex)
packedJson = {
"data" : json,
"config" : config
}
return packedJson
def get_json_from_datasetId(self, trans, dataset_id, treeIndex=0):
"""
For interfacing phyloviz controllers with phyloviz visualization data provider (parsers)
"""
dataset = self.get_dataset(trans, dataset_id)
fileExt, filepath = dataset.ext, dataset.file_name # .name stores the name of the dataset from the orginal upload
json, config = "", {} # config contains properties of the tree and file
if fileExt == "json":
something, json = self.get_data(dataset)
else:
try:
pd = Phyloviz_DataProvider()
json, config = pd.parseFile(filepath, fileExt)
json = json[treeIndex]
except Exception:
pass
config["title"] = dataset.display_name()
config["ext"] = fileExt
config["dataset_id"] = dataset_id
config["treeIndex"] = treeIndex
return json, config
lib/galaxy/web/controllers/tracks.py
+29 -1
View File
@@ -345,6 +345,20 @@ class TracksController( BaseUIController, UsesVisualizationMixin, UsesHistoryDat
# Have data if we get here
return { "status": messages.DATA, "valid_chroms": valid_chroms }
@web.json
def feature_loc( self, trans, hda_ldda, dataset_id, query ):
"""
Returns features, locations in dataset that match query. Format is a
list of features; each feature is a list itself: [name, location]
"""
dataset = self.get_hda_or_ldda( trans, hda_ldda, dataset_id )
converted_dataset = dataset.get_converted_dataset( trans, "fli" )
data_provider = FeatureLocationIndexDataProvider( converted_dataset=converted_dataset )
if data_provider:
return data_provider.get_data( query )
else:
return 'None'
@web.json
def data( self, trans, hda_ldda, dataset_id, chrom, low, high, start_val=0, max_vals=None, **kwargs ):
@@ -373,7 +387,21 @@ class TracksController( BaseUIController, UsesVisualizationMixin, UsesHistoryDat
return return_message
extra_info = None
if 'index' in data_sources and data_sources['index']['name'] == "summary_tree" and kwargs.get("mode", "Auto") == "Auto":
mode = kwargs.get( "mode", "Auto" )
# Handle histogram mode uniquely for now:
if mode == "Coverage":
# Get summary using minimal cutoffs.
tracks_dataset_type = data_sources['index']['name']
converted_dataset = dataset.get_converted_dataset( trans, tracks_dataset_type )
indexer = get_data_provider( tracks_dataset_type )( converted_dataset, dataset )
summary = indexer.get_data( chrom, low, high, resolution=kwargs[ 'resolution' ], detail_cutoff=0, draw_cutoff=0 )
if summary == "detail":
# Use maximum level of detail--2--to get summary data no matter the resolution.
summary = indexer.get_data( chrom, low, high, resolution=kwargs[ 'resolution' ], level=2, detail_cutoff=0, draw_cutoff=0 )
frequencies, max_v, avg_v, delta = summary
return { 'dataset_type': tracks_dataset_type, 'data': frequencies, 'max': max_v, 'avg': avg_v, 'delta': delta }
if 'index' in data_sources and data_sources['index']['name'] == "summary_tree" and mode == "Auto":
# Only check for summary_tree if it's Auto mode (which is the default)
#
# Have to choose between indexer and data provider
lib/galaxy/web/controllers/visualization.py
+7 -3
View File
@@ -5,7 +5,7 @@ from galaxy.web.framework.helpers import time_ago, grids, iff
from galaxy.util.sanitize_html import sanitize_html
class VisualizationListGrid( grids.Grid ):
def get_link( item ):
def get_url_args( item ):
"""
Returns dictionary used to create item link.
"""
@@ -16,6 +16,10 @@ class VisualizationListGrid( grids.Grid ):
action = "paramamonster"
elif item.type == "circster":
action = "circster"
elif item.type == "phyloviz":
# Support phyloviz
controller = "phyloviz"
action = "visualization"
return dict( controller=controller, action=action, id=item.id )
# Grid definition
@@ -24,7 +28,7 @@ class VisualizationListGrid( grids.Grid ):
default_sort_key = "-update_time"
default_filter = dict( title="All", deleted="False", tags="All", sharing="All" )
columns = [
grids.TextColumn( "Title", key="title", attach_popup=True, link=get_link ),
grids.TextColumn( "Title", key="title", attach_popup=True, link=get_url_args ),
grids.TextColumn( "Type", key="type" ),
grids.TextColumn( "Dbkey", key="dbkey" ),
grids.IndividualTagsColumn( "Tags", key="tags", model_tag_association_class=model.VisualizationTagAssociation, filterable="advanced", grid_name="VisualizationListGrid" ),
@@ -42,7 +46,7 @@ class VisualizationListGrid( grids.Grid ):
grids.GridAction( "Create new visualization", dict( action='create' ) )
]
operations = [
grids.GridOperation( "View/Edit", allow_multiple=False, url_args=dict( controller='tracks', action='browser' ) ),
grids.GridOperation( "View/Edit", allow_multiple=False, url_args=get_url_args ),
grids.GridOperation( "Edit Attributes", allow_multiple=False, url_args=dict( action='edit') ),
grids.GridOperation( "Copy", allow_multiple=False, condition=( lambda item: not item.deleted ), async_compatible=False, url_args=dict( action='clone') ),
grids.GridOperation( "Share or Publish", allow_multiple=False, condition=( lambda item: not item.deleted ), async_compatible=False ),
lib/galaxy/web/form_builder.py
+17 -17
View File
@@ -4,7 +4,7 @@ Classes for generating HTML forms
import logging, sys, os, time
from cgi import escape
from galaxy.util import restore_text, relpath, nice_size
from galaxy.util import restore_text, relpath, nice_size, unicodify
from galaxy.web import url_for
from binascii import hexlify
@@ -34,8 +34,8 @@ class TextField(BaseField):
self.size = int( size or 10 )
self.value = value or ""
def get_html( self, prefix="", disabled=False ):
return '<input type="text" name="%s%s" size="%d" value="%s"%s>' \
% ( prefix, self.name, self.size, escape( str( self.value ), quote=True ), self.get_disabled_str( disabled ) )
return unicodify( '<input type="text" name="%s%s" size="%d" value="%s"%s>' \
% ( prefix, self.name, self.size, escape( str( self.value ), quote=True ), self.get_disabled_str( disabled ) ) )
def set_size(self, size):
self.size = int( size )
@@ -53,8 +53,8 @@ class PasswordField(BaseField):
self.size = int( size or 10 )
self.value = value or ""
def get_html( self, prefix="", disabled=False ):
return '<input type="password" name="%s%s" size="%d" value="%s"%s>' \
% ( prefix, self.name, self.size, escape( str( self.value ), quote=True ), self.get_disabled_str( disabled ) )
return unicodify( '<input type="password" name="%s%s" size="%d" value="%s"%s>' \
% ( prefix, self.name, self.size, escape( str( self.value ), quote=True ), self.get_disabled_str( disabled ) ) )
def set_size(self, size):
self.size = int( size )
@@ -74,8 +74,8 @@ class TextArea(BaseField):
self.cols = int(self.size[-1])
self.value = value or ""
def get_html( self, prefix="", disabled=False ):
return '<textarea name="%s%s" rows="%d" cols="%d"%s>%s</textarea>' \
% ( prefix, self.name, self.rows, self.cols, self.get_disabled_str( disabled ), escape( str( self.value ), quote=True ) )
return unicodify( '<textarea name="%s%s" rows="%d" cols="%d"%s>%s</textarea>' \
% ( prefix, self.name, self.rows, self.cols, self.get_disabled_str( disabled ), escape( str( self.value ), quote=True ) ) )
def set_size(self, rows, cols):
self.rows = rows
self.cols = cols
@@ -111,8 +111,8 @@ class CheckboxField(BaseField):
# parsing the request, the value 'true' in the hidden field actually means it is NOT checked.
# See the is_checked() method below. The prefix is necessary in each case to ensure functional
# correctness when the param is inside a conditional.
return '<input type="checkbox" id="%s" name="%s" value="true"%s%s%s><input type="hidden" name="%s%s" value="true"%s>' \
% ( id_name, id_name, checked_text, self.get_disabled_str( disabled ), self.refresh_on_change_text, prefix, self.name, self.get_disabled_str( disabled ) )
return unicodify( '<input type="checkbox" id="%s" name="%s" value="true"%s%s%s><input type="hidden" name="%s%s" value="true"%s>' \
% ( id_name, id_name, checked_text, self.get_disabled_str( disabled ), self.refresh_on_change_text, prefix, self.name, self.get_disabled_str( disabled ) ) )
@staticmethod
def is_checked( value ):
if value == True:
@@ -148,7 +148,7 @@ class FileField(BaseField):
ajax_text = ""
if self.ajax:
ajax_text = ' galaxy-ajax-upload="true"'
return '<input type="file" name="%s%s"%s%s>' % ( prefix, self.name, ajax_text, value_text )
return unicodify( '<input type="file" name="%s%s"%s%s>' % ( prefix, self.name, ajax_text, value_text ) )
class FTPFileField(BaseField):
"""
@@ -223,7 +223,7 @@ class HiddenField(BaseField):
self.name = name
self.value = value or ""
def get_html( self, prefix="" ):
return '<input type="hidden" name="%s%s" value="%s">' % ( prefix, self.name, escape( str( self.value ), quote=True ) )
return unicodify( '<input type="hidden" name="%s%s" value="%s">' % ( prefix, self.name, escape( str( self.value ), quote=True ) ) )
class SelectField(BaseField):
"""
@@ -308,7 +308,7 @@ class SelectField(BaseField):
rval.append( '<div%s><input type="checkbox" name="%s%s" value="%s" id="%s"%s%s><label class="inline" for="%s">%s</label></div>' % \
( style, prefix, self.name, escaped_value, uniq_id, selected_text, self.get_disabled_str( disabled ), uniq_id, escape( str( text ), quote=True ) ) )
ctr += 1
return "\n".join( rval )
return unicodify( "\n".join( rval ) )
def get_html_radio( self, prefix="", disabled=False ):
rval = []
ctr = 0
@@ -333,7 +333,7 @@ class SelectField(BaseField):
uniq_id,
text ) )
ctr += 1
return "\n".join( rval )
return unicodify( "\n".join( rval ) )
def get_html_default( self, prefix="", disabled=False ):
if self.multiple:
multiple = " multiple"
@@ -357,7 +357,7 @@ class SelectField(BaseField):
rval.insert( 0, '<select name="%s%s"%s%s%s%s%s>' % \
( prefix, self.name, multiple, size, self.refresh_on_change_text, last_selected_value, self.get_disabled_str( disabled ) ) )
rval.append( '</select>' )
return "\n".join( rval )
return unicodify( "\n".join( rval ) )
def get_selected( self, return_label=False, return_value=False, multi=False ):
'''
Return the currently selected option's label, value or both as a tuple. For
@@ -513,7 +513,7 @@ class DrillDownField( BaseField ):
find_expanded_options( expanded_options, self.options )
recurse_options( rval, self.options, drilldown_id, expanded_options )
rval.append( '</div>' )
return '\n'.join( rval )
return unicodify( '\n'.join( rval ) )
class AddressField(BaseField):
@staticmethod
@@ -688,8 +688,8 @@ class LibraryField( BaseField ):
else:
ldda_ids = "||".join( [ self.trans.security.encode_id( ldda.id ) for ldda in self.lddas ] )
text = "<br />".join( [ "%s. %s" % (i+1, ldda.name) for i, ldda in enumerate(self.lddas)] )
return '<a href="javascript:void(0);" class="add-librarydataset">%s</a> \
<input type="hidden" name="%s%s" value="%s">' % ( text, prefix, self.name, escape( str(ldda_ids), quote=True ) )
return unicodify( '<a href="javascript:void(0);" class="add-librarydataset">%s</a> \
<input type="hidden" name="%s%s" value="%s">' % ( text, prefix, self.name, escape( str(ldda_ids), quote=True ) ) )
def get_display_text(self):
if self.ldda:
lib/galaxy/web/framework/helpers/grids.py
+6 -3
View File
@@ -728,9 +728,12 @@ class GridOperation( object ):
self.global_operation = global_operation
def get_url_args( self, item ):
if self.url_args:
temp = dict( self.url_args )
temp['id'] = item.id
return temp
if hasattr( self.url_args, '__call__' ):
url_args = self.url_args( item )
else:
url_args = dict( self.url_args )
url_args['id'] = item.id
return url_args
else:
return dict( operation=self.label, id=item.id )
def allowed( self, item ):
lib/galaxy/webapps/community/config.py
+1
View File
@@ -87,6 +87,7 @@ class Configuration( object ):
self.server_name = ''
self.job_manager = ''
self.default_job_handlers = []
self.default_cluster_job_runner = 'local:///'
self.job_handlers = []
self.tool_handlers = []
self.tool_runners = []
lib/galaxy/webapps/community/controllers/admin.py
+8 -3
View File
@@ -696,9 +696,14 @@ class AdminController( BaseUIController, Admin ):
owner = repository_name_owner_list[ 1 ]
repository = get_repository_by_name_and_owner( trans, name, owner )
try:
reset_all_metadata_on_repository( trans, trans.security.encode_id( repository.id ) )
log.debug( "Successfully reset metadata on repository %s" % repository.name )
successful_count += 1
invalid_file_tups = reset_all_metadata_on_repository( trans, trans.security.encode_id( repository.id ) )
if invalid_file_tups:
message = generate_message_for_invalid_tools( invalid_file_tups, repository, None, as_html=False )
log.debug( message )
unsuccessful_count += 1
else:
log.debug( "Successfully reset metadata on repository %s" % repository.name )
successful_count += 1
except Exception, e:
log.debug( "Error attempting to reset metadata on repository '%s': %s" % ( repository.name, str( e ) ) )
unsuccessful_count += 1
lib/galaxy/webapps/community/controllers/common.py
+204 -161
View File
@@ -1,13 +1,14 @@
import os, string, socket, logging, simplejson, binascii, tempfile
import os, string, socket, logging, simplejson, binascii, tempfile, filecmp
from time import strftime
from datetime import *
from galaxy.datatypes.checkers import *
from galaxy.tools import *
from galaxy.util.json import from_json_string, to_json_string
from galaxy.util.hash_util import *
from galaxy.util.shed_util import clone_repository, generate_metadata_for_changeset_revision, get_changectx_for_changeset, get_config_from_disk
from galaxy.util.shed_util import get_configured_ui, get_named_tmpfile_from_ctx, handle_sample_tool_data_table_conf_file, INITIAL_CHANGELOG_HASH
from galaxy.util.shed_util import reset_tool_data_tables, reversed_upper_bounded_changelog, strip_path
from galaxy.util.shed_util import check_tool_input_params, clone_repository, copy_sample_file, generate_metadata_for_changeset_revision
from galaxy.util.shed_util import get_changectx_for_changeset, get_config_from_disk, get_configured_ui, get_named_tmpfile_from_ctx
from galaxy.util.shed_util import handle_sample_tool_data_table_conf_file, INITIAL_CHANGELOG_HASH, load_tool_from_config, reset_tool_data_tables
from galaxy.util.shed_util import reversed_upper_bounded_changelog, strip_path
from galaxy.web.base.controller import *
from galaxy.webapps.community import model
from galaxy.model.orm import *
@@ -105,6 +106,42 @@ class ItemRatings( UsesItemRatings ):
trans.sa_session.flush()
return item_rating
def add_tool_versions( trans, id, repository_metadata, changeset_revisions ):
# Build a dictionary of { 'tool id' : 'parent tool id' } pairs for each tool in repository_metadata.
metadata = repository_metadata.metadata
tool_versions_dict = {}
for tool_dict in metadata.get( 'tools', [] ):
# We have at least 2 changeset revisions to compare tool guids and tool ids.
parent_id = get_parent_id( trans,
id,
tool_dict[ 'id' ],
tool_dict[ 'version' ],
tool_dict[ 'guid' ],
changeset_revisions )
tool_versions_dict[ tool_dict[ 'guid' ] ] = parent_id
if tool_versions_dict:
repository_metadata.tool_versions = tool_versions_dict
trans.sa_session.add( repository_metadata )
trans.sa_session.flush()
def can_use_tool_config_disk_file( trans, repository, repo, file_path, changeset_revision ):
"""
Determine if repository's tool config file on disk can be used. This method is restricted to tool config files since, with the
exception of tool config files, multiple files with the same name will likely be in various directories in the repository and we're
comparing file names only (not relative paths).
"""
if not file_path or not os.path.exists( file_path ):
# The file no longer exists on disk, so it must have been deleted at some previous point in the change log.
return False
if changeset_revision == repository.tip:
return True
file_name = strip_path( file_path )
latest_version_of_file = get_latest_tool_config_revision_from_repository_manifest( repo, file_name, changeset_revision )
can_use_disk_file = filecmp.cmp( file_path, latest_version_of_file )
try:
os.unlink( latest_version_of_file )
except:
pass
return can_use_disk_file
def changeset_is_malicious( trans, id, changeset_revision, **kwd ):
"""Check the malicious flag in repository metadata for a specified change set"""
repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision )
@@ -221,6 +258,16 @@ def compare_workflows( ancestor_workflows, current_workflows ):
else:
return 'subset'
return 'not equal and not subset'
def copy_disk_sample_files_to_dir( trans, repo_files_dir, dest_path ):
sample_files = []
for root, dirs, files in os.walk( repo_files_dir ):
if root.find( '.hg' ) < 0:
for name in files:
if name.endswith( '.sample' ):
relative_path = os.path.join( root, name )
copy_sample_file( trans.app, relative_path, dest_path=dest_path )
sample_files_copied.append( name )
return sample_files
def copy_file_from_disk( filename, repo_dir, dir ):
file_path = None
found = False
@@ -240,9 +287,7 @@ def copy_file_from_disk( filename, repo_dir, dir ):
tmp_filename = None
return tmp_filename
def copy_file_from_manifest( repo, ctx, filename, dir ):
"""
Copy the latest version of the file named filename from the repository manifest to the directory to which dir refers.
"""
"""Copy the latest version of the file named filename from the repository manifest to the directory to which dir refers."""
for changeset in reversed_upper_bounded_changelog( repo, ctx ):
changeset_ctx = repo.changectx( changeset )
fctx = get_file_context_from_ctx( changeset_ctx, filename )
@@ -277,6 +322,42 @@ def generate_clone_url( trans, repository_id ):
return '%s://%s%s/repos/%s/%s' % ( protocol, username, base, repository.user.username, repository.name )
else:
return '%s/repos/%s/%s' % ( base_url, repository.user.username, repository.name )
def generate_message_for_invalid_tools( invalid_file_tups, repository, metadata_dict, as_html=True, displaying_invalid_tool=False ):
if as_html:
new_line = '<br/>'
bold_start = '<b>'
bold_end = '</b>'
else:
new_line = '\n'
bold_start = ''
bold_end = ''
message = ''
if not displaying_invalid_tool:
if metadata_dict:
message += "Metadata was defined for some items in revision '%s'. " % str( repository.tip )
message += "Correct the following problems if necessary and reset metadata.%s" % new_line
else:
message += "Metadata cannot be defined for revision '%s' so this revision cannot be automatically " % str( repository.tip )
message += "installed into a local Galaxy instance. Correct the following problems and reset metadata.%s" % new_line
for itc_tup in invalid_file_tups:
tool_file, exception_msg = itc_tup
if exception_msg.find( 'No such file or directory' ) >= 0:
exception_items = exception_msg.split()
missing_file_items = exception_items[ 7 ].split( '/' )
missing_file = missing_file_items[ -1 ].rstrip( '\'' )
if missing_file.endswith( '.loc' ):
sample_ext = '%s.sample' % missing_file
else:
sample_ext = missing_file
correction_msg = "This file refers to a missing file %s%s%s. " % ( bold_start, str( missing_file ), bold_end )
correction_msg += "Upload a file named %s%s%s to the repository to correct this error." % ( bold_start, sample_ext, bold_end )
else:
if as_html:
correction_msg = exception_msg
else:
correction_msg = exception_msg.replace( '<br/>', new_line ).replace( '<b>', bold_start ).replace( '</b>', bold_end )
message += "%s%s%s - %s%s" % ( bold_start, tool_file, bold_end, correction_msg, new_line )
return message
def generate_tool_guid( trans, repository, tool ):
"""
Generate a guid for the received tool. The form of the guid is
@@ -287,6 +368,19 @@ def generate_tool_guid( trans, repository, tool ):
repository.name,
tool.id,
tool.version )
def get_absolute_path_to_file_in_repository( repo_files_dir, file_name ):
file_path = None
found = False
for root, dirs, files in os.walk( repo_files_dir ):
if root.find( '.hg' ) < 0:
for name in files:
if name == file_name:
file_path = os.path.abspath( os.path.join( root, name ) )
found = True
break
if found:
break
return file_path
def get_category( trans, id ):
"""Get a category from the database"""
return trans.sa_session.query( trans.model.Category ).get( trans.security.decode_id( id ) )
@@ -318,12 +412,44 @@ def get_file_context_from_ctx( ctx, filename ):
if deleted:
return 'DELETED'
return None
def get_ctx_file_path_from_manifest( filename, repo, changeset_revision ):
"""Get the ctx file path for the latest revision of filename from the repository manifest up to the value of changeset_revision."""
stripped_filename = strip_path( filename )
for changeset in reversed_upper_bounded_changelog( repo, changeset_revision ):
manifest_changeset_revision = str( repo.changectx( changeset ) )
manifest_ctx = repo.changectx( changeset )
for ctx_file in manifest_ctx.files():
ctx_file_name = strip_path( ctx_file )
if ctx_file_name == stripped_filename:
return manifest_ctx, ctx_file
return None, None
def get_latest_repository_metadata( trans, decoded_repository_id ):
"""Get last metadata defined for a specified repository from the database"""
return trans.sa_session.query( trans.model.RepositoryMetadata ) \
.filter( trans.model.RepositoryMetadata.table.c.repository_id == decoded_repository_id ) \
.order_by( trans.model.RepositoryMetadata.table.c.id.desc() ) \
.first()
def get_latest_tool_config_revision_from_repository_manifest( repo, filename, changeset_revision ):
"""
Get the latest revision of a tool config file named filename from the repository manifest up to the value of changeset_revision.
This method is restricted to tool_config files rather than any file since it is likely that, with the exception of tool config files,
multiple files will have the same name in various directories within the repository.
"""
stripped_filename = strip_path( filename )
for changeset in reversed_upper_bounded_changelog( repo, changeset_revision ):
manifest_ctx = repo.changectx( changeset )
for ctx_file in manifest_ctx.files():
ctx_file_name = strip_path( ctx_file )
if ctx_file_name == stripped_filename:
fctx = manifest_ctx[ ctx_file ]
fh = tempfile.NamedTemporaryFile( 'wb' )
tmp_filename = fh.name
fh.close()
fh = open( tmp_filename, 'wb' )
fh.write( fctx.data() )
fh.close()
return tmp_filename
return None
def get_list_of_copied_sample_files( repo, ctx, dir ):
"""
Find all sample files (files in the repository with the special .sample extension) in the reversed repository manifest up to ctx. Copy
@@ -416,8 +542,8 @@ def get_repository_by_name_and_owner( trans, name, owner ):
def get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ):
"""Get metadata for a specified repository change set from the database"""
# Make sure there are no duplicate records, and return the single unique record for the changeset_revision. Duplicate records were somehow
# creatd in the past. This may or may not be resolved, so when it is confirmed that the cause of duplicate records has been corrected, tweak
# this method accordingly.
# created in the past. The cause of this issue has been resolved, but we'll leave this method as is for a while longer to ensure all duplicate
# records are removed.
all_metadata_records = trans.sa_session.query( trans.model.RepositoryMetadata ) \
.filter( and_( trans.model.RepositoryMetadata.table.c.repository_id == trans.security.decode_id( id ),
trans.model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) ) \
@@ -542,90 +668,61 @@ def handle_email_alerts( trans, repository, content_alert_str='', new_repo_alert
log.exception( "An error occurred sending a tool shed repository update alert by email." )
def is_downloadable( metadata_dict ):
return 'datatypes' in metadata_dict or 'tools' in metadata_dict or 'workflows' in metadata_dict
def load_tool( trans, config_file ):
"""Load a single tool from the file named by `config_file` and return an instance of `Tool`."""
# Parse XML configuration file and get the root element
tree = util.parse_xml( config_file )
root = tree.getroot()
if root.tag == 'tool':
# Allow specifying a different tool subclass to instantiate
if root.find( "type" ) is not None:
type_elem = root.find( "type" )
module = type_elem.get( 'module', 'galaxy.tools' )
cls = type_elem.get( 'class' )
mod = __import__( module, globals(), locals(), [cls] )
ToolClass = getattr( mod, cls )
elif root.get( 'tool_type', None ) is not None:
ToolClass = tool_types.get( root.get( 'tool_type' ) )
else:
ToolClass = Tool
return ToolClass( config_file, root, trans.app )
return None
def load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_config_filename ):
"""
Return a loaded tool whose tool config file name (e.g., filtering.xml) is the value of tool_config_filename. The value of changeset_revision
is a valid (downloadable) changset revision. The tool config will be located in the repository manifest between the received valid changeset
revision and the first changeset revision in the repository, searching backwards.
"""
def load_from_tmp_config( toolbox, ctx, ctx_file, work_dir ):
tool = None
message = ''
tmp_tool_config = get_named_tmpfile_from_ctx( ctx, ctx_file, work_dir )
if tmp_tool_config:
element_tree = util.parse_xml( tmp_tool_config )
element_tree_root = element_tree.getroot()
# Look for code files required by the tool config.
tmp_code_files = []
for code_elem in element_tree_root.findall( 'code' ):
code_file_name = code_elem.get( 'file' )
tmp_code_file_name = copy_file_from_manifest( repo, ctx, code_file_name, work_dir )
if tmp_code_file_name:
tmp_code_files.append( tmp_code_file_name )
try:
tool = toolbox.load_tool( tmp_tool_config )
except KeyError, e:
message = '<b>%s</b> - This file requires an entry for %s in the tool_data_table_conf.xml file. ' % ( tool_config_filename, str( e ) )
message += 'Upload a file named tool_data_table_conf.xml.sample to the repository that includes the required entry to correct this error. '
except Exception, e:
message = 'Error loading tool: %s. ' % str( e )
for tmp_code_file in tmp_code_files:
try:
os.unlink( tmp_code_file )
except:
pass
try:
os.unlink( tmp_tool_config )
except:
pass
return tool, message
original_tool_data_path = trans.app.config.tool_data_path
tool_config_filename = strip_path( tool_config_filename )
repository = get_repository( trans, repository_id )
repo_files_dir = repository.repo_path
repo = hg.repository( get_configured_ui(), repo_files_dir )
ctx = get_changectx_for_changeset( repo, changeset_revision )
message = ''
tool = None
can_use_disk_file = False
tool_config_filepath = get_absolute_path_to_file_in_repository( repo_files_dir, tool_config_filename )
work_dir = tempfile.mkdtemp()
sample_files, deleted_sample_files = get_list_of_copied_sample_files( repo, ctx, dir=work_dir )
if sample_files:
trans.app.config.tool_data_path = work_dir
if 'tool_data_table_conf.xml.sample' in sample_files:
# Load entries into the tool_data_tables if the tool requires them.
tool_data_table_config = os.path.join( work_dir, 'tool_data_table_conf.xml' )
error, correction_msg = handle_sample_tool_data_table_conf_file( trans.app, tool_data_table_config )
found = False
# Get the latest revision of the tool config from the repository manifest up to the value of changeset_revision.
for changeset in reversed_upper_bounded_changelog( repo, changeset_revision ):
manifest_changeset_revision = str( repo.changectx( changeset ) )
manifest_ctx = repo.changectx( changeset )
for ctx_file in manifest_ctx.files():
ctx_file_name = strip_path( ctx_file )
if ctx_file_name == tool_config_filename:
found = True
break
if found:
tool, message = load_from_tmp_config( trans.app.toolbox, manifest_ctx, ctx_file, work_dir )
break
can_use_disk_file = can_use_tool_config_disk_file( trans, repository, repo, tool_config_filepath, changeset_revision )
if can_use_disk_file:
# Copy all sample files from disk to a temporary directory since the sample files may be in multiple directories.
sample_files = copy_disk_sample_files_to_dir( trans, repo_files_dir, work_dir )
if sample_files:
trans.app.config.tool_data_path = work_dir
if 'tool_data_table_conf.xml.sample' in sample_files:
# Load entries into the tool_data_tables if the tool requires them.
tool_data_table_config = os.path.join( work_dir, 'tool_data_table_conf.xml' )
error, correction_msg = handle_sample_tool_data_table_conf_file( trans.app, tool_data_table_config )
tool, valid, message = load_tool_from_config( trans.app, tool_config_filepath )
if tool is not None:
invalid_files_and_errors_tups = check_tool_input_params( trans.app,
repo_files_dir,
tool_config_filename,
tool,
sample_files,
webapp='community' )
if invalid_files_and_errors_tups:
message = generate_message_for_invalid_tools( invalid_files_and_errors_tups,
repository,
metadata_dict=None,
as_html=True,
displaying_invalid_tool=True )
status = 'error'
else:
# The desired version of the tool config is no longer on disk, so create a temporary work environment and copy the tool config and dependent files.
ctx = get_changectx_for_changeset( repo, changeset_revision )
# We're not currently doing anything with the returned list of deleted_sample_files here. It is intended to help handle sample files that are in
# the manifest, but have been deleted from disk.
sample_files, deleted_sample_files = get_list_of_copied_sample_files( repo, ctx, dir=work_dir )
if sample_files:
trans.app.config.tool_data_path = work_dir
if 'tool_data_table_conf.xml.sample' in sample_files:
# Load entries into the tool_data_tables if the tool requires them.
tool_data_table_config = os.path.join( work_dir, 'tool_data_table_conf.xml' )
error, correction_msg = handle_sample_tool_data_table_conf_file( trans.app, tool_data_table_config )
manifest_ctx, ctx_file = get_ctx_file_path_from_manifest( tool_config_filename, repo, changeset_revision )
if manifest_ctx and ctx_file:
tool, message = load_tool_from_tmp_config( trans, repo, manifest_ctx, ctx_file, work_dir )
try:
shutil.rmtree( work_dir )
except:
@@ -634,50 +731,32 @@ def load_tool_from_changeset_revision( trans, repository_id, changeset_revision,
trans.app.config.tool_data_path = original_tool_data_path
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
reset_tool_data_tables( trans.app )
return tool, message
def load_tool_from_tmp_directory( trans, repo, repo_dir, ctx, filename, dir ):
is_tool_config = False
return repository, tool, message
def load_tool_from_tmp_config( trans, repo, ctx, ctx_file, work_dir ):
tool = None
valid = False
error_message = ''
tmp_config = get_named_tmpfile_from_ctx( ctx, filename, dir )
if tmp_config:
if not ( check_binary( tmp_config ) or check_image( tmp_config ) or check_gzip( tmp_config )[ 0 ]
or check_bz2( tmp_config )[ 0 ] or check_zip( tmp_config ) ):
message = ''
tmp_tool_config = get_named_tmpfile_from_ctx( ctx, ctx_file, work_dir )
if tmp_tool_config:
element_tree = util.parse_xml( tmp_tool_config )
element_tree_root = element_tree.getroot()
# Look for code files required by the tool config.
tmp_code_files = []
for code_elem in element_tree_root.findall( 'code' ):
code_file_name = code_elem.get( 'file' )
tmp_code_file_name = copy_file_from_manifest( repo, ctx, code_file_name, work_dir )
if tmp_code_file_name:
tmp_code_files.append( tmp_code_file_name )
tool, valid, message = load_tool_from_config( trans.app, tmp_tool_config )
for tmp_code_file in tmp_code_files:
try:
# Make sure we're looking at a tool config and not a display application config or something else.
element_tree = util.parse_xml( tmp_config )
element_tree_root = element_tree.getroot()
is_tool_config = element_tree_root.tag == 'tool'
except Exception, e:
log.debug( "Error parsing %s, exception: %s" % ( tmp_config, str( e ) ) )
is_tool_config = False
if is_tool_config:
# Load entries into the tool_data_tables if the tool requires them.
tool_data_table_config = copy_file_from_manifest( repo, ctx, 'tool_data_table_conf.xml.sample', dir )
if tool_data_table_config:
error, correction_msg = handle_sample_tool_data_table_conf_file( trans.app, tool_data_table_config )
# Look for code files required by the tool config. The directory to which dir refers should be removed by the caller.
for code_elem in element_tree_root.findall( 'code' ):
code_file_name = code_elem.get( 'file' )
if not os.path.exists( os.path.join( dir, code_file_name ) ):
tmp_code_file_name = copy_file_from_disk( code_file_name, repo_dir, dir )
if tmp_code_file_name is None:
tmp_code_file_name = copy_file_from_manifest( repo, ctx, code_file_name, dir )
try:
tool = load_tool( trans, tmp_config )
valid = True
except KeyError, e:
valid = False
error_message = 'This file requires an entry for "%s" in the tool_data_table_conf.xml file. Upload a file ' % str( e )
error_message += 'named tool_data_table_conf.xml.sample to the repository that includes the required entry to correct '
error_message += 'this error. '
except Exception, e:
valid = False
error_message = str( e )
# Reset the tool_data_tables by loading the empty tool_data_table_conf.xml file.
reset_tool_data_tables( trans.app )
return is_tool_config, valid, tool, error_message
os.unlink( tmp_code_file )
except:
pass
try:
os.unlink( tmp_tool_config )
except:
pass
return tool, message
def new_tool_metadata_required( trans, repository, metadata_dict ):
"""
Compare the last saved metadata for each tool in the repository with the new metadata in metadata_dict to determine if a new repository_metadata
@@ -854,28 +933,12 @@ def reset_all_metadata_on_repository( trans, id, **kwd ):
clean_repository_metadata( trans, id, changeset_revisions )
# Set tool version information for all downloadable changeset revisions. Get the list of changeset revisions from the changelog.
reset_all_tool_versions( trans, id, repo )
return invalid_file_tups
def set_repository_metadata( trans, repository, content_alert_str='', **kwd ):
"""
Set metadata using the repository's current disk files, returning specific error messages (if any) to alert the repository owner that the changeset
has problems.
"""
def add_tool_versions( trans, id, repository_metadata, changeset_revisions ):
# Build a dictionary of { 'tool id' : 'parent tool id' } pairs for each tool in repository_metadata.
metadata = repository_metadata.metadata
tool_versions_dict = {}
for tool_dict in metadata.get( 'tools', [] ):
# We have at least 2 changeset revisions to compare tool guids and tool ids.
parent_id = get_parent_id( trans,
id,
tool_dict[ 'id' ],
tool_dict[ 'version' ],
tool_dict[ 'guid' ],
changeset_revisions )
tool_versions_dict[ tool_dict[ 'guid' ] ] = parent_id
if tool_versions_dict:
repository_metadata.tool_versions = tool_versions_dict
trans.sa_session.add( repository_metadata )
trans.sa_session.flush()
message = ''
status = 'done'
encoded_id = trans.security.encode_id( repository.id )
@@ -931,27 +994,7 @@ def set_repository_metadata( trans, repository, content_alert_str='', **kwd ):
message += "be defined so this revision cannot be automatically installed into a local Galaxy instance."
status = "error"
if invalid_file_tups:
if metadata_dict:
message += "Metadata was defined for some items in revision '%s'. " % str( repository.tip )
message += "Correct the following problems if necessary and reset metadata.<br/>"
else:
message += "Metadata cannot be defined for revision '%s' so this revision cannot be automatically " % str( repository.tip )
message += "installed into a local Galaxy instance. Correct the following problems and reset metadata.<br/>"
for itc_tup in invalid_file_tups:
tool_file, exception_msg = itc_tup
if exception_msg.find( 'No such file or directory' ) >= 0:
exception_items = exception_msg.split()
missing_file_items = exception_items[ 7 ].split( '/' )
missing_file = missing_file_items[ -1 ].rstrip( '\'' )
if missing_file.endswith( '.loc' ):
sample_ext = '%s.sample' % missing_file
else:
sample_ext = missing_file
correction_msg = "This file refers to a missing file <b>%s</b>. " % str( missing_file )
correction_msg += "Upload a file named <b>%s</b> to the repository to correct this error." % sample_ext
else:
correction_msg = exception_msg
message += "<b>%s</b> - %s<br/>" % ( tool_file, correction_msg )
message = generate_message_for_invalid_tools( invalid_file_tups, repository, metadata_dict )
status = 'error'
return message, status
def set_repository_metadata_due_to_new_tip( trans, repository, content_alert_str=None, **kwd ):
lib/galaxy/webapps/community/controllers/repository.py
+68 -25
View File
@@ -9,9 +9,9 @@ from galaxy.webapps.community.model import directory_hash_id
from galaxy.web.framework.helpers import time_ago, iff, grids
from galaxy.util.json import from_json_string, to_json_string
from galaxy.model.orm import *
from galaxy.util.shed_util import create_repo_info_dict, get_changectx_for_changeset, get_configured_ui, get_repository_file_contents, NOT_TOOL_CONFIGS
from galaxy.util.shed_util import open_repository_files_folder, reversed_lower_upper_bounded_changelog, reversed_upper_bounded_changelog, strip_path
from galaxy.util.shed_util import to_html_escaped, update_repository
from galaxy.util.shed_util import create_repo_info_dict, get_changectx_for_changeset, get_configured_ui, get_repository_file_contents, load_tool_from_config
from galaxy.util.shed_util import NOT_TOOL_CONFIGS, open_repository_files_folder, reversed_lower_upper_bounded_changelog, reversed_upper_bounded_changelog
from galaxy.util.shed_util import strip_path, to_html_escaped, update_repository, url_join
from galaxy.tool_shed.encoding_util import *
from common import *
@@ -246,6 +246,28 @@ class EmailAlertsRepositoryListGrid( RepositoryListGrid ):
grids.GridAction( "User preferences", dict( controller='user', action='index', cntrller='repository', webapp='community' ) )
]
class WritableRepositoryListGrid( RepositoryListGrid ):
def build_initial_query( self, trans, **kwd ):
# TODO: improve performance by adding a db table associating users with repositories for which they have write access.
username = kwd[ 'username' ]
clause_list = []
for repository in trans.sa_session.query( self.model_class ) \
.filter( self.model_class.table.c.deleted == False ):
allow_push = repository.allow_push
if allow_push:
allow_push_usernames = allow_push.split( ',' )
if username in allow_push_usernames:
clause_list.append( self.model_class.table.c.id == repository.id )
if clause_list:
return trans.sa_session.query( self.model_class ) \
.filter( or_( *clause_list ) ) \
.join( model.User.table ) \
.outerjoin( model.RepositoryCategoryAssociation.table ) \
.outerjoin( model.Category.table )
# Return an empty query.
return trans.sa_session.query( self.model_class ) \
.filter( self.model_class.table.c.id < 0 )
class ValidRepositoryListGrid( RepositoryListGrid ):
class CategoryColumn( grids.TextColumn ):
def get_value( self, trans, grid, repository ):
@@ -393,6 +415,7 @@ class RepositoryController( BaseUIController, ItemRatings ):
email_alerts_repository_list_grid = EmailAlertsRepositoryListGrid()
category_list_grid = CategoryListGrid()
valid_category_list_grid = ValidCategoryListGrid()
writable_repository_list_grid = WritableRepositoryListGrid()
def __add_hgweb_config_entry( self, trans, repository, repository_path ):
# Add an entry in the hgweb.config file for a new repository. An entry looks something like:
@@ -458,7 +481,10 @@ class RepositoryController( BaseUIController, ItemRatings ):
metadata = downloadable_revision.metadata
invalid_tools = metadata.get( 'invalid_tools', [] )
for invalid_tool_config in invalid_tools:
invalid_tools_dict[ invalid_tool_config ] = ( repository.id, repository.name, downloadable_revision.changeset_revision )
invalid_tools_dict[ invalid_tool_config ] = ( repository.id,
repository.name,
repository.user.username,
downloadable_revision.changeset_revision )
else:
for repository in trans.sa_session.query( trans.model.Repository ) \
.filter( and_( trans.model.Repository.table.c.deleted == False,
@@ -468,7 +494,10 @@ class RepositoryController( BaseUIController, ItemRatings ):
metadata = downloadable_revision.metadata
invalid_tools = metadata.get( 'invalid_tools', [] )
for invalid_tool_config in invalid_tools:
invalid_tools_dict[ invalid_tool_config ] = ( repository.id, repository.name, downloadable_revision.changeset_revision )
invalid_tools_dict[ invalid_tool_config ] = ( repository.id,
repository.name,
repository.user.username,
downloadable_revision.changeset_revision )
return trans.fill_template( '/webapps/community/repository/browse_invalid_tools.mako',
cntrller=cntrller,
invalid_tools_dict=invalid_tools_dict,
@@ -513,12 +542,15 @@ class RepositoryController( BaseUIController, ItemRatings ):
repository_id = kwd.get( 'id', None )
repository = get_repository( trans, repository_id )
kwd[ 'f-email' ] = repository.user.email
elif operation == "my_repositories":
elif operation == "repositories_i_own":
# Eliminate the current filters if any exist.
for k, v in kwd.items():
if k.startswith( 'f-' ):
del kwd[ k ]
kwd[ 'f-email' ] = trans.user.email
elif operation == "writable_repositories":
kwd[ 'username' ] = trans.user.username
return self.writable_repository_list_grid( trans, **kwd )
elif operation == "repositories_by_category":
# Eliminate the current filters if any exist.
for k, v in kwd.items():
@@ -720,9 +752,10 @@ class RepositoryController( BaseUIController, ItemRatings ):
update = 'true'
no_update = 'false'
else:
# Start building up the url to redirect back to the calling Galaxy instance.
url = '%sadmin_toolshed/update_to_changeset_revision?tool_shed_url=%s' % ( galaxy_url, url_for( '/', qualified=True ) )
url += '&name=%s&owner=%s&changeset_revision=%s&latest_changeset_revision=' % ( repository.name, repository.user.username, changeset_revision )
# Start building up the url to redirect back to the calling Galaxy instance.
url = url_join( galaxy_url,
'admin_toolshed/update_to_changeset_revision?tool_shed_url=%s&name=%s&owner=%s&changeset_revision=%s&latest_changeset_revision=' % \
( url_for( '/', qualified=True ), repository.name, repository.user.username, changeset_revision ) )
if changeset_revision == repository.tip:
# If changeset_revision is the repository tip, there are no additional updates.
if from_update_manager:
@@ -882,8 +915,7 @@ class RepositoryController( BaseUIController, ItemRatings ):
message = util.restore_text( params.get( 'message', '' ) )
status = params.get( 'status', 'done' )
webapp = get_webapp( trans, **kwd )
repository = get_repository( trans, repository_id )
tool, message = load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_config )
repository, tool, message = load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_config )
tool_state = self.__new_state( trans )
is_malicious = changeset_is_malicious( trans, repository_id, repository.tip )
try:
@@ -1366,10 +1398,9 @@ class RepositoryController( BaseUIController, ItemRatings ):
"""Send the list of repository_ids and changeset_revisions to Galaxy so it can begin the installation process."""
galaxy_url = trans.get_cookie( name='toolshedgalaxyurl' )
# Redirect back to local Galaxy to perform install.
url = '%sadmin_toolshed/prepare_for_install' % galaxy_url
url += '?tool_shed_url=%s' % url_for( '/', qualified=True )
url += '&repository_ids=%s' % ','.join( util.listify( repository_ids ) )
url += '&changeset_revisions=%s' % ','.join( util.listify( changeset_revisions ) )
url = url_join( galaxy_url,
'admin_toolshed/prepare_for_install?tool_shed_url=%s&repository_ids=%s&changeset_revisions=%s' % \
( url_for( '/', qualified=True ), ','.join( util.listify( repository_ids ) ), ','.join( util.listify( changeset_revisions ) ) ) )
return trans.response.send_redirect( url )
@web.expose
def load_invalid_tool( self, trans, repository_id, tool_config, changeset_revision, **kwd ):
@@ -1378,8 +1409,7 @@ class RepositoryController( BaseUIController, ItemRatings ):
status = params.get( 'status', 'error' )
webapp = get_webapp( trans, **kwd )
repository_clone_url = generate_clone_url( trans, repository_id )
repository = get_repository( trans, repository_id )
tool, error_message = load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_config )
repository, tool, error_message = load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_config )
tool_state = self.__new_state( trans )
is_malicious = changeset_is_malicious( trans, repository_id, repository.tip )
try:
@@ -1752,9 +1782,14 @@ class RepositoryController( BaseUIController, ItemRatings ):
status=status )
@web.expose
def reset_all_metadata( self, trans, id, **kwd ):
reset_all_metadata_on_repository( trans, id, **kwd )
message = "All repository metadata has been reset."
status = 'done'
invalid_file_tups = reset_all_metadata_on_repository( trans, id, **kwd )
if invalid_file_tups:
repository = get_repository( trans, id )
message = generate_message_for_invalid_tools( invalid_file_tups, repository, None )
status = 'error'
else:
message = "All repository metadata has been reset."
status = 'done'
return trans.response.send_redirect( web.url_for( controller='repository',
action='manage_repository',
id=id,
@@ -2185,6 +2220,8 @@ class RepositoryController( BaseUIController, ItemRatings ):
status = params.get( 'status', 'done' )
webapp = get_webapp( trans, **kwd )
repository = get_repository( trans, repository_id )
repo_files_dir = repository.repo_path
repo = hg.repository( get_configured_ui(), repo_files_dir )
tool_metadata_dict = {}
tool_lineage = []
tool = None
@@ -2195,12 +2232,18 @@ class RepositoryController( BaseUIController, ItemRatings ):
if 'tools' in metadata:
for tool_metadata_dict in metadata[ 'tools' ]:
if tool_metadata_dict[ 'id' ] == tool_id:
relative_path_to_tool_config = tool_metadata_dict[ 'tool_config' ]
guid = tool_metadata_dict[ 'guid' ]
try:
# We may be attempting to load a tool that no longer exists in the repository tip.
tool = load_tool( trans, os.path.abspath( tool_metadata_dict[ 'tool_config' ] ) )
except:
tool = None
full_path = os.path.abspath( relative_path_to_tool_config )
can_use_disk_file = can_use_tool_config_disk_file( trans, repository, repo, full_path, changeset_revision )
if can_use_disk_file:
tool, valid, message = load_tool_from_config( trans.app, full_path )
else:
# We're attempting to load a tool using a config that no longer exists on disk.
work_dir = tempfile.mkdtemp()
manifest_ctx, ctx_file = get_ctx_file_path_from_manifest( relative_path_to_tool_config, repo, changeset_revision )
if manifest_ctx and ctx_file:
tool, message = load_tool_from_tmp_config( trans, repo, manifest_ctx, ctx_file, work_dir )
break
if guid:
tool_lineage = self.get_versions_of_tool( trans, repository, repository_metadata, guid )
lib/galaxy/webapps/community/controllers/workflow.py
+1 -1
View File
@@ -48,7 +48,7 @@ class RepoToolModule( ToolModule ):
self.errors = None
for tool_dict in tools_metadata:
if self.tool_id in [ tool_dict[ 'id' ], tool_dict[ 'guid' ] ]:
self.tool, message = load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_dict[ 'tool_config' ] )
repository, self.tool, message = load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_dict[ 'tool_config' ] )
if message and self.tool is None:
self.errors = 'unavailable'
break
scripts/api/workflow_delete.py
+27
View File
@@ -0,0 +1,27 @@
#!/usr/bin/env python
"""
# ---------------------------------------------- #
# PARKLAB, Author: RPARK
API example script for deleting workflows
# ---------------------------------------------- #
Example calls:
python workflow_delete.py <api_key> <galaxy_url>/api/workflows/<workflow id> True
"""
import os, sys
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import delete
try:
assert sys.argv[2]
except IndexError:
print 'usage: %s key url [purge (true/false)] ' % os.path.basename( sys.argv[0] )
sys.exit( 1 )
try:
data = {}
data[ 'purge' ] = sys.argv[3]
except IndexError:
pass
delete( sys.argv[1], sys.argv[2], data )
scripts/api/workflow_execute_parameters.py
+55
View File
@@ -0,0 +1,55 @@
#!/usr/bin/env python
"""
# ---------------------------------------------- #
# PARKLAB, Author: RPARK
# ---------------------------------------------- #
Execute workflows from the command line.
Example calls:
python workflow_execute.py <api_key> <galaxy_url>/api/workflows <workflow_id> 'hist_id=<history_id>' '38=hda=<file_id>' 'param=tool=name=value'
python workflow_execute_parameters.py <api_key> http://localhost:8080/api/workflows 1cd8e2f6b131e891 'Test API' '69=ld=a799d38679e985db' '70=ld=33b43b4e7093c91f' 'param=peakcalling_spp=aligner=bowtie' 'param=bowtie_wrapper=suppressHeader=True' 'param=peakcalling_spp=window_size=1000'
"""
import os, sys
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import submit
def main():
try:
print("workflow_execute:py:");
data = {}
data['workflow_id'] = sys.argv[3]
data['history'] = sys.argv[4]
data['ds_map'] = {}
#########################################################
### Trying to pass in parameter for my own dictionary ###
data['parameters'] = {};
# DBTODO If only one input is given, don't require a step
# mapping, just use it for everything?
for v in sys.argv[5:]:
print("Multiple arguments ");
print(v);
try:
step, src, ds_id = v.split('=');
data['ds_map'][step] = {'src':src, 'id':ds_id};
except ValueError:
print("VALUE ERROR:");
wtype, wtool, wparam, wvalue = v.split('=');
try:
data['parameters'][wtool] = {'param':wparam, 'value':wvalue}
except ValueError:
print("TOOL ID ERROR:");
except IndexError:
print 'usage: %s key url workflow_id history step=src=dataset_id' % os.path.basename(sys.argv[0])
sys.exit(1)
submit( sys.argv[1], sys.argv[2], data )
if __name__ == '__main__':
main()
scripts/api/workflow_import_from_file_rpark.py
+39
View File
@@ -0,0 +1,39 @@
#!/usr/bin/env python
"""
python rpark_import_workflow_from_file.py 35a24ae2643785ff3d046c98ea362c7f http://localhost:8080/api/workflows/import 'spp_submodule.ga'
python rpark_import_workflow_from_file.py 35a24ae2643785ff3d046c98ea362c7f http://localhost:8080/api/workflows/import 'spp_submodule.ga'
"""
import os, sys
sys.path.insert( 0, os.path.dirname( __file__ ) )
from common import submit
### Rpark edit ###
import simplejson
def openWorkflow(in_file):
with open(in_file) as f:
temp_data = simplejson.load(f)
return temp_data;
try:
assert sys.argv[2]
except IndexError:
print 'usage: %s key url [name] ' % os.path.basename( sys.argv[0] )
sys.exit( 1 )
try:
#data = {}
#data[ 'name' ] = sys.argv[3]
data = {};
workflow_dict = openWorkflow(sys.argv[3]);
data ['workflow'] = workflow_dict;
except IndexError:
pass
submit( sys.argv[1], sys.argv[2], data )
scripts/migrate_tools/0004_tools.sh
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
cd `dirname $0`/../..
python ./scripts/migrate_tools/migrate_tools.py 0004_tools.xml $@
scripts/migrate_tools/0004_tools.xml
+12
View File
@@ -0,0 +1,12 @@
<?xml version="1.0"?>
<toolshed name="toolshed.g2.bx.psu.edu">
<repository name="blast_datatypes" description="Datatypes for BLAST" changeset_revision="e1c29f302301" />
<repository name="ncbi_blast_plus" description="Galaxy wrappers for NCBI BLAST+" changeset_revision="d375502056f1">
<tool id="blastxml_to_tabular" version="0.0.8" file="blastxml_to_tabular.xml"/>
<tool id="ncbi_blastn_wrapper" version="0.0.11" file="ncbi_blastn_wrapper.xml"/>
<tool id="ncbi_blastp_wrapper" version="0.0.11" file="ncbi_blastp_wrapper.xml"/>
<tool id="ncbi_blastx_wrapper" version="0.0.11" file="ncbi_blastx_wrapper.xml"/>
<tool id="ncbi_tblastn_wrapper" version="0.0.11" file="ncbi_tblastn_wrapper.xml"/>
<tool id="ncbi_tblastx_wrapper" version="0.0.11" file="ncbi_tblastx_wrapper.xml"/>
</repository>
</toolshed>
static/june_2007_style/blue_colors.ini
+1
View File
@@ -55,6 +55,7 @@ history_queued_bg=#EEEEEE
peek_table_header=#023858
# Masthead
masthead_bg=#2C3143
masthead_bg_highlight=#333
masthead_text=#eeeeee
masthead_bg_hatch=-
masthead_link=#eeeeee
static/june_2007_style/galaxy_bootstrap_variables.less
+1 -1
View File
@@ -144,7 +144,7 @@
// -------------------------
@navbarHeight: 32px;
@navbarBackground: @masthead_bg;
@navbarBackgroundHighlight: @grayDark;
@navbarBackgroundHighlight: @masthead_bg_highlight;
@navbarText: @grayLight;
@navbarLinkColor: @grayLight;
static/scripts/autocomplete_tagging.js → static/scripts/galaxy.autocom_tagging.js
View File
static/scripts/jquery.event.drag.js.patch
-10
View File
@@ -1,10 +0,0 @@
96c96
< if ( drag.dragging ){
---
> if ( drag.dragging ) {
99c99,101
< }
---
> } else {
> hijack( event, "dragclickonly", elem );
> }
static/scripts/IE7.js → static/scripts/libs/IE/IE7.js
View File
static/scripts/IE8.js → static/scripts/libs/IE/IE8.js
View File
static/scripts/blank.gif → static/scripts/libs/IE/blank.gif
View File
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 48 B

After

Width:  |  Height:  |  Size: 48 B

static/scripts/excanvas.js → static/scripts/libs/IE/excanvas.js
View File
static/scripts/ie7-recalc.js → static/scripts/libs/IE/ie7-recalc.js
View File
static/scripts/libs/backbone-relational.js → static/scripts/libs/backbone/backbone-relational.js
View File
static/scripts/libs/backbone.js → static/scripts/libs/backbone/backbone.js
View File
static/scripts/bootstrap.js → static/scripts/libs/bootstrap.js
Vendored
View File
static/scripts/farbtastic.js → static/scripts/libs/farbtastic.js
View File
static/scripts/jqtouch.js → static/scripts/libs/jquery/jqtouch.js
Vendored
View File
static/scripts/jquery.autocomplete.js → static/scripts/libs/jquery/jquery.autocomplete.js
View File
static/scripts/jquery.cookie.js → static/scripts/libs/jquery/jquery.cookie.js
View File
static/scripts/jquery.dynatree.js → static/scripts/libs/jquery/jquery.dynatree.js
View File
static/scripts/jquery.event.drag.js → static/scripts/libs/jquery/jquery.event.drag.js
View File
static/scripts/jquery.event.drop.js → static/scripts/libs/jquery/jquery.event.drop.js
View File
static/scripts/jquery.event.hover.js → static/scripts/libs/jquery/jquery.event.hover.js
View File
static/scripts/jquery.form.js → static/scripts/libs/jquery/jquery.form.js
View File
static/scripts/jquery.js → static/scripts/libs/jquery/jquery.js
Vendored
View File
static/scripts/jquery.mousewheel.js → static/scripts/libs/jquery/jquery.mousewheel.js
View File
static/scripts/jquery.rating.js → static/scripts/libs/jquery/jquery.rating.js
View File
static/scripts/jquery.tipsy.js → static/scripts/libs/jquery/jquery.tipsy.js
View File
static/scripts/ui.core.js → static/scripts/libs/jquery/jquery.ui.core.js
Vendored
View File
static/scripts/jquery.ui.sortable.slider.js → static/scripts/libs/jquery/jquery.ui.sortable.slider.js
View File
static/scripts/jquery.wymeditor.js → static/scripts/libs/jquery/jquery.wymeditor.js
View File
static/scripts/jstorage.js → static/scripts/libs/jquery/jstorage.js
Vendored
View File
static/scripts/json2.js → static/scripts/libs/json2.js
View File
static/scripts/packed/blank.gif
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 48 B

static/scripts/packed/bootstrap.js
Vendored
-1
View File
File diff suppressed because one or more lines are too long
static/scripts/packed/autocomplete_tagging.js → static/scripts/packed/galaxy.autocom_tagging.js
View File
static/scripts/packed/IE7.js → static/scripts/packed/libs/IE/IE7.js
View File
static/scripts/packed/IE8.js → static/scripts/packed/libs/IE/IE8.js
View File

Some files were not shown because too many files have changed in this diff Show More