- Performance of indexers much improved.

- Indexing for tracks done in background with a visual treatment done to Trackster

- DB builds can be uploaded by a user (chromInfo/len extension).

- TODO: Add ability to change the dbkey of a dataset to any arbitrary string value.
This commit is contained in:
Ian Schenck
2009-04-23 13:46:52 -04:00
parent 20ae1859f3
commit feda672ffa
23 changed files with 245 additions and 55 deletions
+4 -1
View File
@@ -8,6 +8,9 @@
<converter file="interval_to_coverage.xml" target_datatype="coverage"/>
</datatype>
<datatype extension="binseq.zip" type="galaxy.datatypes.images:Binseq" mimetype="application/zip" display_in_upload="true"/>
<datatype extension="len" type="galaxy.datatypes.chrominfo:ChromInfo" display_in_upload="true">
<!-- no converters yet -->
</datatype>
<datatype extension="coverage" type="galaxy.datatypes.coverage:LastzCoverage" display_in_upload="true">
<indexer file="coverage.xml" />
</datatype>
@@ -31,7 +34,7 @@
<datatype extension="html" type="galaxy.datatypes.images:Html" mimetype="text/html"/>
<datatype extension="interval" type="galaxy.datatypes.interval:Interval" display_in_upload="true">
<converter file="interval_to_bed_converter.xml" target_datatype="bed"/>
<indexer file="interval.xml" />
<indexer file="interval_awk.xml" />
</datatype>
<datatype extension="jpg" type="galaxy.datatypes.images:Image" mimetype="image/jpeg"/>
<datatype extension="laj" type="galaxy.datatypes.images:Laj"/>
+2 -2
View File
@@ -40,7 +40,7 @@ class Data( object ):
__metaclass__ = DataMeta
"""Add metadata elements"""
MetadataElement( name="dbkey", desc="Database/Build", default="?", param=metadata.SelectParameter, multiple=False, values=util.dbnames, no_value="?" )
MetadataElement( name="dbkey", desc="Database/Build", default="?", param=metadata.DBKeyParameter, multiple=False, no_value="?" )
"""Stores the set of display applications, and viewing methods, supported by this datatype """
supported_display_apps = {}
@@ -242,7 +242,7 @@ class Data( object ):
def after_edit( self, dataset ):
"""This function is called on the dataset after metadata is edited."""
dataset.clear_associated_files( metadata_safe = True )
@property
def has_resolution(self):
return False
+1 -1
View File
@@ -37,7 +37,7 @@ def write_chrom(max, out_base, instream):
os.rename( fname+".npy", fname )
# Write average
for window in 10, 100, 1000, 10000:
for window in 10, 100, 1000, 10000, 100000:
input = scores.copy()
size = len( input )
input.resize( ( ( size / window ), window ) )
@@ -0,0 +1,43 @@
BEGIN {
# from galaxy.utils
mapped_chars[">"] = "__gt__"
mapped_chars["<"] = "__lt__"
mapped_chars["'"] = "__sq__"
mapped_chars["\""] = "__dq__"
mapped_chars["\\["] = "__ob__"
mapped_chars["\\]"] = "__cb__"
mapped_chars["\\{"] = "__oc__"
mapped_chars["\\}"] = "__cc__"
mapped_chars["@"] = "__at__"
# additional, not in galaxy.utils
mapped_chars["/"] = "__fs__"
mapped_chars["^manifest\.tab$"] = "__manifest.tab__"
}
function escape_filename( name )
{
for( char in mapped_chars ) {
gsub( char, mapped_chars[char], name )
}
return name
}
!_[$chrom]++ {
# close files only when we switch to a new one.
fn && close(fn)
fn = storepath "/" escape_filename($1) }
{
print $0 >> fn;
# the || part is needed to catch 0 length chromosomes, which
# should never happen but...
if ($end > chroms[$chrom] || !chroms[$chrom])
chroms[$chrom] = $end }
END {
fn = storepath "/manifest.tab"
for( x in chroms ) {
# add line to manifest
print x "\t" chroms[x] >> fn
chromfile = storepath "/" escape_filename(x)
# sort in-place
system( "sort -f -n -k " chrom " -k " start " -k " end " -o " chromfile " " chromfile )
close(chromfile)
}
}
@@ -29,13 +29,11 @@ def divide( intervals, out_path ):
manifest[chrom] = max(manifest.get(chrom,0),line.end)
if not lastchrom == chrom:
if current_file:
current_file.flush()
current_file.close()
current_file = open( os.path.join( out_path, "%s" % chrom), "a" )
print >> current_file, "\t".join(line)
lastchrom = chrom
if current_file:
current_file.flush()
current_file.close()
return manifest
@@ -0,0 +1,16 @@
<tool id="INDEXER_Interval_0" name="Index Interval for Track Viewer">
<!-- Used internally to generate track indexes -->
<command interpreter="awk -f">interval.awk
chrom=${input_dataset.metadata.chromCol} start=${input_dataset.metadata.startCol}
end=${input_dataset.metadata.endCol} strand=${input_dataset.metadata.strandCol}
storepath=${store_path}
$input_dataset 2&gt;&amp;1
</command>
<inputs>
<page>
<param format="interval" name="input_dataset" type="data" label="Choose intervals"/>
</page>
</inputs>
<help>
</help>
</tool>
+4 -2
View File
@@ -18,6 +18,8 @@ pkg_resources.require("numpy>=1.2.1")
from numpy import *
import tempfile
import os
from galaxy.tracks.store import sanitize_name
def write_chrom(max, out_base, instream):
@@ -35,7 +37,7 @@ def write_chrom(max, out_base, instream):
os.rename( fname+".npy", fname )
# Write average
for window in 10, 100, 1000, 10000:
for window in 10, 100, 1000, 10000, 100000:
input = scores.copy()
size = len( input )
input.resize( ( ( size / window ), window ) )
@@ -60,7 +62,7 @@ def main():
LEN[chrom] = max2( LEN.get(chrom,0), pos+1 )
for chrom, stream in chroms.items():
stream.seek(0)
prefix = os.path.join(sys.argv[2], chrom)
prefix = os.path.join(sys.argv[2], sanitize_name(chrom))
write_chrom( LEN[chrom], prefix, stream )
manifest_file = open( os.path.join( sys.argv[2], "manifest.tab" ),"w" )
+2 -2
View File
@@ -792,7 +792,7 @@ class Wiggle( Tabular ):
# Determine appropriate resolution to plot ~1000 points
resolution = ( 10 ** math.ceil( math.log10( range / 1000 ) ) )
# Restrict to valid range
resolution = min( resolution, 10000 )
resolution = min( resolution, 100000 )
resolution = max( resolution, 1 )
# Memory map the array (don't load all the data)
data = numpy.load( data )
@@ -809,7 +809,7 @@ class Wiggle( Tabular ):
# Determine appropriate resolution to plot ~1000 points
resolution = math.ceil( 10 ** math.ceil( math.log10( range / 1000 ) ) )
# Restrict to valid range
resolution = min( resolution, 10000 )
resolution = min( resolution, 100000 )
resolution = max( resolution, 1 )
return resolution
+16 -1
View File
@@ -289,7 +289,22 @@ class SelectParameter( MetadataParameter ):
if value is None: return []
if not isinstance( value, list ): return [value]
return value
class DBKeyParameter( SelectParameter ):
def get_html_field( self, value=None, context={}, other_values={}, values=None, **kwd):
try:
values = kwd['trans'].db_builds
except AttributeError: pass
return super(DBKeyParameter, self).get_html_field( value, context, other_values, values, **kwd)
def get_html( self, value=None, context={}, other_values={}, values=None, **kwd):
try:
values = kwd['trans'].db_builds
except AttributeError: pass
return super(DBKeyParameter, self).get_html( value, context, other_values, values, **kwd)
class RangeParameter( SelectParameter ):
def __init__( self, spec ):
SelectParameter.__init__( self, spec )
+1 -1
View File
@@ -3,7 +3,7 @@ Provides mapping between extensions and datatypes, mime-types, etc.
"""
import os
import logging
import data, tabular, interval, images, sequence, qualityscore, genetics, xml, coverage, tracks
import data, tabular, interval, images, sequence, qualityscore, genetics, xml, coverage, tracks, chrominfo
import galaxy.util
from galaxy.util.odict import odict
+11
View File
@@ -107,6 +107,7 @@ class DefaultToolAction( object ):
out_data = {}
# Collect any input datasets from the incoming parameters
inp_data = self.collect_input_datasets( tool, incoming, trans )
# Deal with input dataset names, 'dbkey' and types
input_names = []
input_ext = 'data'
@@ -119,6 +120,16 @@ class DefaultToolAction( object ):
data = NoneDataset( datatypes_registry = trans.app.datatypes_registry )
if data.dbkey not in [None, '?']:
input_dbkey = data.dbkey
# Collect chromInfo dataset and add as parameters to incoming
db_datasets = {}
db_dataset = trans.db_dataset_for( input_dbkey )
if db_dataset:
db_datasets[ "chromInfo" ] = db_dataset
incoming[ "chromInfo" ] = db_dataset.file_name
else:
incoming[ "chromInfo" ] = os.path.join( trans.app.config.tool_data_path, 'shared','ucsc','chrom', "%s.len" % input_dbkey )
inp_data.update( db_datasets )
# Determine output dataset permission/roles list
existing_datasets = [ inp for inp in inp_data.values() if inp ]
+1 -1
View File
@@ -21,7 +21,7 @@ class UploadToolAction( object ):
def execute( self, tool, trans, incoming={}, set_output_hid = True ):
data_file = incoming['file_data']
file_type = incoming['file_type']
dbkey = incoming['dbkey']
dbkey = incoming['other_dbkey'] or incoming['dbkey']
url_paste = incoming['url_paste']
is_multi_byte = False
space_to_tab = False
+3 -3
View File
@@ -657,7 +657,7 @@ class GenomeBuildParameter( SelectToolParameter ):
>>> # Create a mock transcation with 'hg17' as the current build
>>> from galaxy.util.bunch import Bunch
>>> trans = Bunch( history=Bunch( genome_build='hg17' ) )
>>> trans = Bunch( history=Bunch( genome_build='hg17' ), db_builds=util.dbnames )
>>> p = GenomeBuildParameter( None, XML(
... '''
@@ -692,10 +692,10 @@ class GenomeBuildParameter( SelectToolParameter ):
"""
def get_options( self, trans, other_values ):
last_used_build = trans.history.genome_build
for dbkey, build_name in util.dbnames:
for dbkey, build_name in trans.db_builds:
yield build_name, dbkey, ( dbkey == last_used_build )
def get_legal_values( self, trans, other_values ):
return set( dbkey for dbkey, _ in util.dbnames )
return set( dbkey for dbkey, _ in trans.db_builds )
class ColumnListParameter( SelectToolParameter ):
"""
+13 -1
View File
@@ -1,5 +1,17 @@
import os
import re
from string import Template
from galaxy.util import sanitize_text
# extra mappings/escape to keep users from traversing around the
# filesystem and wreaking havoc
extra_mappings = { r"/": "__fs__", r"^manifest.tab$": "__manifest.tab__" }
def sanitize_name( name ):
name = sanitize_text( name )
for key, value in extra_mappings.items():
name = re.sub( key, value, name )
return name
class TemplateSubber( object ):
def __init__(self, obj):
@@ -56,7 +68,7 @@ class TrackStore( object ):
fd.close()
def _get_object_path( self, chrom, resolution ):
object_name = chrom
object_name = sanitize_name(chrom)
if resolution: object_name += "_%d" % resolution
return os.path.join( self.path, object_name )
+5 -1
View File
@@ -234,11 +234,15 @@ class RootController( BaseController ):
if spec.get("readonly"):
continue
optional = params.get("is_"+name, None)
other = params.get("or_"+name, None)
if optional and optional == 'true':
# optional element... == 'true' actually means it is NOT checked (and therefore omitted)
setattr(data.metadata, name, None)
else:
setattr( data.metadata, name, spec.unwrap( params.get (name, None) ) )
if other:
setattr( data.metadata, name, other )
else:
setattr( data.metadata, name, spec.unwrap( params.get (name, None) ) )
data.datatype.after_edit( data )
trans.app.model.flush()
+23 -18
View File
@@ -1,13 +1,11 @@
from mako import exceptions
from mako.template import Template
from mako.lookup import TemplateLookup
import math
import mimeparse
from galaxy.tracks import messages
from galaxy.util.json import to_json_string
from galaxy.web.base.controller import *
from galaxy.web.framework import simplejson
from galaxy import web
from galaxy.tracks import messages
import mimeparse
from galaxy.util.json import to_json_string
import math
class MultiResponse(object):
"""
@@ -82,18 +80,19 @@ class WebRoot( BaseController ):
def build( self, trans, **kwargs ):
trans.session["track_sets"] = list(kwargs.keys())
trans.session.save()
waiting = False
for id, value in kwargs.items():
status = self.data_handler( trans, id )
if status == messages.PENDING:
waiting = True
if not waiting:
return trans.response.send_redirect( web.url_for( controller='tracks', action='chroms', dbkey=trans.session["track_dbkey"]) )
return trans.fill_template( 'tracks/build.mako' )
#waiting = False
#for id, value in kwargs.items():
# status = self.data_handler( trans, id )
# if status == messages.PENDING:
# waiting = True
#if not waiting:
return trans.response.send_redirect( web.url_for( controller='tracks/', action='index', chrom="" ) )
#return trans.fill_template( 'tracks/build.mako' )
@web.expose
def index(self, trans, **kwargs):
tracks = []
dbkey = ""
for track in trans.session["track_sets"]:
dataset = trans.app.model.HistoryDatasetAssociation.get( track )
tracks.append({
@@ -101,17 +100,23 @@ class WebRoot( BaseController ):
"name": dataset.name,
"id": dataset.id
})
dbkey = dataset.dbkey
chrom = kwargs.get("chrom","")
LEN = self.chroms_handler(trans, trans.session["track_dbkey"]).get(chrom,0)
return trans.fill_template( 'tracks/index.mako',
tracks=tracks, chrom=chrom,
tracks=tracks, chrom=chrom, dbkey=dbkey,
LEN=LEN )
def chroms_handler(self, trans, dbkey ):
db_manifest = os.path.join( trans.app.config.tool_data_path, 'shared','ucsc','chrom', "%s.len" % dbkey )
db_manifest = trans.db_dataset_for( dbkey )
if not db_manifest:
db_manifest = os.path.join( trans.app.config.tool_data_path, 'shared','ucsc','chrom', "%s.len" % dbkey )
else:
db_manifest = db_manifest.file_name
manifest = {}
if os.path.exists( db_manifest ):
for line in open( db_manifest ):
if line.startswith("#"): continue
line = line.rstrip("\r\n")
fields = line.split("\t")
manifest[fields[0]] = int(fields[1])
+25
View File
@@ -547,6 +547,31 @@ class UniverseWebTransaction( base.DefaultWebTransaction ):
template = Template( source=template_string,
searchList=[context or kwargs, dict(caller=self)] )
return str(template)
@property
def db_builds( self ):
"""
Returns the builds defined by galaxy and the builds defined by
the user (chromInfo in history).
"""
dbnames = list()
datasets = self.app.model.HistoryDatasetAssociation.filter_by(deleted=False, history_id=self.history.id, extension="len").all()
if len(datasets) > 0:
dbnames.append( (util.dbnames.default_value, '--------- User Defined Builds ----------') )
for dataset in datasets:
dbnames.append( (dataset.dbkey, dataset.name) )
dbnames.extend( util.dbnames )
return dbnames
def db_dataset_for( self, dbkey ):
"""
Returns the db_file dataset associated/needed by `dataset`, or `None`.
"""
datasets = self.app.model.HistoryDatasetAssociation.filter_by(deleted=False, history_id=self.history.id, extension="len").all()
for ds in datasets:
if dbkey == ds.dbkey:
return ds
return None
class FormBuilder( object ):
"""
+20 -7
View File
@@ -85,7 +85,7 @@ $.extend( TiledTrack.prototype, Track.prototype, {
var resolution = Math.pow( 10, Math.ceil( Math.log( range / DENSITY ) / Math.log( 10 ) ) );
resolution = Math.max( resolution, 1 );
resolution = Math.min( resolution, 10000 );
resolution = Math.min( resolution, 100000 );
var parent_element = $("<div style='position: relative;'></div>");
this.content_div.children( ":first" ).remove();
@@ -152,10 +152,20 @@ $.extend( DataCache.prototype, {
var low = position * DENSITY * resolution;
var high = ( position + 1 ) * DENSITY * resolution;
cache[resolution][position] = { state: "loading" };
$.getJSON( "data" + this.type, { chr: this.view.chr, low: low, high: high, dataset_id: this.track.dataset_id }, function ( data ) {
cache[resolution][position] = { state: "loaded", values: data };
$(document).trigger( "redraw" );
});
// use closure to preserve this and parameters for getJSON
var fetcher = function (ref) {
return function () {
$.getJSON( "data" + ref.type, { chr: ref.view.chr, low: low, high: high, dataset_id: ref.track.dataset_id }, function ( data ) {
if( data == "pending" ) {
setTimeout( fetcher, 5000 );
} else {
cache[resolution][position] = { state: "loaded", values: data };
}
$(document).trigger( "redraw" );
});
};
}(this);
fetcher();
}
return cache[resolution][position];
}
@@ -288,8 +298,11 @@ $.extend( FeatureTrack.prototype, TiledTrack.prototype, {
var chunk = this.cache.get( resolution, tile_index );
if ( chunk.state == "loading" ) {
return null;
}
parent_element.addClass("loading");
return null;
} else {
parent_element.removeClass("loading");
}
var values = chunk.values;
for ( var index in values ) {
+4 -1
View File
@@ -85,7 +85,10 @@ body {
}
.loading {
background: #DDDDDD;
background-image: url("/static/images/loading_large_white_bg.gif");
background-position: center center;
background-repeat: no-repeat;
min-height: 100px;
}
.label-track .label {
+1 -1
View File
@@ -46,7 +46,7 @@
${spec.desc}:
</label>
<div style="float: left; width: 250px; margin-right: 10px;">
${data.metadata.get_html_by_name( name )}
${data.metadata.get_html_by_name( name, trans=trans )}
</div>
<div style="clear: both"></div>
</div>
+47 -8
View File
@@ -17,18 +17,19 @@ ${parent.stylesheets()}
${parent.late_javascripts()}
<script type="text/javascript" src="/static/scripts/jquery.event.drag.js"></script>
<script type="text/javascript" src="/static/scripts/trackster.js"></script>
<script>
<script type="text/javascript">
var view = new View( "${chrom}", ${LEN}, 0, ${LEN} );
var view = new View( "${chrom}", ${LEN}, 0, ${max(LEN,1)} );
var tracks = new TrackLayout( view );
var dbkey = "${dbkey}";
$(function() {
tracks.add( new LabelTrack( view, $("#viewport" ) ) );
%for track in tracks:
tracks.add( new ${track["type"]}( "${track["name"]}", view, $("#viewport" ), ${track["id"]} ) );
%endfor
$(document).bind( "redraw", function( e ) {
tracks.redraw();
});
@@ -56,9 +57,43 @@ ${parent.late_javascripts()}
view.high = new_high;
tracks.redraw();
});
tracks.redraw();
load_chroms();
});
var load_chroms = function () {
var fetcher = function (ref) {
return function () {
$.getJSON( "chroms", { dbkey: dbkey }, function ( data ) {
// Hacky - check length of "object"
var chrom_length = 0;
for (key in data) chrom_length++;
if( chrom_length == 0 ) {
setTimeout( fetcher, 5000 );
} else {
var chrom_options = '';
for (key in data) {
if( key == view.chr ) {
chrom_options += '<option value="' + key + '" selected="true">' + key + '</option>';
} else {
chrom_options += '<option value="' + key + '">' + key + '</option>';
}
}
$("#chrom").html(chrom_options);
$("#chrom").bind( "change", function ( e ) {
$("#chr").submit();
});
if( view.chr == "" ) {
$("#chrom option:first").attr("selected", true);
$("#chrom").trigger( "change" );
}
}
});
};
}(this);
fetcher();
};
</script>
</%def>
@@ -79,11 +114,14 @@ ${parent.late_javascripts()}
<div id="nav">
<div id="nav-controls">
<form name="chr" id="chr" method="GET">
<a href="#" onclick="javascript:view.left(5);tracks.redraw();">&lt;&lt;</a>
<a href="#" onclick="javascript:view.left(2);tracks.redraw();">&lt;</a>
<span style="display: inline-block; width: 30em; text-align: center;">Viewing ${chrom}:<span id="low">0</span>-<span id="high">180857866</span></span>
<span style="display: inline-block; width: 30em; text-align: center;">Viewing
<select id="chrom" name="chrom">
<option value="">loading</option>
</select>
<span id="low">0</span>-<span id="high">180857866</span></span>
<span style="display: inline-block; width: 10em;">
<a href="#" onclick="javascript:view.zoom_in(2);tracks.redraw();">+</a>
<a href="#" onclick="javascript:view.zoom_out(2);tracks.redraw();">-</a>
@@ -91,6 +129,7 @@ ${parent.late_javascripts()}
<a href="#" onclick="javascript:view.right(2);tracks.redraw();">&gt;</a>
<a href="#" onclick="javascript:view.right(5);tracks.redraw();">&gt;&gt;</a>
</form>
</div>
</div>
+1
View File
@@ -21,6 +21,7 @@
</options>
</param>
<param name="dbkey" type="genomebuild" label="Genome" />
<param name="other_dbkey" type="text" label="Or user-defined Genome" />
</inputs>
<help>
+2 -2
View File
@@ -1,6 +1,6 @@
<tool id="gops_complement_1" name="Complement">
<description>intervals of a query</description>
<command interpreter="python">gops_complement.py $input1 $output -1 ${input1.metadata.chromCol},${input1.metadata.startCol},${input1.metadata.endCol},${input1.metadata.strandCol} -l ${GALAXY_DATA_INDEX_DIR}/shared/ucsc/chrom/${dbkey}.len $allchroms</command>
<command interpreter="python">gops_complement.py $input1 $output -1 ${input1.metadata.chromCol},${input1.metadata.startCol},${input1.metadata.endCol},${input1.metadata.strandCol} -l ${chromInfo} $allchroms</command>
<inputs>
<param format="interval" name="input1" type="data">
<label>Complement regions of</label>
@@ -58,4 +58,4 @@ See Galaxy Interval Operation Screencasts_ (right click to open this link in ano
.. image:: ../static/operation_icons/gops_complement.gif
</help>
</tool>
</tool>