From feda672ffa7a98317837d6e464743188fa9b4e63 Mon Sep 17 00:00:00 2001 From: Ian Schenck Date: Thu, 23 Apr 2009 13:46:52 -0400 Subject: [PATCH] - Performance of indexers much improved. - Indexing for tracks done in background with a visual treatment done to Trackster - DB builds can be uploaded by a user (chromInfo/len extension). - TODO: Add ability to change the dbkey of a dataset to any arbitrary string value. --- datatypes_conf.xml.sample | 5 +- lib/galaxy/datatypes/data.py | 4 +- lib/galaxy/datatypes/indexers/coverage.py | 2 +- lib/galaxy/datatypes/indexers/interval.awk | 43 +++++++++++++++ lib/galaxy/datatypes/indexers/interval.py | 2 - .../datatypes/indexers/interval_awk.xml | 16 ++++++ lib/galaxy/datatypes/indexers/wiggle.py | 6 +- lib/galaxy/datatypes/interval.py | 4 +- lib/galaxy/datatypes/metadata.py | 17 +++++- lib/galaxy/datatypes/registry.py | 2 +- lib/galaxy/tools/actions/__init__.py | 11 ++++ lib/galaxy/tools/actions/upload.py | 2 +- lib/galaxy/tools/parameters/basic.py | 6 +- lib/galaxy/tracks/store.py | 14 ++++- lib/galaxy/web/controllers/root.py | 6 +- lib/galaxy/web/controllers/tracks.py | 41 ++++++++------ lib/galaxy/web/framework/__init__.py | 25 +++++++++ static/scripts/trackster.js | 27 ++++++--- static/trackster.css | 5 +- templates/dataset/edit_attributes.mako | 2 +- templates/tracks/index.mako | 55 ++++++++++++++++--- tools/data_source/upload.xml | 1 + tools/new_operations/complement.xml | 4 +- 23 files changed, 245 insertions(+), 55 deletions(-) create mode 100644 lib/galaxy/datatypes/indexers/interval.awk create mode 100644 lib/galaxy/datatypes/indexers/interval_awk.xml diff --git a/datatypes_conf.xml.sample b/datatypes_conf.xml.sample index 645cb00c2ee..13543cab296 100644 --- a/datatypes_conf.xml.sample +++ b/datatypes_conf.xml.sample @@ -8,6 +8,9 @@ + + + @@ -31,7 +34,7 @@ - + diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index 9934eb40df3..ef407cfca13 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -40,7 +40,7 @@ class Data( object ): __metaclass__ = DataMeta """Add metadata elements""" - MetadataElement( name="dbkey", desc="Database/Build", default="?", param=metadata.SelectParameter, multiple=False, values=util.dbnames, no_value="?" ) + MetadataElement( name="dbkey", desc="Database/Build", default="?", param=metadata.DBKeyParameter, multiple=False, no_value="?" ) """Stores the set of display applications, and viewing methods, supported by this datatype """ supported_display_apps = {} @@ -242,7 +242,7 @@ class Data( object ): def after_edit( self, dataset ): """This function is called on the dataset after metadata is edited.""" dataset.clear_associated_files( metadata_safe = True ) - + @property def has_resolution(self): return False diff --git a/lib/galaxy/datatypes/indexers/coverage.py b/lib/galaxy/datatypes/indexers/coverage.py index 1cb94955b0a..2bfec91c25f 100644 --- a/lib/galaxy/datatypes/indexers/coverage.py +++ b/lib/galaxy/datatypes/indexers/coverage.py @@ -37,7 +37,7 @@ def write_chrom(max, out_base, instream): os.rename( fname+".npy", fname ) # Write average - for window in 10, 100, 1000, 10000: + for window in 10, 100, 1000, 10000, 100000: input = scores.copy() size = len( input ) input.resize( ( ( size / window ), window ) ) diff --git a/lib/galaxy/datatypes/indexers/interval.awk b/lib/galaxy/datatypes/indexers/interval.awk new file mode 100644 index 00000000000..fa84d6a0593 --- /dev/null +++ b/lib/galaxy/datatypes/indexers/interval.awk @@ -0,0 +1,43 @@ +BEGIN { + # from galaxy.utils + mapped_chars[">"] = "__gt__" + mapped_chars["<"] = "__lt__" + mapped_chars["'"] = "__sq__" + mapped_chars["\""] = "__dq__" + mapped_chars["\\["] = "__ob__" + mapped_chars["\\]"] = "__cb__" + mapped_chars["\\{"] = "__oc__" + mapped_chars["\\}"] = "__cc__" + mapped_chars["@"] = "__at__" + # additional, not in galaxy.utils + mapped_chars["/"] = "__fs__" + mapped_chars["^manifest\.tab$"] = "__manifest.tab__" +} +function escape_filename( name ) +{ + for( char in mapped_chars ) { + gsub( char, mapped_chars[char], name ) + } + return name +} +!_[$chrom]++ { + # close files only when we switch to a new one. + fn && close(fn) + fn = storepath "/" escape_filename($1) } +{ + print $0 >> fn; + # the || part is needed to catch 0 length chromosomes, which + # should never happen but... + if ($end > chroms[$chrom] || !chroms[$chrom]) + chroms[$chrom] = $end } +END { + fn = storepath "/manifest.tab" + for( x in chroms ) { + # add line to manifest + print x "\t" chroms[x] >> fn + chromfile = storepath "/" escape_filename(x) + # sort in-place + system( "sort -f -n -k " chrom " -k " start " -k " end " -o " chromfile " " chromfile ) + close(chromfile) + } +} \ No newline at end of file diff --git a/lib/galaxy/datatypes/indexers/interval.py b/lib/galaxy/datatypes/indexers/interval.py index ca044c931a1..43693cfe567 100644 --- a/lib/galaxy/datatypes/indexers/interval.py +++ b/lib/galaxy/datatypes/indexers/interval.py @@ -29,13 +29,11 @@ def divide( intervals, out_path ): manifest[chrom] = max(manifest.get(chrom,0),line.end) if not lastchrom == chrom: if current_file: - current_file.flush() current_file.close() current_file = open( os.path.join( out_path, "%s" % chrom), "a" ) print >> current_file, "\t".join(line) lastchrom = chrom if current_file: - current_file.flush() current_file.close() return manifest diff --git a/lib/galaxy/datatypes/indexers/interval_awk.xml b/lib/galaxy/datatypes/indexers/interval_awk.xml new file mode 100644 index 00000000000..965a9434b0f --- /dev/null +++ b/lib/galaxy/datatypes/indexers/interval_awk.xml @@ -0,0 +1,16 @@ + + + interval.awk + chrom=${input_dataset.metadata.chromCol} start=${input_dataset.metadata.startCol} + end=${input_dataset.metadata.endCol} strand=${input_dataset.metadata.strandCol} + storepath=${store_path} + $input_dataset 2>&1 + + + + + + + + + diff --git a/lib/galaxy/datatypes/indexers/wiggle.py b/lib/galaxy/datatypes/indexers/wiggle.py index ec4dd15c816..5ba1380aaa1 100644 --- a/lib/galaxy/datatypes/indexers/wiggle.py +++ b/lib/galaxy/datatypes/indexers/wiggle.py @@ -18,6 +18,8 @@ pkg_resources.require("numpy>=1.2.1") from numpy import * import tempfile import os +from galaxy.tracks.store import sanitize_name + def write_chrom(max, out_base, instream): @@ -35,7 +37,7 @@ def write_chrom(max, out_base, instream): os.rename( fname+".npy", fname ) # Write average - for window in 10, 100, 1000, 10000: + for window in 10, 100, 1000, 10000, 100000: input = scores.copy() size = len( input ) input.resize( ( ( size / window ), window ) ) @@ -60,7 +62,7 @@ def main(): LEN[chrom] = max2( LEN.get(chrom,0), pos+1 ) for chrom, stream in chroms.items(): stream.seek(0) - prefix = os.path.join(sys.argv[2], chrom) + prefix = os.path.join(sys.argv[2], sanitize_name(chrom)) write_chrom( LEN[chrom], prefix, stream ) manifest_file = open( os.path.join( sys.argv[2], "manifest.tab" ),"w" ) diff --git a/lib/galaxy/datatypes/interval.py b/lib/galaxy/datatypes/interval.py index f6b18414a25..be0c07ed258 100644 --- a/lib/galaxy/datatypes/interval.py +++ b/lib/galaxy/datatypes/interval.py @@ -792,7 +792,7 @@ class Wiggle( Tabular ): # Determine appropriate resolution to plot ~1000 points resolution = ( 10 ** math.ceil( math.log10( range / 1000 ) ) ) # Restrict to valid range - resolution = min( resolution, 10000 ) + resolution = min( resolution, 100000 ) resolution = max( resolution, 1 ) # Memory map the array (don't load all the data) data = numpy.load( data ) @@ -809,7 +809,7 @@ class Wiggle( Tabular ): # Determine appropriate resolution to plot ~1000 points resolution = math.ceil( 10 ** math.ceil( math.log10( range / 1000 ) ) ) # Restrict to valid range - resolution = min( resolution, 10000 ) + resolution = min( resolution, 100000 ) resolution = max( resolution, 1 ) return resolution diff --git a/lib/galaxy/datatypes/metadata.py b/lib/galaxy/datatypes/metadata.py index 233f59a9a38..07e2c7ad93d 100644 --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -289,7 +289,22 @@ class SelectParameter( MetadataParameter ): if value is None: return [] if not isinstance( value, list ): return [value] return value - + + +class DBKeyParameter( SelectParameter ): + def get_html_field( self, value=None, context={}, other_values={}, values=None, **kwd): + try: + values = kwd['trans'].db_builds + except AttributeError: pass + return super(DBKeyParameter, self).get_html_field( value, context, other_values, values, **kwd) + + def get_html( self, value=None, context={}, other_values={}, values=None, **kwd): + try: + values = kwd['trans'].db_builds + except AttributeError: pass + return super(DBKeyParameter, self).get_html( value, context, other_values, values, **kwd) + + class RangeParameter( SelectParameter ): def __init__( self, spec ): SelectParameter.__init__( self, spec ) diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 77c74d52832..1cf0adc0871 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -3,7 +3,7 @@ Provides mapping between extensions and datatypes, mime-types, etc. """ import os import logging -import data, tabular, interval, images, sequence, qualityscore, genetics, xml, coverage, tracks +import data, tabular, interval, images, sequence, qualityscore, genetics, xml, coverage, tracks, chrominfo import galaxy.util from galaxy.util.odict import odict diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index 6b8e42bd24b..4d62e1f4bc8 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -107,6 +107,7 @@ class DefaultToolAction( object ): out_data = {} # Collect any input datasets from the incoming parameters inp_data = self.collect_input_datasets( tool, incoming, trans ) + # Deal with input dataset names, 'dbkey' and types input_names = [] input_ext = 'data' @@ -119,6 +120,16 @@ class DefaultToolAction( object ): data = NoneDataset( datatypes_registry = trans.app.datatypes_registry ) if data.dbkey not in [None, '?']: input_dbkey = data.dbkey + + # Collect chromInfo dataset and add as parameters to incoming + db_datasets = {} + db_dataset = trans.db_dataset_for( input_dbkey ) + if db_dataset: + db_datasets[ "chromInfo" ] = db_dataset + incoming[ "chromInfo" ] = db_dataset.file_name + else: + incoming[ "chromInfo" ] = os.path.join( trans.app.config.tool_data_path, 'shared','ucsc','chrom', "%s.len" % input_dbkey ) + inp_data.update( db_datasets ) # Determine output dataset permission/roles list existing_datasets = [ inp for inp in inp_data.values() if inp ] diff --git a/lib/galaxy/tools/actions/upload.py b/lib/galaxy/tools/actions/upload.py index d286b815dae..82a51b7e071 100644 --- a/lib/galaxy/tools/actions/upload.py +++ b/lib/galaxy/tools/actions/upload.py @@ -21,7 +21,7 @@ class UploadToolAction( object ): def execute( self, tool, trans, incoming={}, set_output_hid = True ): data_file = incoming['file_data'] file_type = incoming['file_type'] - dbkey = incoming['dbkey'] + dbkey = incoming['other_dbkey'] or incoming['dbkey'] url_paste = incoming['url_paste'] is_multi_byte = False space_to_tab = False diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 14f33f51026..cf00b587e59 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -657,7 +657,7 @@ class GenomeBuildParameter( SelectToolParameter ): >>> # Create a mock transcation with 'hg17' as the current build >>> from galaxy.util.bunch import Bunch - >>> trans = Bunch( history=Bunch( genome_build='hg17' ) ) + >>> trans = Bunch( history=Bunch( genome_build='hg17' ), db_builds=util.dbnames ) >>> p = GenomeBuildParameter( None, XML( ... ''' @@ -692,10 +692,10 @@ class GenomeBuildParameter( SelectToolParameter ): """ def get_options( self, trans, other_values ): last_used_build = trans.history.genome_build - for dbkey, build_name in util.dbnames: + for dbkey, build_name in trans.db_builds: yield build_name, dbkey, ( dbkey == last_used_build ) def get_legal_values( self, trans, other_values ): - return set( dbkey for dbkey, _ in util.dbnames ) + return set( dbkey for dbkey, _ in trans.db_builds ) class ColumnListParameter( SelectToolParameter ): """ diff --git a/lib/galaxy/tracks/store.py b/lib/galaxy/tracks/store.py index 3009bd1c60f..07523a7ed56 100644 --- a/lib/galaxy/tracks/store.py +++ b/lib/galaxy/tracks/store.py @@ -1,5 +1,17 @@ import os +import re from string import Template +from galaxy.util import sanitize_text + +# extra mappings/escape to keep users from traversing around the +# filesystem and wreaking havoc +extra_mappings = { r"/": "__fs__", r"^manifest.tab$": "__manifest.tab__" } + +def sanitize_name( name ): + name = sanitize_text( name ) + for key, value in extra_mappings.items(): + name = re.sub( key, value, name ) + return name class TemplateSubber( object ): def __init__(self, obj): @@ -56,7 +68,7 @@ class TrackStore( object ): fd.close() def _get_object_path( self, chrom, resolution ): - object_name = chrom + object_name = sanitize_name(chrom) if resolution: object_name += "_%d" % resolution return os.path.join( self.path, object_name ) diff --git a/lib/galaxy/web/controllers/root.py b/lib/galaxy/web/controllers/root.py index 823bc29d6f0..972a50c6eb7 100644 --- a/lib/galaxy/web/controllers/root.py +++ b/lib/galaxy/web/controllers/root.py @@ -234,11 +234,15 @@ class RootController( BaseController ): if spec.get("readonly"): continue optional = params.get("is_"+name, None) + other = params.get("or_"+name, None) if optional and optional == 'true': # optional element... == 'true' actually means it is NOT checked (and therefore omitted) setattr(data.metadata, name, None) else: - setattr( data.metadata, name, spec.unwrap( params.get (name, None) ) ) + if other: + setattr( data.metadata, name, other ) + else: + setattr( data.metadata, name, spec.unwrap( params.get (name, None) ) ) data.datatype.after_edit( data ) trans.app.model.flush() diff --git a/lib/galaxy/web/controllers/tracks.py b/lib/galaxy/web/controllers/tracks.py index 479568055ce..0e840667a54 100644 --- a/lib/galaxy/web/controllers/tracks.py +++ b/lib/galaxy/web/controllers/tracks.py @@ -1,13 +1,11 @@ -from mako import exceptions -from mako.template import Template -from mako.lookup import TemplateLookup +import math + +import mimeparse +from galaxy.tracks import messages +from galaxy.util.json import to_json_string from galaxy.web.base.controller import * from galaxy.web.framework import simplejson -from galaxy import web -from galaxy.tracks import messages -import mimeparse -from galaxy.util.json import to_json_string -import math + class MultiResponse(object): """ @@ -82,18 +80,19 @@ class WebRoot( BaseController ): def build( self, trans, **kwargs ): trans.session["track_sets"] = list(kwargs.keys()) trans.session.save() - waiting = False - for id, value in kwargs.items(): - status = self.data_handler( trans, id ) - if status == messages.PENDING: - waiting = True - if not waiting: - return trans.response.send_redirect( web.url_for( controller='tracks', action='chroms', dbkey=trans.session["track_dbkey"]) ) - return trans.fill_template( 'tracks/build.mako' ) + #waiting = False + #for id, value in kwargs.items(): + # status = self.data_handler( trans, id ) + # if status == messages.PENDING: + # waiting = True + #if not waiting: + return trans.response.send_redirect( web.url_for( controller='tracks/', action='index', chrom="" ) ) + #return trans.fill_template( 'tracks/build.mako' ) @web.expose def index(self, trans, **kwargs): tracks = [] + dbkey = "" for track in trans.session["track_sets"]: dataset = trans.app.model.HistoryDatasetAssociation.get( track ) tracks.append({ @@ -101,17 +100,23 @@ class WebRoot( BaseController ): "name": dataset.name, "id": dataset.id }) + dbkey = dataset.dbkey chrom = kwargs.get("chrom","") LEN = self.chroms_handler(trans, trans.session["track_dbkey"]).get(chrom,0) return trans.fill_template( 'tracks/index.mako', - tracks=tracks, chrom=chrom, + tracks=tracks, chrom=chrom, dbkey=dbkey, LEN=LEN ) def chroms_handler(self, trans, dbkey ): - db_manifest = os.path.join( trans.app.config.tool_data_path, 'shared','ucsc','chrom', "%s.len" % dbkey ) + db_manifest = trans.db_dataset_for( dbkey ) + if not db_manifest: + db_manifest = os.path.join( trans.app.config.tool_data_path, 'shared','ucsc','chrom', "%s.len" % dbkey ) + else: + db_manifest = db_manifest.file_name manifest = {} if os.path.exists( db_manifest ): for line in open( db_manifest ): + if line.startswith("#"): continue line = line.rstrip("\r\n") fields = line.split("\t") manifest[fields[0]] = int(fields[1]) diff --git a/lib/galaxy/web/framework/__init__.py b/lib/galaxy/web/framework/__init__.py index 9d8457713e5..d99d15e1573 100644 --- a/lib/galaxy/web/framework/__init__.py +++ b/lib/galaxy/web/framework/__init__.py @@ -547,6 +547,31 @@ class UniverseWebTransaction( base.DefaultWebTransaction ): template = Template( source=template_string, searchList=[context or kwargs, dict(caller=self)] ) return str(template) + + @property + def db_builds( self ): + """ + Returns the builds defined by galaxy and the builds defined by + the user (chromInfo in history). + """ + dbnames = list() + datasets = self.app.model.HistoryDatasetAssociation.filter_by(deleted=False, history_id=self.history.id, extension="len").all() + if len(datasets) > 0: + dbnames.append( (util.dbnames.default_value, '--------- User Defined Builds ----------') ) + for dataset in datasets: + dbnames.append( (dataset.dbkey, dataset.name) ) + dbnames.extend( util.dbnames ) + return dbnames + + def db_dataset_for( self, dbkey ): + """ + Returns the db_file dataset associated/needed by `dataset`, or `None`. + """ + datasets = self.app.model.HistoryDatasetAssociation.filter_by(deleted=False, history_id=self.history.id, extension="len").all() + for ds in datasets: + if dbkey == ds.dbkey: + return ds + return None class FormBuilder( object ): """ diff --git a/static/scripts/trackster.js b/static/scripts/trackster.js index 83036c67e47..e349a19fa42 100644 --- a/static/scripts/trackster.js +++ b/static/scripts/trackster.js @@ -85,7 +85,7 @@ $.extend( TiledTrack.prototype, Track.prototype, { var resolution = Math.pow( 10, Math.ceil( Math.log( range / DENSITY ) / Math.log( 10 ) ) ); resolution = Math.max( resolution, 1 ); - resolution = Math.min( resolution, 10000 ); + resolution = Math.min( resolution, 100000 ); var parent_element = $("
"); this.content_div.children( ":first" ).remove(); @@ -152,10 +152,20 @@ $.extend( DataCache.prototype, { var low = position * DENSITY * resolution; var high = ( position + 1 ) * DENSITY * resolution; cache[resolution][position] = { state: "loading" }; - $.getJSON( "data" + this.type, { chr: this.view.chr, low: low, high: high, dataset_id: this.track.dataset_id }, function ( data ) { - cache[resolution][position] = { state: "loaded", values: data }; - $(document).trigger( "redraw" ); - }); + // use closure to preserve this and parameters for getJSON + var fetcher = function (ref) { + return function () { + $.getJSON( "data" + ref.type, { chr: ref.view.chr, low: low, high: high, dataset_id: ref.track.dataset_id }, function ( data ) { + if( data == "pending" ) { + setTimeout( fetcher, 5000 ); + } else { + cache[resolution][position] = { state: "loaded", values: data }; + } + $(document).trigger( "redraw" ); + }); + }; + }(this); + fetcher(); } return cache[resolution][position]; } @@ -288,8 +298,11 @@ $.extend( FeatureTrack.prototype, TiledTrack.prototype, { var chunk = this.cache.get( resolution, tile_index ); if ( chunk.state == "loading" ) { - return null; - } + parent_element.addClass("loading"); + return null; + } else { + parent_element.removeClass("loading"); + } var values = chunk.values; for ( var index in values ) { diff --git a/static/trackster.css b/static/trackster.css index d90ec9b0eb3..b160fea3b71 100644 --- a/static/trackster.css +++ b/static/trackster.css @@ -85,7 +85,10 @@ body { } .loading { - background: #DDDDDD; + background-image: url("/static/images/loading_large_white_bg.gif"); + background-position: center center; + background-repeat: no-repeat; + min-height: 100px; } .label-track .label { diff --git a/templates/dataset/edit_attributes.mako b/templates/dataset/edit_attributes.mako index f60f47b30de..930d201b68e 100644 --- a/templates/dataset/edit_attributes.mako +++ b/templates/dataset/edit_attributes.mako @@ -46,7 +46,7 @@ ${spec.desc}:
- ${data.metadata.get_html_by_name( name )} + ${data.metadata.get_html_by_name( name, trans=trans )}
diff --git a/templates/tracks/index.mako b/templates/tracks/index.mako index e7794140576..3df253c5d52 100644 --- a/templates/tracks/index.mako +++ b/templates/tracks/index.mako @@ -17,18 +17,19 @@ ${parent.stylesheets()} ${parent.late_javascripts()} - @@ -79,11 +114,14 @@ ${parent.late_javascripts()} diff --git a/tools/data_source/upload.xml b/tools/data_source/upload.xml index 4d12af2624e..a55c11fe1bb 100644 --- a/tools/data_source/upload.xml +++ b/tools/data_source/upload.xml @@ -21,6 +21,7 @@ + diff --git a/tools/new_operations/complement.xml b/tools/new_operations/complement.xml index be661ee6f4a..4b318fb9787 100644 --- a/tools/new_operations/complement.xml +++ b/tools/new_operations/complement.xml @@ -1,6 +1,6 @@ intervals of a query - gops_complement.py $input1 $output -1 ${input1.metadata.chromCol},${input1.metadata.startCol},${input1.metadata.endCol},${input1.metadata.strandCol} -l ${GALAXY_DATA_INDEX_DIR}/shared/ucsc/chrom/${dbkey}.len $allchroms + gops_complement.py $input1 $output -1 ${input1.metadata.chromCol},${input1.metadata.startCol},${input1.metadata.endCol},${input1.metadata.strandCol} -l ${chromInfo} $allchroms @@ -58,4 +58,4 @@ See Galaxy Interval Operation Screencasts_ (right click to open this link in ano .. image:: ../static/operation_icons/gops_complement.gif - \ No newline at end of file +