diff --git a/datatypes_conf.xml.sample b/datatypes_conf.xml.sample index 76bba775482..eead3e0e436 100644 --- a/datatypes_conf.xml.sample +++ b/datatypes_conf.xml.sample @@ -26,6 +26,7 @@ + @@ -38,7 +39,6 @@ - @@ -173,24 +173,20 @@ - - - - - - - - - - - - - - - + + + + + + + + + + + + + diff --git a/lib/galaxy/datatypes/converters/interval_to_coverage.py b/lib/galaxy/datatypes/converters/interval_to_coverage.py new file mode 100644 index 00000000000..9b4bfccfa10 --- /dev/null +++ b/lib/galaxy/datatypes/converters/interval_to_coverage.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +""" +Converter to generate 3 (or 4) column base-pair coverage from an interval file. + +usage: %prog bed_file out_file + -1, --cols1=N,N,N,N: Columns for chrom, start, end, strand in interval file + -2, --cols2=N,N,N,N: Columns for chrom, start, end, strand in coverage file +""" +import sys +from galaxy import eggs +import pkg_resources; pkg_resources.require( "bx-python" ) +from bx.intervals import io +from bx.cookbook import doc_optparse + +INTERVAL_METADATA = ('chromCol', + 'startCol', + 'endCol', + 'strandCol',) + +COVERAGE_METADATA = ('chromCol', + 'positionCol', + 'forwardCol', + 'reverseCol',) + +def main( interval, coverage ): + chroms = dict() + for record in interval: + if not type( record ) is io.GenomicInterval: continue + chrom = chroms[record.chrom] = chroms.get(record.chrom, dict()) + for position in xrange(record.start, record.end): + coverages = chrom[position] = chrom.get(position,[0,0]) + if record.strand == "-": coverages[1] += 1 + else: coverages[0] += 1 + for chrom in sorted(chroms.iterkeys()): + positions = chroms[chrom] + for position in sorted(positions.iterkeys()): + coverage.write( chrom=chrom, position=position, forward=positions[position][0], reverse=positions[position][1] ) + +class CoverageWriter( object ): + def __init__( self, out_stream=None, chromCol=0, positionCol=1, forwardCol=2, reverseCol=3 ): + self.chromCol, self.positionCol, self.forwardCol, self.reverseCol = chromCol, positionCol, forwardCol, reverseCol + self.nfields = max( chromCol, positionCol, forwardCol, reverseCol )+1 + self.out_stream = out_stream + self.nlines = 0 + + def write(self, chrom="chr", position=0, forward=0, reverse=0 ): + self.nlines += 1 + if self.nlines % 64000: self.out_stream.flush() + outlist = [None] * self.nfields + outlist[self.chromCol] = str(chrom) + outlist[self.positionCol] = str(position) + if self.reverseCol == -1: outlist[self.forwardCol] = str(forward + reverse) + else: + outlist[self.forwardCol] = str(forward) + outlist[self.reverseCol] = str(reverse) + self.out_stream.write("%s\n" % "\t".join( outlist )) + + def flush(self): + self.out_stream.flush() + +if __name__ == "__main__": + options, args = doc_optparse.parse( __doc__ ) + try: + chr_col_1, start_col_1, end_col_1, strand_col_1 = [int(x)-1 for x in options.cols1.split(',')] + chr_col_2, position_col_2, forward_col_2, reverse_col_2 = [int(x)-1 for x in options.cols2.split(',')] + in_fname, out_fname = args + except: + doc_optparse.exception() + + coverage = CoverageWriter( out_stream = open(out_fname, "a"), + chromCol = chr_col_2, positionCol = position_col_2, + forwardCol = forward_col_2, reverseCol = reverse_col_2, ) + interval = io.NiceReaderWrapper( open(in_fname, "r"), + chrom_col=chr_col_1, + start_col=start_col_1, + end_col=end_col_1, + strand_col=strand_col_1, + fix_strand=True ) + main( interval, coverage ) + coverage.flush() \ No newline at end of file diff --git a/lib/galaxy/datatypes/converters/interval_to_coverage.xml b/lib/galaxy/datatypes/converters/interval_to_coverage.xml new file mode 100644 index 00000000000..3ad64f7b9ba --- /dev/null +++ b/lib/galaxy/datatypes/converters/interval_to_coverage.xml @@ -0,0 +1,18 @@ + + + + interval_to_coverage.py $input1 $output1 + -1 ${input1.metadata.chromCol},${input1.metadata.startCol},${input1.metadata.endCol},${input1.metadata.strandCol} + -2 ${output1.metadata.chromCol},${output1.metadata.positionCol},${output1.metadata.forwardCol},${output1.metadata.reverseCol} + + + + + + + + + + + + diff --git a/lib/galaxy/web/base/controller.py b/lib/galaxy/web/base/controller.py index 534124e2131..f2e95dfbcbd 100644 --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -28,4 +28,7 @@ class BaseController( object ): Root = BaseController """ Deprecated: `BaseController` used to be available under the name `Root` -""" \ No newline at end of file +""" + +class ControllerUnavailable( Exception ): + pass \ No newline at end of file diff --git a/lib/galaxy/web/buildapp.py b/lib/galaxy/web/buildapp.py index 797a4aca263..65a5d315ea8 100644 --- a/lib/galaxy/web/buildapp.py +++ b/lib/galaxy/web/buildapp.py @@ -28,13 +28,18 @@ def add_controllers( webapp, app ): them to the webapp. """ from galaxy.web.base.controller import BaseController + from galaxy.web.base.controller import ControllerUnavailable import galaxy.web.controllers controller_dir = galaxy.web.controllers.__path__[0] for fname in os.listdir( controller_dir ): if not( fname.startswith( "_" ) ) and fname.endswith( ".py" ): name = fname[:-3] module_name = "galaxy.web.controllers." + name - module = __import__( module_name ) + try: + module = __import__( module_name ) + except ControllerUnavailable, exc: + log.debug("%s could not be loaded: %s" % (module_name, str(exc))) + continue for comp in module_name.split( "." )[1:]: module = getattr( module, comp ) # Look for a controller inside the modules diff --git a/lib/galaxy/web/controllers/genetrack.py b/lib/galaxy/web/controllers/genetrack.py index ae81a277cd4..f4b13e2b09c 100644 --- a/lib/galaxy/web/controllers/genetrack.py +++ b/lib/galaxy/web/controllers/genetrack.py @@ -1,22 +1,127 @@ import time, glob, os +from itertools import cycle -import pkg_resources -pkg_resources.require("GeneTrack") - -import atlas -from atlas import sql -from atlas import util as atlas_utils -from atlas.web import formlib from mako import exceptions from mako.template import Template from mako.lookup import TemplateLookup from galaxy.web.base.controller import * +try: + import pkg_resources + pkg_resources.require("GeneTrack") + import atlas + from atlas import sql + from atlas import hdf + from atlas import util as atlas_utils + from atlas.web import formlib, feature_query, feature_filter + from atlas.web import label_cache as atlas_label_cache + from atlas.plotting.const import * + from atlas.plotting.tracks import prefab + from atlas.plotting.tracks import chart + from atlas.plotting import tracks +except Exception, exc: + raise ControllerUnavailable("GeneTrack could not import a required dependency: %s" % str(exc)) + pkg_resources.require( "Paste" ) import paste.httpexceptions +# Database helpers +SHOW_LABEL_LIMIT = 10000 +color = cycle( [LIGHT, WHITE] ) + +def list_labels(session): + """ + Returns a list of labels that will be plotted in order. + """ + labels = sql.Label + query = session.query(labels).order_by("-id") + return query + +def open_databases( conf ): + """ + A helper function that returns handles to the hdf and sql databases + """ + db = hdf.hdf_open( conf.HDF_DATABASE, mode='r' ) + session = sql.get_session( conf.SQL_URI ) + return db, session + +def hdf_query(db, name, param, autosize=False ): + """ + Schema specific hdf query. + Note that returns data as columns not rows. + """ + if not hdf.has_node(db=db, name=name): + atlas.warn( 'missing label %s' % name ) + return [], [], [], [] + data = hdf.GroupData( db=db, name=name) + istart, iend = data.get_indices(label=param.chrom, start=param.start, stop=param.end) + table = data.get_table(label=param.chrom) + if autosize: + # attempts to reduce the number of points + size = len( table.cols.ix[istart:iend] ) + step = max( [1, size/1200] ) + else: + step = 1 + + ix = table.cols.ix[istart:iend:step].tolist() + wx = table.cols.wx[istart:iend:step].tolist() + cx = table.cols.cx[istart:iend:step].tolist() + ax = table.cols.ax[istart:iend:step].tolist() + return ix, wx, cx, ax + +# Chart helpers +def build_tracks( param, conf, data_label, fit_label, pred_label, strand, show=False ): + """ + Builds tracks + """ + # gets all the labels for a fast lookup + label_cache = atlas_label_cache( conf ) + + # get database handles for hdf and sql + db, session = open_databases( conf ) + + # fetching x and y coordinates for bar and fit (line) for + # each strand plus (p), minus (m), all (a) + bix, bpy, bmy, bay = hdf_query( db=db, name=data_label, param=param ) + fix, fpy, fmy, fay = hdf_query( db=db, name=fit_label, param=param ) + + # close the hdf database + db.close() + + # get all features within the range + all = feature_query( session=session, param=param ) + + # draws the barchart and the nucleosome chart below it + if strand == 'composite': + bar = prefab.composite_bartrack( fix=fix, fay=fay, bix=bix, bay=bay, param=param) + else: + bar = prefab.twostrand_bartrack( fix=fix, fmy=fmy, fpy=fpy, bix=bix, bmy=bmy, bpy=bpy, param=param) + + charts = list() + charts.append( bar ) + + return charts + +def feature_chart(param=None, session=None, label=None, label_dict={}): + # draw the ORF tracks + all = feature_filter(feature_query(session=session, param=param), name=label, kdict=label_dict) + if len(all) == 0: return [] + opts = track_options( + xscale=param.xscale, w=param.width, fgColor=PURPLE, + show_labels=param.show_labels, ylabel=str(label), + bgColor=color.next() + ) + return [ + tracks.split_tracks(features=all, options=opts, split=param.show_labels, track_type='vector') + ] + +def consolidate_charts( charts, param ): + # create the multiplot + opt = chart_options( w=param.width ) + multi = chart.MultiChart(options=opt, charts=charts) + return multi + # SETUP Track Builders -from mod454.trackbuilder import build_tracks import functools def twostrand_tracks( param=None, conf=None ): return build_tracks( data_label=conf.LABEL, fit_label=conf.FIT_LABEL, pred_label=conf.PRED_LABEL, param=param, conf=conf, strand='twostrand') @@ -96,6 +201,7 @@ class WebRoot(BaseController): """ Main request handler """ + color = cycle( [LIGHT, WHITE] ) data = trans.app.model.HistoryDatasetAssociation.get( dataset_id ) if not data: raise paste.httpexceptions.HTTPRequestRangeNotSatisfiable( "Invalid reference dataset id: %s." % str( dataset_id ) ) @@ -108,10 +214,15 @@ class WebRoot(BaseController): FIT_LABEL = "%s-SIGMA-%d" % (data.metadata.label, 20), PRED_LABEL = "PRED-%s-SIGMA-%d" % (data.metadata.label, 20), ) - from atlas import hdf - db = hdf.hdf_open( conf.HDF_DATABASE, mode='r' ) - conf.CHROM_FIELDS = [(x,x) for x in hdf.GroupData(db=db, name=conf.LABEL).labels] - db.close() + session = sql.get_session( conf.SQL_URI ) + + if os.path.exists( conf.HDF_DATABASE ): + db = hdf.hdf_open( conf.HDF_DATABASE, mode='r' ) + conf.CHROM_FIELDS = [(x,x) for x in hdf.GroupData(db=db, name=conf.LABEL).labels] + db.close() + else: + query = session.execute(sql.select([sql.feature_table.c.chrom]).distinct()) + conf.CHROM_FIELDS = [(x.chrom,x.chrom) for x in query] # generate a new form based on the configuration form = formlib.main_form( conf ) @@ -147,14 +258,25 @@ class WebRoot(BaseController): # get the template and the function used to generate the tracks tmpl_name, track_maker = conf.PLOT_MAPPER[param.plot] - if track_maker is not None: - # generate the name that the image will be stored at - fname, fpath = atlas_utils.make_tempfile( dir=conf.IMAGE_DIR, suffix='.png') - param.fname = fname + charts = [] - # generate the track - track_chart = track_maker( param=param, conf=conf ) - track_chart.save(fname=fpath) + fname, fpath = atlas_utils.make_tempfile( dir=conf.IMAGE_DIR, suffix='.png') + param.fname = fname + + # set the scale of the plot + param.xscale = [ param.start, param.end ] + + # when visualizing on wide scales labels are not useful + param.show_labels = ( param.end - param.start ) <= SHOW_LABEL_LIMIT + + if track_maker is not None and os.path.exists( conf.HDF_DATABASE ): + # generate the fit track + charts = track_maker( param=param, conf=conf ) + + for label in list_labels( session ): + charts.extend( feature_chart(param=param, session=session, label=label.name, label_dict={label.name:label.id}) ) + track_chart = consolidate_charts( charts, param ) + track_chart.save(fname=fpath) return trans.fill_template_mako(tmpl_name, conf=conf, form=form, param=param, dataset_id=dataset_id) diff --git a/scripts/paster.py b/scripts/paster.py index 446cb9de90b..5b6c9262493 100755 --- a/scripts/paster.py +++ b/scripts/paster.py @@ -12,7 +12,6 @@ assert sys.version_info[:2] >= ( 2, 4 ) new_path = [ os.path.join( os.getcwd(), "lib" ) ] new_path.extend( sys.path[1:] ) # remove scripts/ from the path sys.path = new_path -print sys.path from galaxy import eggs import pkg_resources diff --git a/templates/genetrack/base.html b/templates/genetrack/base.html new file mode 100644 index 00000000000..cd381d54b93 --- /dev/null +++ b/templates/genetrack/base.html @@ -0,0 +1,29 @@ + + + + +${self.title()} + + + + +<%def name="title()"> + Title + + +<%def name="footer()"> + + + + + + ${self.body()} + ${self.footer()} + + + diff --git a/templates/genetrack/index.html b/templates/genetrack/index.html new file mode 100644 index 00000000000..23dee5cea67 --- /dev/null +++ b/templates/genetrack/index.html @@ -0,0 +1,75 @@ +## index.html +<%inherit file="base.html"/> +<%def name="title()"> + Index + + +

${conf.TITLE}

+ +
+ + + + + % if form.errors(): + + % endif + + + + + + + + + + + + + + + + + +
+ % for ekey, evalue in form.errors().items(): + ERROR:    ${ekey}:    ${evalue}
+ % endfor +
+ More + + Chromosome: ${form.chrom.tag()}   + Feature: ${form.feature.tag()}   + Width: ${form.zoom.tag()}   + Plot: ${form.plot.tag()}   + + + +
+ +     + +     + +     + +
+ +
+ +
+ + +
diff --git a/templates/genetrack/search.html b/templates/genetrack/search.html new file mode 100644 index 00000000000..d707a6da479 --- /dev/null +++ b/templates/genetrack/search.html @@ -0,0 +1,55 @@ +## search.html +<%! +from itertools import cycle +colors = cycle( [ 'even', 'odd' ] ) +%> + +<%inherit file="base.html"/> +<%def name="title()"> + Search + + +

Search

+ +
+
+ Search terms + + +
+
+ +% if param.word: + + % if len(query)>0: +

Showing the best ${len(query)} matches

+ + + + + % for color, row in zip(colors, query): + ${makerow(color, row)} + % endfor +
Name + Chromosome + Start:End + Type +
+ + % else: +

No results found

+ % endif + +%endif + +
+<%def name="makerow(color, row)"> + + ${row.name} + ${row.chrom} + ${row.start}:${row.end} + ${row.label.name} + + + + diff --git a/tools/visualization/genetrack.py b/tools/visualization/genetrack.py index bd27d5df4d2..14c290f05d2 100644 --- a/tools/visualization/genetrack.py +++ b/tools/visualization/genetrack.py @@ -14,15 +14,19 @@ import pkg_resources pkg_resources.require("GeneTrack") pkg_resources.require("bx-python") -from atlas import commands -from bx.cookbook import doc_optparse -import os import commands as oscommands +from atlas import commands +from atlas import sql +from bx.cookbook import doc_optparse +from bx.intervals import io + +import os import tempfile +from functools import partial SIGMA = 20 WIDTH = 5 * SIGMA -EXCLUSION_ZONE = 147 +EXCLUSION_ZONE = 147 def main(label, fit, feats, data_dir, output): os.mkdir(data_dir) @@ -31,14 +35,14 @@ def main(label, fit, feats, data_dir, output): CLOBBER = True, DATA_SIZE = 3*10**6, MINIMUM_PEAK_SIZE = 0.1, - LOADER_ENABLED = True, - FITTER_ENABLED = True, - PREDICTOR_ENABLED = True, - EXPORTER_ENABLED = True, + LOADER_ENABLED = False, + FITTER_ENABLED = False, + PREDICTOR_ENABLED = False, + EXPORTER_ENABLED = False, LOADER = loader, FITTER = fitter, PREDICTOR = predictor, - EXPORTER = exporter, + EXPORTER = partial( commands.exporter, formatter=commands.bed_formatter), HDF_DATABASE = os.path.join( data_dir, "data.hdf" ), SQL_URI = "sqlite:///%s" % os.path.join( data_dir, "features.sqlite" ), SIGMA = SIGMA, @@ -51,12 +55,23 @@ def main(label, fit, feats, data_dir, output): RIGHT_SHIFT = EXCLUSION_ZONE / 2, EXPORT_LABELS = [ "PRED-%s-SIGMA-%d" % ( label,SIGMA ) ], EXPORT_DIR = os.path.join( data_dir ), - DATA_FILE=fit[1], + DATA_FILE=fit and fit[1] or None, fit=fit, feats=feats, ) + if fit: + # Turn on fit processing. + conf.LOADER_ENABLED = True, + conf.FITTER_ENABLED = True, + conf.PREDICTOR_ENABLED = True, + conf.EXPORTER_ENABLED = True, + for feat in feats: + load_feature_files(conf, feats) commands.execute(conf) - + outname = "%s.%s.txt" % (conf.__name__, conf.EXPORT_LABELS[0] ) + if os.path.exists( os.path.join(data_dir, outname) ): + os.rename( os.path.join(data_dir, outname), output) + # mod454 seems to be a module without a package. The necessary funcitons are # stubbed out here until I'm sure of their final home. INS @@ -97,8 +112,40 @@ def predictor( conf ): from mod454.predictor import predictor as mod454_predictor return mod454_predictor( conf ) -def exporter( conf ): - return commands.bed_exporter(conf) +def load_feature_files( conf, feats): + """ + Loads features from file names + """ + engine = sql.get_engine( conf.SQL_URI ) + sql.drop_indices(engine) + conn = engine.connect() + for label, fname, col_spec in feats: + label_id = sql.make_label(engine, name=label, clobber=False) + reader = io.NiceReaderWrapper( open(fname,"r"), + chrom_col=col_spec.chromCol, + start_col=col_spec.startCol, + end_col=col_spec.endCol, + strand_col=col_spec.strandCol, + fix_strand=False ) + values = list() + for interval in reader: + print interval + if not type( interval ) is io.GenomicInterval: continue + row = {'label_id':label_id, + 'name':col_spec.nameCol == -1 and "%s-%s" % (str(interval.start), str(interval.end)) or interval.fields[col_spec.nameCol], + 'altname':"", + 'chrom':interval.chrom, + 'start':interval.start, + 'end':interval.end, + 'strand':interval.strand, + 'value':0, + 'freetext':""} + values.append(row) + insert = sql.feature_table.insert() + conn.execute( insert, values) + conn.close() + sql.create_indices(engine) + class Bunch( object ): def __init__(self, **kwargs): @@ -115,9 +162,12 @@ if __name__ == "__main__": options, args = doc_optparse.parse( __doc__ ) try: label = options.label - fit_name, fit_meta = options.fits.split(':')[0], [int(x)-1 for x in options.fits.split(':')[1:]] - fit_meta = Bunch(chromCol=fit_meta[0], positionCol=fit_meta[1], forwardCol=fit_meta[2], reverseCol=fit_meta[3]) - fit = ( label, fit_name, fit_meta, ) + if options.fits: + fit_name, fit_meta = options.fits.split(':')[0], [int(x)-1 for x in options.fits.split(':')[1:]] + fit_meta = Bunch(chromCol=fit_meta[0], positionCol=fit_meta[1], forwardCol=fit_meta[2], reverseCol=fit_meta[3]) + fit = ( label, fit_name, fit_meta, ) + else: + fit = [] # split apart the string into nested lists, preserves order if options.feats: feats = [ ( @@ -127,7 +177,7 @@ if __name__ == "__main__": strandCol=int(strandCol)-1, nameCol=int(nameCol)-1), ) for feat_label, fname, chromCol, startCol, endCol, strandCol, nameCol - in ( feat.split(':') for feat in options.feats.split(',') )] + in ( feat.split(':') for feat in options.feats.split(',') if len(feat) > 0 )] else: feats = [] data_dir = options.data diff --git a/tools/visualization/genetrack.xml b/tools/visualization/genetrack.xml index 65c5f519b31..8ab46560391 100644 --- a/tools/visualization/genetrack.xml +++ b/tools/visualization/genetrack.xml @@ -8,7 +8,10 @@ genetrack.py -l $data_label - -1 ${fit_data}:${fit_data.metadata.chromCol}:${fit_data.metadata.positionCol}:${fit_data.metadata.forwardCol}:${fit_data.metadata.reverseCol} + #if not str($fit_data) == "None" + -1 + ${fit_data}:${fit_data.metadata.chromCol}:${fit_data.metadata.positionCol}:${fit_data.metadata.forwardCol}:${fit_data.metadata.reverseCol} + #end if #if $feature_data -2 #end if @@ -23,7 +26,7 @@ [a-zA-Z0-9]{0,25} - + @@ -36,7 +39,13 @@ - + + + tables + atlas + pychartdir + numpy + This tool takes the input Fit Data and creates a peak and curve plot showing the reads and fitness on each basepair. Features can be plotted below as tracks.