opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

data providers: add data to factory decorator to allow datasets api to parse query strings into provider settings; datasets API: allow new providers to be used in raw_data; clean up and pack scripts

Browse Source
This commit is contained in:
Carl Eberhard
2013-06-24 14:06:16 -04:00
parent 0b6ffe0ac4
commit 9071ea8463
20 changed files with 328 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/datatypes/binary.py
+10 -10
View File
@@ -267,25 +267,25 @@ class Bam( Binary ):
# bam does not use '#' to indicate comments/headers - we need to strip out those headers from the std. providers
#TODO:?? seems like there should be an easier way to do/inherit this - metadata.comment_char?
#TODO: incorporate samtools options to control output: regions first, then flags, etc.
@dataproviders.decorators.dataprovider_factory( 'line' )
@dataproviders.decorators.dataprovider_factory( 'line', dataproviders.line.FilteredLineDataProvider.settings )
def line_dataprovider( self, dataset, **settings ):
samtools_source = dataproviders.dataset.SamtoolsDataProvider( dataset )
settings[ 'comment_char' ] = '@'
return dataproviders.line.FilteredLineDataProvider( samtools_source, **settings )
@dataproviders.decorators.dataprovider_factory( 'regex-line' )
@dataproviders.decorators.dataprovider_factory( 'regex-line', dataproviders.line.RegexLineDataProvider.settings )
def regex_line_dataprovider( self, dataset, **settings ):
samtools_source = dataproviders.dataset.SamtoolsDataProvider( dataset )
settings[ 'comment_char' ] = '@'
return dataproviders.line.RegexLineDataProvider( samtools_source, **settings )
@dataproviders.decorators.dataprovider_factory( 'column' )
@dataproviders.decorators.dataprovider_factory( 'column', dataproviders.column.ColumnarDataProvider.settings )
def column_dataprovider( self, dataset, **settings ):
samtools_source = dataproviders.dataset.SamtoolsDataProvider( dataset )
settings[ 'comment_char' ] = '@'
return dataproviders.column.ColumnarDataProvider( samtools_source, **settings )
@dataproviders.decorators.dataprovider_factory( 'map' )
@dataproviders.decorators.dataprovider_factory( 'map', dataproviders.column.MapDataProvider.settings )
def map_dataprovider( self, dataset, **settings ):
samtools_source = dataproviders.dataset.SamtoolsDataProvider( dataset )
settings[ 'comment_char' ] = '@'
@@ -293,30 +293,30 @@ class Bam( Binary ):
# these can't be used directly - may need BamColumn, BamMap (Bam metadata -> column/map)
# OR - see genomic_region_dataprovider
#@dataproviders.decorators.dataprovider_factory( 'dataset-column' )
#@dataproviders.decorators.dataprovider_factory( 'dataset-column', dataproviders.column.ColumnarDataProvider.settings )
#def dataset_column_dataprovider( self, dataset, **settings ):
# settings[ 'comment_char' ] = '@'
# return super( Sam, self ).dataset_column_dataprovider( dataset, **settings )
#@dataproviders.decorators.dataprovider_factory( 'dataset-map' )
#@dataproviders.decorators.dataprovider_factory( 'dataset-map', dataproviders.column.MapDataProvider.settings )
#def dataset_map_dataprovider( self, dataset, **settings ):
# settings[ 'comment_char' ] = '@'
# return super( Sam, self ).dataset_map_dataprovider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'header' )
@dataproviders.decorators.dataprovider_factory( 'header', dataproviders.line.RegexLineDataProvider.settings )
def header_dataprovider( self, dataset, **settings ):
# in this case we can use an option of samtools view to provide just what we need (w/o regex)
samtools_source = dataproviders.dataset.SamtoolsDataProvider( dataset, '-H' )
return dataproviders.line.RegexLineDataProvider( samtools_source, **settings )
@dataproviders.decorators.dataprovider_factory( 'id-seq-qual' )
@dataproviders.decorators.dataprovider_factory( 'id-seq-qual', dataproviders.column.MapDataProvider.settings )
def id_seq_qual_dataprovider( self, dataset, **settings ):
settings[ 'indeces' ] = [ 0, 9, 10 ]
settings[ 'column_types' ] = [ 'str', 'str', 'str' ]
settings[ 'column_names' ] = [ 'id', 'seq', 'qual' ]
return self.map_dataprovider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'genomic-region' )
@dataproviders.decorators.dataprovider_factory( 'genomic-region', dataproviders.column.ColumnarDataProvider.settings )
def genomic_region_dataprovider( self, dataset, **settings ):
# GenomicRegionDataProvider currently requires a dataset as source - may not be necc.
#TODO:?? consider (at least) the possible use of a kwarg: metadata_source (def. to source.dataset),
@@ -330,7 +330,7 @@ class Bam( Binary ):
settings[ 'column_types' ] = [ 'str', 'int', 'int' ]
return self.column_dataprovider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'genomic-region-map' )
@dataproviders.decorators.dataprovider_factory( 'genomic-region-map', dataproviders.column.MapDataProvider.settings )
def genomic_region_map_dataprovider( self, dataset, **settings ):
settings[ 'indeces' ] = [ 2, 3, 3 ]
settings[ 'column_types' ] = [ 'str', 'int', 'int' ]
lib/galaxy/datatypes/data.py
+4 -5
View File
@@ -593,7 +593,6 @@ class Data( object ):
Base dataprovider factory for all datatypes that returns the proper provider
for the given `data_format` or raises a `NoProviderAvailable`.
"""
#TODO:?? is this handling super class providers?
if self.has_dataprovider( data_format ):
return self.dataproviders[ data_format ]( self, dataset, **settings )
raise dataproviders.exceptions.NoProviderAvailable( self, data_format )
@@ -603,12 +602,12 @@ class Data( object ):
dataset_source = dataproviders.dataset.DatasetDataProvider( dataset )
return dataproviders.base.DataProvider( dataset_source, **settings )
@dataproviders.decorators.dataprovider_factory( 'chunk' )
@dataproviders.decorators.dataprovider_factory( 'chunk', dataproviders.chunk.ChunkDataProvider.settings )
def chunk_dataprovider( self, dataset, **settings ):
dataset_source = dataproviders.dataset.DatasetDataProvider( dataset )
return dataproviders.chunk.ChunkDataProvider( dataset_source, **settings )
@dataproviders.decorators.dataprovider_factory( 'chunk64' )
@dataproviders.decorators.dataprovider_factory( 'chunk64', dataproviders.chunk.Base64ChunkDataProvider.settings )
def chunk64_dataprovider( self, dataset, **settings ):
dataset_source = dataproviders.dataset.DatasetDataProvider( dataset )
return dataproviders.chunk.Base64ChunkDataProvider( dataset_source, **settings )
@@ -785,7 +784,7 @@ class Text( Data ):
split = classmethod(split)
# ------------- Dataproviders
@dataproviders.decorators.dataprovider_factory( 'line' )
@dataproviders.decorators.dataprovider_factory( 'line', dataproviders.line.FilteredLineDataProvider.settings )
def line_dataprovider( self, dataset, **settings ):
"""
Returns an iterator over the dataset's lines (that have been `strip`ed)
@@ -794,7 +793,7 @@ class Text( Data ):
dataset_source = dataproviders.dataset.DatasetDataProvider( dataset )
return dataproviders.line.FilteredLineDataProvider( dataset_source, **settings )
@dataproviders.decorators.dataprovider_factory( 'regex-line' )
@dataproviders.decorators.dataprovider_factory( 'regex-line', dataproviders.line.RegexLineDataProvider.settings )
def regex_line_dataprovider( self, dataset, **settings ):
"""
Returns an iterator over the dataset's lines
lib/galaxy/datatypes/dataproviders/base.py
+47 -2
View File
@@ -22,14 +22,44 @@ add datum entry/exit point methods: possibly decode, encode
icorporate existing visualization/dataproviders
some of the sources (esp. in datasets) don't need to be re-created
YAGNI: InterleavingMultiSourceDataProvider, CombiningMultiSourceDataProvider
datasets API entry point:
kwargs should be parsed from strings 2 layers up (in the DatasetsAPI) - that's the 'proper' place for that.
but how would it know how/what to parse if it doesn't have access to the classes used in the provider?
Building a giant list by sweeping all possible dprov classes doesn't make sense
For now - I'm burying them in the class __init__s - but I don't like that
"""
import logging
log = logging.getLogger( __name__ )
# ----------------------------------------------------------------------------- base classes
class HasSettings( type ):
"""
Metaclass for data providers that allows defining and inheriting
a dictionary named 'settings'.
Useful for allowing class level access to expected variable types
passed to class `__init__` functions so they can be parsed from a query string.
"""
# yeah - this is all too acrobatic
def __new__( cls, name, base_classes, attributes ):
settings = {}
# get settings defined in base classes
for base_class in base_classes:
base_settings = getattr( base_class, 'settings', None )
if base_settings:
settings.update( base_settings )
# get settings defined in this class
new_settings = attributes.pop( 'settings', None )
if new_settings:
settings.update( new_settings )
attributes[ 'settings' ] = settings
return type.__new__( cls, name, base_classes, attributes )
# ----------------------------------------------------------------------------- base classes
class DataProvider( object ):
"""
@@ -39,6 +69,12 @@ class DataProvider( object ):
(c) do not allow write methods
(but otherwise implement the other file object interface methods)
"""
# a definition of expected types for keyword arguments sent to __init__
# useful for controlling how query string dictionaries can be parsed into correct types for __init__
# empty in this base class
__metaclass__ = HasSettings
settings = {}
def __init__( self, source, **kwargs ):
"""
:param source: the source that this iterator will loop over.
@@ -130,13 +166,16 @@ class FilteredDataProvider( DataProvider ):
- `num_valid_data_read`: how many data have been returned from `filter`.
- `num_data_returned`: how many data has this provider yielded.
"""
# not useful here - we don't want functions over the query string
#settings.update({ 'filter_fn': 'function' })
def __init__( self, source, filter_fn=None, **kwargs ):
"""
:param filter_fn: a lambda or function that will be passed a datum and
return either the (optionally modified) datum or None.
"""
super( FilteredDataProvider, self ).__init__( source, **kwargs )
self.filter_fn = filter_fn
self.filter_fn = filter_fn if hasattr( filter_fn, '__call__' ) else None
# count how many data we got from the source
self.num_data_read = 0
# how many valid data have we gotten from the source
@@ -179,6 +218,12 @@ class LimitedOffsetDataProvider( FilteredDataProvider ):
Useful for grabbing sections from a source (e.g. pagination).
"""
# define the expected types of these __init__ arguments so they can be parsed out from query strings
settings = {
'limit' : 'int',
'offset': 'int'
}
#TODO: may want to squash this into DataProvider
def __init__( self, source, offset=0, limit=None, **kwargs ):
"""
lib/galaxy/datatypes/dataproviders/chunk.py
+6 -2
View File
@@ -26,6 +26,10 @@ class ChunkDataProvider( base.DataProvider ):
"""
MAX_CHUNK_SIZE = 2**16
DEFAULT_CHUNK_SIZE = MAX_CHUNK_SIZE
settings = {
'chunk_index' : 'int',
'chunk_size' : 'int'
}
#TODO: subclass from LimitedOffsetDataProvider?
# see web/framework/base.iterate_file, util/__init__.file_reader, and datatypes.tabular
@@ -38,8 +42,8 @@ class ChunkDataProvider( base.DataProvider ):
(gen. in bytes).
"""
super( ChunkDataProvider, self ).__init__( source, **kwargs )
self.chunk_size = chunk_size
self.chunk_pos = chunk_index * self.chunk_size
self.chunk_size = int( chunk_size )
self.chunk_pos = int( chunk_index ) * self.chunk_size
def validate_source( self, source ):
"""
lib/galaxy/datatypes/dataproviders/column.py
+21 -9
View File
@@ -29,6 +29,14 @@ class ColumnarDataProvider( line.RegexLineDataProvider ):
the same number of columns as the number of indeces asked for (even if they
are filled with None).
"""
settings = {
'indeces' : 'list:int',
'column_count' : 'int',
'column_types' : 'list:str',
'parse_columns' : 'bool',
'deliminator' : 'str'
}
def __init__( self, source, indeces=None,
column_count=None, column_types=None, parsers=None, parse_columns=True,
deliminator='\t', **kwargs ):
@@ -91,11 +99,11 @@ class ColumnarDataProvider( line.RegexLineDataProvider ):
# how/whether to parse each column value
self.parsers = {}
if parse_columns:
self.parsers = self._get_default_parsers()
self.parsers = self.get_default_parsers()
# overwrite with user desired parsers
self.parsers.update( parsers or {} )
def _get_default_parsers( self ):
def get_default_parsers( self ):
"""
Return parser dictionary keyed for each columnar type
(as defined in datatypes).
@@ -132,7 +140,7 @@ class ColumnarDataProvider( line.RegexLineDataProvider ):
#'gffstrand': # -, +, ?, or '.' for None, etc.
}
def _parse_value( self, val, type ):
def parse_value( self, val, type ):
"""
Attempt to parse and return the given value based on the given type.
@@ -153,7 +161,7 @@ class ColumnarDataProvider( line.RegexLineDataProvider ):
return None
return val
def _get_column_type( self, index ):
def get_column_type( self, index ):
"""
Get the column type for the parser from `self.column_types` or `None`
if the type is unavailable.
@@ -165,18 +173,18 @@ class ColumnarDataProvider( line.RegexLineDataProvider ):
except IndexError, ind_err:
return None
def _parse_column_at_index( self, columns, parser_index, index ):
def parse_column_at_index( self, columns, parser_index, index ):
"""
Get the column type for the parser from `self.column_types` or `None`
if the type is unavailable.
"""
try:
return self._parse_value( columns[ index ], self._get_column_type( parser_index ) )
return self.parse_value( columns[ index ], self.get_column_type( parser_index ) )
# if a selected index is not within columns, return None
except IndexError, index_err:
return None
def _parse_columns_from_line( self, line ):
def parse_columns_from_line( self, line ):
"""
Returns a list of the desired, parsed columns.
:param line: the line to parse
@@ -188,13 +196,13 @@ class ColumnarDataProvider( line.RegexLineDataProvider ):
selected_indeces = self.selected_column_indeces or list( xrange( len( all_columns ) ) )
parsed_columns = []
for parser_index, column_index in enumerate( selected_indeces ):
parsed_columns.append( self._parse_column_at_index( all_columns, parser_index, column_index ) )
parsed_columns.append( self.parse_column_at_index( all_columns, parser_index, column_index ) )
return parsed_columns
def __iter__( self ):
parent_gen = super( ColumnarDataProvider, self ).__iter__()
for line in parent_gen:
columns = self._parse_columns_from_line( line )
columns = self.parse_columns_from_line( line )
yield columns
#TODO: implement column filters here and not below - flatten hierarchy
@@ -223,6 +231,10 @@ class MapDataProvider( ColumnarDataProvider ):
.. note: that the subclass constructors are passed kwargs - so they're
params (limit, offset, etc.) are also applicable here.
"""
settings = {
'column_names' : 'list:str',
}
def __init__( self, source, column_names=None, **kwargs ):
"""
:param column_names: an ordered list of strings that will be used as the keys
lib/galaxy/datatypes/dataproviders/dataset.py
+50 -5
View File
@@ -141,7 +141,7 @@ class DatasetDataProvider( base.DataProvider ):
"""
# metadata columns are 1-based indeces
column = getattr( self.dataset.metadata, name )
return ( column - 1 ) if isinstance( column, int ) else None
return ( column - 1 ) if ( isinstance( column, int ) and column > 0 ) else None
def get_genomic_region_indeces( self, check=False ):
"""
@@ -271,6 +271,12 @@ class GenomicRegionDataProvider( column.ColumnarDataProvider ):
"""
# dictionary keys when named_columns=True
COLUMN_NAMES = [ 'chrom', 'start', 'end' ]
settings = {
'chrom_column' : 'int',
'start_column' : 'int',
'end_column' : 'int',
'named_columns' : 'bool',
}
def __init__( self, dataset, chrom_column=None, start_column=None, end_column=None, named_columns=False, **kwargs ):
"""
@@ -333,6 +339,14 @@ class IntervalDataProvider( column.ColumnarDataProvider ):
'chrom', 'start', 'end' (and 'strand' and 'name' if available).
"""
COLUMN_NAMES = [ 'chrom', 'start', 'end', 'strand', 'name' ]
settings = {
'chrom_column' : 'int',
'start_column' : 'int',
'end_column' : 'int',
'strand_column' : 'int',
'name_column' : 'int',
'named_columns' : 'bool',
}
def __init__( self, dataset, chrom_column=None, start_column=None, end_column=None,
strand_column=None, name_column=None, named_columns=False, **kwargs ):
@@ -349,25 +363,40 @@ class IntervalDataProvider( column.ColumnarDataProvider ):
dataset_source = DatasetDataProvider( dataset )
# get genomic indeces and add strand and name
self.column_names = []
indeces = []
#TODO: this is sort of involved and oogly
if chrom_column == None:
chrom_column = dataset_source.get_metadata_column_index_by_name( 'chromCol' )
if chrom_column != None:
self.column_names.append( 'chrom' )
indeces.append( chrom_column )
if start_column == None:
start_column = dataset_source.get_metadata_column_index_by_name( 'startCol' )
if start_column != None:
self.column_names.append( 'start' )
indeces.append( start_column )
if end_column == None:
end_column = dataset_source.get_metadata_column_index_by_name( 'endCol' )
if end_column != None:
self.column_names.append( 'end' )
indeces.append( end_column )
if strand_column == None:
strand_column = dataset_source.get_metadata_column_index_by_name( 'strandCol' )
if strand_column != None:
self.column_names.append( 'strand' )
indeces.append( strand_column )
if name_column == None:
name_column = dataset_source.get_metadata_column_index_by_name( 'nameCol' )
indeces = [ chrom_column, start_column, end_column, strand_column, name_column ]
kwargs.update({ 'indeces' : indeces })
if name_column != None:
self.column_names.append( 'name' )
indeces.append( name_column )
kwargs.update({ 'indeces' : indeces })
if not kwargs.get( 'column_types', None ):
kwargs.update({ 'column_types' : dataset_source.get_metadata_column_types( indeces=indeces ) })
self.named_columns = named_columns
if self.named_columns:
self.column_names = self.COLUMN_NAMES
super( IntervalDataProvider, self ).__init__( dataset_source, **kwargs )
@@ -390,6 +419,10 @@ class FastaDataProvider( base.FilteredDataProvider ):
sequence: <joined lines of nucleotide/amino data>
}
"""
settings = {
'ids' : 'list:str',
}
def __init__( self, source, ids=None, **kwargs ):
"""
:param ids: optionally return only ids (and sequences) that are in this list.
@@ -419,6 +452,10 @@ class TwoBitFastaDataProvider( DatasetDataProvider ):
sequence: <joined lines of nucleotide/amino data>
}
"""
settings = {
'ids' : 'list:str',
}
def __init__( self, source, ids=None, **kwargs ):
"""
:param ids: optionally return only ids (and sequences) that are in this list.
@@ -445,6 +482,10 @@ class WiggleDataProvider( base.LimitedOffsetDataProvider ):
Class that returns chrom, pos, data from a wiggle source.
"""
COLUMN_NAMES = [ 'chrom', 'pos', 'value' ]
settings = {
'named_columns' : 'bool',
'column_names' : 'list:str',
}
def __init__( self, source, named_columns=False, column_names=None, **kwargs ):
"""
@@ -483,6 +524,10 @@ class BigWigDataProvider( base.LimitedOffsetDataProvider ):
Class that returns chrom, pos, data from a wiggle source.
"""
COLUMN_NAMES = [ 'chrom', 'pos', 'value' ]
settings = {
'named_columns' : 'bool',
'column_names' : 'list:str',
}
def __init__( self, source, chrom, start, end, named_columns=False, column_names=None, **kwargs ):
"""
lib/galaxy/datatypes/dataproviders/decorators.py
+61 -3
View File
@@ -87,17 +87,40 @@ def has_dataproviders( cls ):
# log.debug( '\t\t ', fn.__doc__ )
return cls
def dataprovider_factory( name ):
def dataprovider_factory( name, settings=None ):
"""
Wraps a class method and marks it as a dataprovider factory.
Wraps a class method and marks it as a dataprovider factory and creates a
function to parse query strings to __init__ arguments as the
`parse_query_string_settings` attribute of the factory function.
An example use of the `parse_query_string_settings`:
..example::
kwargs = dataset.datatype.dataproviders[ provider ].parse_query_string_settings( query_kwargs )
return list( dataset.datatype.dataprovider( dataset, provider, **kwargs ) )
:param name: what name/key to register the factory under in `cls.dataproviders`
:param type: any hashable var
:type name: any hashable var
:param settings: dictionary containing key/type pairs for parsing query strings
to __init__ arguments
:type settings: dictionary
"""
#TODO:?? use *args for settings allowing mulitple dictionaries
# make a function available through the name->provider dispatch to parse query strings
# callable like:
# settings_dict = dataproviders[ provider_name ].parse_query_string_settings( query_kwargs )
#TODO: ugh - overly complicated but the best I could think of
def parse_query_string_settings( query_kwargs ):
return _parse_query_string_settings( query_kwargs, settings )
#log.debug( 'dataprovider:', name )
def named_dataprovider_factory( func ):
#log.debug( 'named_dataprovider_factory:', name, '->', func.__name__ )
setattr( func, _DATAPROVIDER_METHOD_NAME_KEY, name )
setattr( func, 'parse_query_string_settings', parse_query_string_settings )
setattr( func, 'settings', settings )
#TODO: I want a way to inherit settings from the previous provider( this_name ) instead of defining over and over
#log.debug( '\t setting:', getattr( func, _DATAPROVIDER_METHOD_NAME_KEY ) )
@wraps( func )
def wrapped_dataprovider_factory( self, *args, **kwargs ):
@@ -105,3 +128,38 @@ def dataprovider_factory( name ):
return func( self, *args, **kwargs )
return wrapped_dataprovider_factory
return named_dataprovider_factory
def _parse_query_string_settings( query_kwargs, settings=None ):
"""
Parse the values in `query_kwargs` from strings to the proper types
listed in the same key in `settings`.
"""
def list_from_query_string( s ):
# assume csv
return s.split( ',' )
parsers = {
'int' : int,
'float' : float,
'bool' : bool,
'list:str' : lambda s: list_from_query_string( s ),
'list:int' : lambda s: [ int( i ) for i in list_from_query_string( s ) ],
}
settings = settings or {}
# yay! yet another set of query string parsers! <-- sarcasm
# work through the keys in settings finding matching keys in query_kwargs
# if found in both, get the expected/needed type from settings and store the new parsed value
# if we can't parse it (no parser, bad value), delete the key from query_kwargs so the provider will use the defaults
for key in settings:
if key in query_kwargs:
#TODO: this would be the place to sanitize any strings
query_value = query_kwargs[ key ]
needed_type = settings[ key ]
try:
query_kwargs[ key ] = parsers[ needed_type ]( query_value )
except ( KeyError, ValueError ):
del query_kwargs[ key ]
#TODO:?? do we want to remove query_kwarg entries NOT in settings?
return query_kwargs
lib/galaxy/datatypes/dataproviders/line.py
+11
View File
@@ -27,6 +27,12 @@ class FilteredLineDataProvider( base.LimitedOffsetDataProvider ):
to return.
"""
DEFAULT_COMMENT_CHAR = '#'
settings = {
'string_lines' : 'bool',
'provide_blank' : 'bool',
'comment_char' : 'str',
}
def __init__( self, source, strip_lines=True, provide_blank=False, comment_char=DEFAULT_COMMENT_CHAR, **kwargs ):
"""
:param strip_lines: remove whitespace from the beginning an ending
@@ -78,6 +84,11 @@ class RegexLineDataProvider( FilteredLineDataProvider ):
.. note:: the regex matches are effectively OR'd (if **any** regex matches
the line it is considered valid and will be provided).
"""
settings = {
'regex_list' : 'list:str',
'invert' : 'bool',
}
def __init__( self, source, regex_list=None, invert=False, **kwargs ):
"""
:param regex_list: list of strings or regular expression strings that will
lib/galaxy/datatypes/interval.py
+18 -10
View File
@@ -334,20 +334,24 @@ class Interval( Tabular ):
return None
# ------------- Dataproviders
@dataproviders.decorators.dataprovider_factory( 'genomic-region' )
@dataproviders.decorators.dataprovider_factory( 'genomic-region',
dataproviders.dataset.GenomicRegionDataProvider.settings )
def genomic_region_dataprovider( self, dataset, **settings ):
return dataproviders.dataset.GenomicRegionDataProvider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'genomic-region-map' )
@dataproviders.decorators.dataprovider_factory( 'genomic-region-map',
dataproviders.dataset.GenomicRegionDataProvider.settings )
def genomic_region_map_dataprovider( self, dataset, **settings ):
settings[ 'named_columns' ] = True
return self.genomic_region_dataprovider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'interval' )
@dataproviders.decorators.dataprovider_factory( 'interval',
dataproviders.dataset.IntervalDataProvider.settings )
def interval_dataprovider( self, dataset, **settings ):
return dataproviders.dataset.IntervalDataProvider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'interval-map' )
@dataproviders.decorators.dataprovider_factory( 'interval-map',
dataproviders.dataset.IntervalDataProvider.settings )
def interval_map_dataprovider( self, dataset, **settings ):
settings[ 'named_columns' ] = True
return self.interval_dataprovider( dataset, **settings )
@@ -809,20 +813,24 @@ class Gff( Tabular, _RemoteCallMixin ):
# ------------- Dataproviders
# redefine bc super is Tabular
@dataproviders.decorators.dataprovider_factory( 'genomic-region' )
@dataproviders.decorators.dataprovider_factory( 'genomic-region',
dataproviders.dataset.GenomicRegionDataProvider.settings )
def genomic_region_dataprovider( self, dataset, **settings ):
return dataproviders.dataset.GenomicRegionDataProvider( dataset, 0, 3, 4, **settings )
@dataproviders.decorators.dataprovider_factory( 'genomic-region-map' )
@dataproviders.decorators.dataprovider_factory( 'genomic-region-map',
dataproviders.dataset.GenomicRegionDataProvider.settings )
def genomic_region_map_dataprovider( self, dataset, **settings ):
settings[ 'named_columns' ] = True
return self.genomic_region_dataprovider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'interval' )
@dataproviders.decorators.dataprovider_factory( 'interval',
dataproviders.dataset.IntervalDataProvider.settings )
def interval_dataprovider( self, dataset, **settings ):
return dataproviders.dataset.IntervalDataProvider( dataset, 0, 3, 4, 6, 2, **settings )
@dataproviders.decorators.dataprovider_factory( 'interval-map' )
@dataproviders.decorators.dataprovider_factory( 'interval-map',
dataproviders.dataset.IntervalDataProvider.settings )
def interval_map_dataprovider( self, dataset, **settings ):
settings[ 'named_columns' ] = True
return self.interval_dataprovider( dataset, **settings )
@@ -1193,12 +1201,12 @@ class Wiggle( Tabular, _RemoteCallMixin ):
return resolution
# ------------- Dataproviders
@dataproviders.decorators.dataprovider_factory( 'wiggle' )
@dataproviders.decorators.dataprovider_factory( 'wiggle', dataproviders.dataset.WiggleDataProvider.settings )
def wiggle_dataprovider( self, dataset, **settings ):
dataset_source = dataproviders.dataset.DatasetDataProvider( dataset )
return dataproviders.dataset.WiggleDataProvider( dataset_source, **settings )
@dataproviders.decorators.dataprovider_factory( 'wiggle-map' )
@dataproviders.decorators.dataprovider_factory( 'wiggle-map', dataproviders.dataset.WiggleDataProvider.settings )
def wiggle_map_dataprovider( self, dataset, **settings ):
dataset_source = dataproviders.dataset.DatasetDataProvider( dataset )
settings[ 'named_columns' ] = True
lib/galaxy/datatypes/sequence.py
-11
View File
@@ -15,8 +15,6 @@ import galaxy.model
from galaxy import util
from sniff import *
from galaxy.datatypes import dataproviders
import pkg_resources
pkg_resources.require("simplejson")
import simplejson
@@ -399,15 +397,6 @@ class Fasta( Sequence ):
f.close()
_count_split = classmethod(_count_split)
def provider( self, dataset, data_format, **settings ):
from galaxy.dataproviders import dataset as dataset_providers
if data_format == 'id_seq':
source = dataset_providers.DatasetDataProvider( dataset )
return dataset_providers.FastaDataProvider( source, **settings )
return super( Fasta, self ).provider( dataset, data_format, **settings )
class csFasta( Sequence ):
""" Class representing the SOLID Color-Space sequence ( csfasta ) """
lib/galaxy/datatypes/tabular.py
+26 -20
View File
@@ -345,26 +345,25 @@ class Tabular( data.Text ):
return vizs
# ------------- Dataproviders
@dataproviders.decorators.dataprovider_factory( 'column' )
@dataproviders.decorators.dataprovider_factory( 'column', dataproviders.column.ColumnarDataProvider.settings )
def column_dataprovider( self, dataset, **settings ):
"""Uses column settings that are passed in"""
print 'Tabular.comment_char:', settings.get( 'comment_char', None )
dataset_source = dataproviders.dataset.DatasetDataProvider( dataset )
return dataproviders.column.ColumnarDataProvider( dataset_source, **settings )
@dataproviders.decorators.dataprovider_factory( 'dataset-column' )
@dataproviders.decorators.dataprovider_factory( 'dataset-column',
dataproviders.column.ColumnarDataProvider.settings )
def dataset_column_dataprovider( self, dataset, **settings ):
"""Attempts to get column settings from dataset.metadata"""
return dataproviders.dataset.DatasetColumnarDataProvider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'map' )
@dataproviders.decorators.dataprovider_factory( 'map', dataproviders.column.MapDataProvider.settings )
def map_dataprovider( self, dataset, **settings ):
"""Uses column settings that are passed in"""
dataset_source = dataproviders.dataset.DatasetDataProvider( dataset )
return dataproviders.column.MapDataProvider( dataset_source, **settings )
@dataproviders.decorators.dataprovider_factory( 'dataset-map' )
@dataproviders.decorators.dataprovider_factory( 'dataset-map', dataproviders.column.MapDataProvider.settings )
def dataset_map_dataprovider( self, dataset, **settings ):
"""Attempts to get column settings from dataset.metadata"""
return dataproviders.dataset.DatasetMapDataProvider( dataset, **settings )
@@ -502,55 +501,58 @@ class Sam( Tabular ):
# ------------- Dataproviders
# sam does not use '#' to indicate comments/headers - we need to strip out those headers from the std. providers
#TODO:?? seems like there should be an easier way to do this - metadata.comment_char?
@dataproviders.decorators.dataprovider_factory( 'line' )
@dataproviders.decorators.dataprovider_factory( 'line', dataproviders.line.FilteredLineDataProvider.settings )
def line_dataprovider( self, dataset, **settings ):
settings[ 'comment_char' ] = '@'
return super( Sam, self ).line_dataprovider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'regex-line' )
@dataproviders.decorators.dataprovider_factory( 'regex-line', dataproviders.line.RegexLineDataProvider.settings )
def regex_line_dataprovider( self, dataset, **settings ):
settings[ 'comment_char' ] = '@'
return super( Sam, self ).regex_line_dataprovider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'column' )
@dataproviders.decorators.dataprovider_factory( 'column', dataproviders.column.ColumnarDataProvider.settings )
def column_dataprovider( self, dataset, **settings ):
settings[ 'comment_char' ] = '@'
return super( Sam, self ).column_dataprovider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'dataset-column' )
@dataproviders.decorators.dataprovider_factory( 'dataset-column',
dataproviders.column.ColumnarDataProvider.settings )
def dataset_column_dataprovider( self, dataset, **settings ):
settings[ 'comment_char' ] = '@'
return super( Sam, self ).dataset_column_dataprovider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'map' )
@dataproviders.decorators.dataprovider_factory( 'map', dataproviders.column.MapDataProvider.settings )
def map_dataprovider( self, dataset, **settings ):
settings[ 'comment_char' ] = '@'
return super( Sam, self ).map_dataprovider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'dataset-map' )
@dataproviders.decorators.dataprovider_factory( 'dataset-map', dataproviders.column.MapDataProvider.settings )
def dataset_map_dataprovider( self, dataset, **settings ):
settings[ 'comment_char' ] = '@'
return super( Sam, self ).dataset_map_dataprovider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'header' )
@dataproviders.decorators.dataprovider_factory( 'header', dataproviders.line.RegexLineDataProvider.settings )
def header_dataprovider( self, dataset, **settings ):
dataset_source = dataproviders.dataset.DatasetDataProvider( dataset )
headers_source = dataproviders.line.RegexLineDataProvider( dataset_source, regex_list=[ '^@' ] )
return dataproviders.line.RegexLineDataProvider( headers_source, **settings )
@dataproviders.decorators.dataprovider_factory( 'id-seq-qual' )
@dataproviders.decorators.dataprovider_factory( 'id-seq-qual', map_dataprovider.settings )
def id_seq_qual_dataprovider( self, dataset, **settings ):
# provided as an example of a specified column map (w/o metadata)
settings[ 'indeces' ] = [ 0, 9, 10 ]
settings[ 'column_names' ] = [ 'id', 'seq', 'qual' ]
return self.map_dataprovider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'genomic-region' )
@dataproviders.decorators.dataprovider_factory( 'genomic-region',
dataproviders.dataset.GenomicRegionDataProvider.settings )
def genomic_region_dataprovider( self, dataset, **settings ):
settings[ 'comment_char' ] = '@'
return dataproviders.dataset.GenomicRegionDataProvider( dataset, 2, 3, 3, **settings )
@dataproviders.decorators.dataprovider_factory( 'genomic-region-map' )
@dataproviders.decorators.dataprovider_factory( 'genomic-region-map',
dataproviders.dataset.GenomicRegionDataProvider.settings )
def genomic_region_map_dataprovider( self, dataset, **settings ):
settings[ 'comment_char' ] = '@'
return dataproviders.dataset.GenomicRegionDataProvider( dataset, 2, 3, 3, True, **settings )
@@ -621,11 +623,13 @@ class Pileup( Tabular ):
return False
# ------------- Dataproviders
@dataproviders.decorators.dataprovider_factory( 'genomic-region' )
@dataproviders.decorators.dataprovider_factory( 'genomic-region',
dataproviders.dataset.GenomicRegionDataProvider.settings )
def genomic_region_dataprovider( self, dataset, **settings ):
return dataproviders.dataset.GenomicRegionDataProvider( dataset, **settings )
@dataproviders.decorators.dataprovider_factory( 'genomic-region-map' )
@dataproviders.decorators.dataprovider_factory( 'genomic-region-map',
dataproviders.dataset.GenomicRegionDataProvider.settings )
def genomic_region_map_dataprovider( self, dataset, **settings ):
settings[ 'named_columns' ] = True
return self.genomic_region_dataprovider( dataset, **settings )
@@ -668,11 +672,13 @@ class Vcf( Tabular ):
dataset.metadata.sample_names = line.split()[ 9: ]
# ------------- Dataproviders
@dataproviders.decorators.dataprovider_factory( 'genomic-region' )
@dataproviders.decorators.dataprovider_factory( 'genomic-region',
dataproviders.dataset.GenomicRegionDataProvider.settings )
def genomic_region_dataprovider( self, dataset, **settings ):
return dataproviders.dataset.GenomicRegionDataProvider( dataset, 0, 1, 1, **settings )
@dataproviders.decorators.dataprovider_factory( 'genomic-region-map' )
@dataproviders.decorators.dataprovider_factory( 'genomic-region-map',
dataproviders.dataset.GenomicRegionDataProvider.settings )
def genomic_region_map_dataprovider( self, dataset, **settings ):
settings[ 'named_columns' ] = True
return self.genomic_region_dataprovider( dataset, **settings )
lib/galaxy/visualization/data_providers/registry.py
+1 -1
View File
@@ -32,7 +32,7 @@ class DataProviderRegistry( object ):
"bigwig": genome.BigWigDataProvider,
"bigbed": genome.BigBedDataProvider,
"column": ColumnDataProvider
"column_with_stats": ColumnDataProvider
}
def get_data_provider( self, trans, name=None, source='data', raw=False, original_dataset=None ):
lib/galaxy/visualization/registry.py
+26 -14
View File
@@ -15,22 +15,27 @@ import logging
log = logging.getLogger( __name__ )
__TODO__ = """
BUGS:
anon users clicking a viz link gets 'must be' msg in galaxy_main (w/ masthead)
should not show visualizations (no icon)?
newick files aren't being sniffed prop? - datatype is txt
BUGS:
anon users clicking a viz link gets 'must be' msg in galaxy_main (w/ masthead)
should not show visualizations (no icon)?
newick files aren't being sniffed prop? - datatype is txt
have parsers create objects instead of dicts
allow data_sources with no model_class but have tests (isAdmin, etc.)
maybe that's an instance of User model_class?
some confused vocabulary in docs, var names
tests:
anding, grouping, not
data_sources:
lists of
add description element to visualization.
have parsers create objects instead of dicts
allow data_sources with no model_class but have tests (isAdmin, etc.)
maybe that's an instance of User model_class?
some confused vocabulary in docs, var names
tests:
anding, grouping, not
data_sources:
lists of
add description element to visualization.
TESTS to add:
has dataprovider
user is admin
"""
# ------------------------------------------------------------------- the registry
class VisualizationsRegistry( object ):
"""
Main responsibilities are:
@@ -93,6 +98,12 @@ class VisualizationsRegistry( object ):
"""
self.listings = VisualizationsConfigParser.parse( self.configuration_filepath )
#TODO: def get_visualization( self, trans, visualization_name, target_object ):
# """
# Is the visualization with the given `visualization_name` applicable
# to the `target_object`?
# """
# -- building links to visualizations from objects --
def get_visualizations( self, trans, target_object ):
"""
@@ -151,10 +162,11 @@ class VisualizationsRegistry( object ):
# convert datatypes to their actual classes (for use with isinstance)
test_result = trans.app.datatypes_registry.get_datatype_class_by_name( test_result )
if not test_result:
# warn if can't find class, but continue
# warn if can't find class, but continue (with other tests)
log.warn( 'visualizations_registry cannot find class (%s) for applicability test', test_result )
continue
#NOTE: tests are OR'd, if any test passes - the visualization can be applied
if test_fn( target_object, test_result ):
#log.debug( 'test passed' )
return True
lib/galaxy/webapps/galaxy/api/datasets.py
+18 -3
View File
@@ -6,6 +6,7 @@ from galaxy.visualization.data_providers.genome import FeatureLocationIndexDataP
from galaxy.web.base.controller import BaseAPIController, UsesVisualizationMixin, UsesHistoryDatasetAssociationMixin
from galaxy.web.base.controller import UsesHistoryMixin
from galaxy.web.framework.helpers import is_true
from galaxy.datatypes import dataproviders
import logging
log = logging.getLogger( __name__ )
@@ -217,10 +218,24 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin, UsesHistory
return msg
registry = trans.app.data_provider_registry
# allow the caller to specifiy which provider is used
if provider and provider in registry.dataset_type_name_to_data_provider:
data_provider = registry.dataset_type_name_to_data_provider[ provider ]( dataset )
# or have it look up by datatype
# pulling from the original providers if possible, then the new providers
if provider:
if provider in registry.dataset_type_name_to_data_provider:
data_provider = registry.dataset_type_name_to_data_provider[ provider ]( dataset )
elif dataset.datatype.has_dataprovider( provider ):
kwargs = dataset.datatype.dataproviders[ provider ].parse_query_string_settings( kwargs )
# use dictionary to allow more than the data itself to be returned (data totals, other meta, etc.)
return {
'data': list( dataset.datatype.dataprovider( dataset, provider, **kwargs ) )
}
else:
raise dataproviders.exceptions.NoProviderAvailable( dataset.datatype, provider )
# no provider name: look up by datatype
else:
data_provider = registry.get_data_provider( trans, raw=True, original_dataset=dataset )
static/scripts/mvc/visualizations/scatterplotControlForm.js
+1 -1
View File
@@ -562,7 +562,7 @@ var ScatterplotControlForm = BaseView.extend( LoggableMixin ).extend({
var params = {
data_type : 'raw_data',
provider : 'column',
provider : 'column_with_stats',
columns : '[' + columns + ']'
};
this.log( '\t data settings (url params):', params );
static/scripts/packed/mvc/data.js
+1 -1
View File
File diff suppressed because one or more lines are too long
static/scripts/packed/mvc/dataset/hda-base.js
+1 -1
View File
File diff suppressed because one or more lines are too long
static/scripts/packed/mvc/dataset/hda-edit.js
+1 -1
View File
File diff suppressed because one or more lines are too long
static/scripts/packed/mvc/visualizations/scatterplotControlForm.js
+1 -1
View File
File diff suppressed because one or more lines are too long
visualizations_conf.xml.sample
+24 -24
View File
@@ -84,30 +84,30 @@
(e.g. hda_ldda can be 'hda' or 'ldda' and modifies/informs dataset_id to fetch an HDA or LDDA)
-->
<!ELEMENT param (#PCDATA)>
<!-- param tells the registry how to parse the query string param back into a resource/data_source.
For example, if a query string has "dataset_id=NNN" and the type is 'dataset', the registry
will attempt to fetch the hda with id of NNN from the database and pass it to the template.
(text): the query string param key this source will be parsed from (e.g. dataset_id)
REQUIRED
type: the type of the resource.
Can be: str (DEFAULT), bool, int, float, json, visualization, dbkey, dataset, or hda_ldda.
default: if a param is not passed on the query string (and is not required) OR the given param
fails to parse, this value is used instead.
DEFAULT: None
required: set this to true if the param is required for the template. Rendering will with an error
if the param hasn't been sent.
DEFAULT: false
csv: set this to true if the param is a comma separated list. The registry will attempt to
parse each value as the given type and send the result as a list to the template.
DEFAULT: false
constrain_to: (currently unused) constain a param to a set of values, error if not valid.
DEFAULT: don't constrain
var_name_in_template: a new name for the resource/variable to use in the template. E.g. an initial
query string param key might be 'dataset_id' in the URL, the registry parses it into an HDA,
and if var_name_in_template is set to 'hda', the template will be able to access the HDA
with the variable name 'hda' (as in hda.title).
DEFAULT: keep the original query string name
-->
<!-- param tells the registry how to parse the query string param back into a resource/data_source.
For example, if a query string has "dataset_id=NNN" and the type is 'dataset', the registry
will attempt to fetch the hda with id of NNN from the database and pass it to the template.
(text): the query string param key this source will be parsed from (e.g. dataset_id)
REQUIRED
type: the type of the resource.
Can be: str (DEFAULT), bool, int, float, json, visualization, dbkey, dataset, or hda_ldda.
default: if a param is not passed on the query string (and is not required) OR the given param
fails to parse, this value is used instead.
DEFAULT: None
required: set this to true if the param is required for the template. Rendering will with an error
if the param hasn't been sent.
DEFAULT: false
csv: set this to true if the param is a comma separated list. The registry will attempt to
parse each value as the given type and send the result as a list to the template.
DEFAULT: false
constrain_to: (currently unused) constain a param to a set of values, error if not valid.
DEFAULT: don't constrain
var_name_in_template: a new name for the resource/variable to use in the template. E.g. an initial
query string param key might be 'dataset_id' in the URL, the registry parses it into an HDA,
and if var_name_in_template is set to 'hda', the template will be able to access the HDA
with the variable name 'hda' (as in hda.title).
DEFAULT: keep the original query string name
-->
<!ATTLIST param
type CDATA #IMPLIED
default CDATA #IMPLIED