From dd40fd7c05c1e7f6e7afdccb9514860c14d683d5 Mon Sep 17 00:00:00 2001 From: Jeremy Goecks Date: Mon, 17 Sep 2012 12:29:48 -0400 Subject: [PATCH] Data providers framework enhancements: (a) add registry object; (b) move all provider lookup code into registry; and (c) integrate ColumnDataProvider into raw_data requests. --- lib/galaxy/app.py | 3 + .../visualization/data_providers/__init__.py | 3 + .../basic.py} | 65 ++++++++++++++++- .../genome.py} | 57 +-------------- .../visualization/data_providers/registry.py | 73 +++++++++++++++++++ .../phyloviz/phyloviz_dataprovider.py | 2 +- lib/galaxy/web/api/datasets.py | 45 ++++++------ lib/galaxy/web/api/tools.py | 7 +- lib/galaxy/web/base/controller.py | 5 +- lib/galaxy/web/controllers/visualization.py | 5 +- 10 files changed, 178 insertions(+), 87 deletions(-) create mode 100644 lib/galaxy/visualization/data_providers/__init__.py rename lib/galaxy/visualization/{data_providers.py => data_providers/basic.py} (54%) rename lib/galaxy/visualization/{genome/data_providers.py => data_providers/genome.py} (96%) create mode 100644 lib/galaxy/visualization/data_providers/registry.py diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index de364211654..ccf6c76235a 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -13,6 +13,7 @@ from galaxy.objectstore import build_object_store_from_config import galaxy.quota from galaxy.tags.tag_handler import GalaxyTagHandler from galaxy.visualization.genomes import Genomes +from galaxy.visualization.data_providers.registry import DataProviderRegistry from galaxy.tools.imp_exp import load_history_imp_exp_tools from galaxy.tools.genome_index import load_genome_index_tools from galaxy.sample_tracking import external_service_types @@ -73,6 +74,8 @@ class UniverseApplication( object ): self.tag_handler = GalaxyTagHandler() # Genomes self.genomes = Genomes( self ) + # Data providers registry. + self.data_provider_registry = DataProviderRegistry() # Tool data tables self.tool_data_tables = galaxy.tools.data.ToolDataTableManager( self.config.tool_data_path, self.config.tool_data_table_config_path ) # Initialize the tools, making sure the list of tool configs includes the reserved migrated_tools_conf.xml file. diff --git a/lib/galaxy/visualization/data_providers/__init__.py b/lib/galaxy/visualization/data_providers/__init__.py new file mode 100644 index 00000000000..6dd520aaa91 --- /dev/null +++ b/lib/galaxy/visualization/data_providers/__init__.py @@ -0,0 +1,3 @@ +""" +Galaxy visualization/visual analysis data providers. +""" \ No newline at end of file diff --git a/lib/galaxy/visualization/data_providers.py b/lib/galaxy/visualization/data_providers/basic.py similarity index 54% rename from lib/galaxy/visualization/data_providers.py rename to lib/galaxy/visualization/data_providers/basic.py index 1895ce8553e..eeedea875e9 100644 --- a/lib/galaxy/visualization/data_providers.py +++ b/lib/galaxy/visualization/data_providers/basic.py @@ -1,4 +1,6 @@ import sys +from galaxy.datatypes.tabular import Tabular +from galaxy.util.json import from_json_string class BaseDataProvider( object ): """ @@ -60,11 +62,13 @@ class ColumnDataProvider( BaseDataProvider ): # Attribute init. self.original_dataset = original_dataset - def get_data( self, cols, start_val=0, max_vals=sys.maxint ): + def get_data( self, cols, start_val=0, max_vals=sys.maxint, **kwargs ): """ Returns data from specified columns in dataset. Format is list of lists where each list is a line of data. """ + + cols = from_json_string( cols ) def cast_val( val, type ): """ Cast value based on type. """ @@ -91,3 +95,62 @@ class ColumnDataProvider( BaseDataProvider ): f.close() return data + +class DataProviderRegistry( object ): + """ + Registry for data providers that enables listing and lookup. + """ + + def __init__( self ): + # Mapping from dataset type name to a class that can fetch data from a file of that + # type. First key is converted dataset type; if result is another dict, second key + # is original dataset type. TODO: This needs to be more flexible. + self.dataset_type_name_to_data_provider = { + "tabix": { + Vcf: VcfTabixDataProvider, + Bed: BedTabixDataProvider, + Gtf: GtfTabixDataProvider, + ENCODEPeak: ENCODEPeakTabixDataProvider, + Interval: IntervalTabixDataProvider, + ChromatinInteractions: ChromatinInteractionsTabixDataProvider, + "default" : TabixDataProvider + }, + "interval_index": IntervalIndexDataProvider, + "bai": BamDataProvider, + "bam": SamDataProvider, + "summary_tree": SummaryTreeDataProvider, + "bigwig": BigWigDataProvider, + "bigbed": BigBedDataProvider + } + + def get_data_provider( name=None, original_dataset=None ): + """ + Returns data provider class by name and/or original dataset. + """ + data_provider = None + if name: + value = dataset_type_name_to_data_provider[ name ] + if isinstance( value, dict ): + # Get converter by dataset extension; if there is no data provider, + # get the default. + data_provider = value.get( original_dataset.datatype.__class__, value.get( "default" ) ) + else: + data_provider = value + elif original_dataset: + # Look up data provider from datatype's informaton. + try: + # Get data provider mapping and data provider for 'data'. If + # provider available, use it; otherwise use generic provider. + _ , data_provider_mapping = original_dataset.datatype.get_track_type() + if 'data_standalone' in data_provider_mapping: + data_provider_name = data_provider_mapping[ 'data_standalone' ] + else: + data_provider_name = data_provider_mapping[ 'data' ] + if data_provider_name: + data_provider = self.get_data_provider( name=data_provider_name, original_dataset=original_dataset ) + else: + data_provider = GenomeDataProvider + except: + pass + return data_provider + \ No newline at end of file diff --git a/lib/galaxy/visualization/genome/data_providers.py b/lib/galaxy/visualization/data_providers/genome.py similarity index 96% rename from lib/galaxy/visualization/genome/data_providers.py rename to lib/galaxy/visualization/data_providers/genome.py index 010e3600f1e..422201afd93 100644 --- a/lib/galaxy/visualization/genome/data_providers.py +++ b/lib/galaxy/visualization/data_providers/genome.py @@ -17,7 +17,7 @@ from bx.interval_index_file import Indexes from bx.bbi.bigwig_file import BigWigFile from galaxy.util.lrucache import LRUCache from galaxy.visualization.tracks.summary import * -from galaxy.visualization.data_providers import BaseDataProvider +from galaxy.visualization.data_providers.basic import BaseDataProvider import galaxy_utils.sequence.vcf from galaxy.datatypes.tabular import Tabular, Vcf from galaxy.datatypes.interval import Interval, Bed, Gff, Gtf, ENCODEPeak, ChromatinInteractions @@ -160,7 +160,7 @@ class GenomeDataProvider( BaseDataProvider ): """ raise Exception( "Unimplemented Function" ) - def get_data( self, chrom, start, end, start_val=0, max_vals=sys.maxint, **kwargs ): + def get_data( self, chrom=None, low=None, high=None, start_val=0, max_vals=sys.maxint, **kwargs ): """ Returns data in region defined by chrom, start, and end. start_val and max_vals are used to denote the data to return: start_val is the first element to @@ -169,6 +169,7 @@ class GenomeDataProvider( BaseDataProvider ): Return value must be a dictionary with the following attributes: dataset_type, data """ + start, end = int( low ), int( high ) iterator = self.get_iterator( chrom, start, end ) return self.process_data( iterator, start_val, max_vals, **kwargs ) @@ -1452,58 +1453,6 @@ class ChromatinInteractionsTabixDataProvider( TabixDataProvider, ChromatinIntera # -- Helper methods. -- # -# Mapping from dataset type name to a class that can fetch data from a file of that -# type. First key is converted dataset type; if result is another dict, second key -# is original dataset type. TODO: This needs to be more flexible. -dataset_type_name_to_data_provider = { - "tabix": { - Vcf: VcfTabixDataProvider, - Bed: BedTabixDataProvider, - Gtf: GtfTabixDataProvider, - ENCODEPeak: ENCODEPeakTabixDataProvider, - Interval: IntervalTabixDataProvider, - ChromatinInteractions: ChromatinInteractionsTabixDataProvider, - "default" : TabixDataProvider - }, - "interval_index": IntervalIndexDataProvider, - "bai": BamDataProvider, - "bam": SamDataProvider, - "summary_tree": SummaryTreeDataProvider, - "bigwig": BigWigDataProvider, - "bigbed": BigBedDataProvider -} - -def get_data_provider( name=None, original_dataset=None ): - """ - Returns data provider class by name and/or original dataset. - """ - data_provider = None - if name: - value = dataset_type_name_to_data_provider[ name ] - if isinstance( value, dict ): - # Get converter by dataset extension; if there is no data provider, - # get the default. - data_provider = value.get( original_dataset.datatype.__class__, value.get( "default" ) ) - else: - data_provider = value - elif original_dataset: - # Look up data provider from datatype's informaton. - try: - # Get data provider mapping and data provider for 'data'. If - # provider available, use it; otherwise use generic provider. - _ , data_provider_mapping = original_dataset.datatype.get_track_type() - if 'data_standalone' in data_provider_mapping: - data_provider_name = data_provider_mapping[ 'data_standalone' ] - else: - data_provider_name = data_provider_mapping[ 'data' ] - if data_provider_name: - data_provider = get_data_provider( name=data_provider_name, original_dataset=original_dataset ) - else: - data_provider = GenomeDataProvider - except: - pass - return data_provider - def package_gff_feature( feature, no_detail=False, filter_cols=[] ): """ Package a GFF feature in an array for data providers. """ feature = convert_gff_coords_to_bed( feature ) diff --git a/lib/galaxy/visualization/data_providers/registry.py b/lib/galaxy/visualization/data_providers/registry.py new file mode 100644 index 00000000000..75dadd1d2e9 --- /dev/null +++ b/lib/galaxy/visualization/data_providers/registry.py @@ -0,0 +1,73 @@ +from galaxy.visualization.data_providers.basic import ColumnDataProvider +from galaxy.visualization.data_providers.genome import * + +class DataProviderRegistry( object ): + """ + Registry for data providers that enables listing and lookup. + """ + + def __init__( self ): + # Mapping from dataset type name to a class that can fetch data from a file of that + # type. First key is converted dataset type; if result is another dict, second key + # is original dataset type. + self.dataset_type_name_to_data_provider = { + "tabix": { + Vcf: VcfTabixDataProvider, + Bed: BedTabixDataProvider, + Gtf: GtfTabixDataProvider, + ENCODEPeak: ENCODEPeakTabixDataProvider, + Interval: IntervalTabixDataProvider, + ChromatinInteractions: ChromatinInteractionsTabixDataProvider, + "default" : TabixDataProvider + }, + "interval_index": IntervalIndexDataProvider, + "bai": BamDataProvider, + "bam": SamDataProvider, + "summary_tree": SummaryTreeDataProvider, + "bigwig": BigWigDataProvider, + "bigbed": BigBedDataProvider + } + + def get_data_provider( self, name=None, raw=False, original_dataset=None ): + """ + Returns data provider class by name and/or original dataset. + """ + + # If getting raw data, use original dataset type to get data provider. + if raw: + if isinstance( original_dataset.datatype, Gff ): + return RawGFFDataProvider + elif isinstance( original_dataset.datatype, Bed ): + return RawBedDataProvider + elif isinstance( original_dataset.datatype, Vcf ): + return RawVcfDataProvider + elif isinstance( original_dataset.datatype, Tabular ): + return ColumnDataProvider + + # Using converted dataset, so get corrsponding data provider. + data_provider = None + if name: + value = self.dataset_type_name_to_data_provider[ name ] + if isinstance( value, dict ): + # Get converter by dataset extension; if there is no data provider, + # get the default. + data_provider = value.get( original_dataset.datatype.__class__, value.get( "default" ) ) + else: + data_provider = value + elif original_dataset: + # Look up data provider from datatype's informaton. + try: + # Get data provider mapping and data provider for 'data'. If + # provider available, use it; otherwise use generic provider. + _ , data_provider_mapping = original_dataset.datatype.get_track_type() + if 'data_standalone' in data_provider_mapping: + data_provider_name = data_provider_mapping[ 'data_standalone' ] + else: + data_provider_name = data_provider_mapping[ 'data' ] + if data_provider_name: + data_provider = self.get_data_provider( name=data_provider_name, original_dataset=original_dataset ) + else: + data_provider = GenomeDataProvider + except: + pass + return data_provider \ No newline at end of file diff --git a/lib/galaxy/visualization/phyloviz/phyloviz_dataprovider.py b/lib/galaxy/visualization/phyloviz/phyloviz_dataprovider.py index 87b528e4cb3..329d506b95c 100644 --- a/lib/galaxy/visualization/phyloviz/phyloviz_dataprovider.py +++ b/lib/galaxy/visualization/phyloviz/phyloviz_dataprovider.py @@ -1,7 +1,7 @@ from newickparser import Newick_Parser from nexusparser import Nexus_Parser from phyloxmlparser import Phyloxml_Parser -from galaxy.visualization.data_providers import BaseDataProvider +from galaxy.visualization.data_providers.basic import BaseDataProvider # TODO: bring this class into line with BaseDataProvider by # using BaseDataProvider.init() and providing original dataset diff --git a/lib/galaxy/web/api/datasets.py b/lib/galaxy/web/api/datasets.py index a3416fe074d..5ccbe6bff29 100644 --- a/lib/galaxy/web/api/datasets.py +++ b/lib/galaxy/web/api/datasets.py @@ -7,9 +7,9 @@ from galaxy import util, datatypes, jobs, web, util from galaxy.web.base.controller import * from galaxy.util.sanitize_html import sanitize_html from galaxy.model.orm import * -from galaxy.datatypes.interval import Gff, Bed +from galaxy.visualization.data_providers.genome import * +from galaxy.visualization.data_providers.basic import ColumnDataProvider from galaxy.datatypes.tabular import Vcf -from galaxy.visualization.genome.data_providers import * from galaxy.model import NoConverterException, ConverterDependencyException log = logging.getLogger( __name__ ) @@ -90,16 +90,17 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin ): # the client. valid_chroms = None # Check for data in the genome window. + data_provider_registry = trans.app.data_provider_registry if data_sources.get( 'index' ): tracks_dataset_type = data_sources['index']['name'] converted_dataset = dataset.get_converted_dataset( trans, tracks_dataset_type ) - indexer = get_data_provider( tracks_dataset_type )( converted_dataset, dataset ) + indexer = data_provider_registry.get_data_provider( tracks_dataset_type )( converted_dataset, dataset ) if not indexer.has_data( chrom ): return messages.NO_DATA #valid_chroms = indexer.valid_chroms() else: # Standalone data provider - standalone_provider = get_data_provider( data_sources['data_standalone']['name'] )( dataset ) + standalone_provider = data_provider_registry.get_data_provider( data_sources['data_standalone']['name'] )( dataset ) kwargs = {"stats": True} if not standalone_provider.has_data( chrom ): return messages.NO_DATA @@ -147,11 +148,12 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin ): extra_info = None mode = kwargs.get( "mode", "Auto" ) # Handle histogram mode uniquely for now: + data_provider_registry = trans.app.data_provider_registry if mode == "Coverage": # Get summary using minimal cutoffs. tracks_dataset_type = data_sources['index']['name'] converted_dataset = dataset.get_converted_dataset( trans, tracks_dataset_type ) - indexer = get_data_provider( tracks_dataset_type )( converted_dataset, dataset ) + indexer = data_provider_registry.get_data_provider( tracks_dataset_type )( converted_dataset, dataset ) summary = indexer.get_data( chrom, low, high, resolution=kwargs[ 'resolution' ], detail_cutoff=0, draw_cutoff=0 ) if summary == "detail": # Use maximum level of detail--2--to get summary data no matter the resolution. @@ -165,7 +167,7 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin ): # Have to choose between indexer and data provider tracks_dataset_type = data_sources['index']['name'] converted_dataset = dataset.get_converted_dataset( trans, tracks_dataset_type ) - indexer = get_data_provider( tracks_dataset_type )( converted_dataset, dataset ) + indexer = data_provider_registry.get_data_provider( tracks_dataset_type )( converted_dataset, dataset ) summary = indexer.get_data( chrom, low, high, resolution=kwargs[ 'resolution' ] ) if summary is None: return { 'dataset_type': tracks_dataset_type, 'data': None } @@ -180,11 +182,11 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin ): # Get data provider. if "data_standalone" in data_sources: tracks_dataset_type = data_sources['data_standalone']['name'] - data_provider_class = get_data_provider( name=tracks_dataset_type, original_dataset=dataset ) + data_provider_class = data_provider_registry.get_data_provider( name=tracks_dataset_type, original_dataset=dataset ) data_provider = data_provider_class( original_dataset=dataset ) else: tracks_dataset_type = data_sources['data']['name'] - data_provider_class = get_data_provider( name=tracks_dataset_type, original_dataset=dataset ) + data_provider_class = data_provider_registry.get_data_provider( name=tracks_dataset_type, original_dataset=dataset ) converted_dataset = dataset.get_converted_dataset( trans, tracks_dataset_type ) deps = dataset.get_converted_dataset_deps( trans, tracks_dataset_type ) data_provider = data_provider_class( converted_dataset=converted_dataset, original_dataset=dataset, dependencies=deps ) @@ -198,7 +200,7 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin ): result.update( { 'dataset_type': tracks_dataset_type, 'extra_info': extra_info } ) return result - def _raw_data( self, trans, dataset, chrom, low, high, **kwargs ): + def _raw_data( self, trans, dataset, **kwargs ): """ Uses original (raw) dataset to return data. This method is useful when the dataset is not yet indexed and hence using data would @@ -209,22 +211,19 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin ): msg = self.check_dataset_state( trans, dataset ) if msg: return msg - - low, high = int( low ), int( high ) - + # Return data. data = None - # TODO: for raw data requests, map dataset type to provider using dict in data_providers.py - if isinstance( dataset.datatype, Gff ): - data = RawGFFDataProvider( original_dataset=dataset ).get_data( chrom, low, high, **kwargs ) + data_provider = trans.app.data_provider_registry.get_data_provider( raw=True, original_dataset=dataset ) + if data_provider == ColumnDataProvider: + data = data_provider( original_dataset=dataset ).get_data( **kwargs ) + else: + # Default to genomic data. + # FIXME: need better way to set dataset_type. + low, high = int( kwargs.get( 'low' ) ), int( kwargs.get( 'high' ) ) + data = data_provider( original_dataset=dataset ).get_data( start=low, end=high, **kwargs ) data[ 'dataset_type' ] = 'interval_index' data[ 'extra_info' ] = None - elif isinstance( dataset.datatype, Bed ): - data = RawBedDataProvider( original_dataset=dataset ).get_data( chrom, low, high, **kwargs ) - data[ 'dataset_type' ] = 'interval_index' - data[ 'extra_info' ] = None - elif isinstance( dataset.datatype, Vcf ): - data = RawVcfDataProvider( original_dataset=dataset ).get_data( chrom, low, high, **kwargs ) - data[ 'dataset_type' ] = 'tabix' - data[ 'extra_info' ] = None + if isinstance( dataset.datatype, Vcf ): + data[ 'dataset_type' ] = 'tabix' return data diff --git a/lib/galaxy/web/api/tools.py b/lib/galaxy/web/api/tools.py index 167f07fb0a3..9e944a9c1c7 100644 --- a/lib/galaxy/web/api/tools.py +++ b/lib/galaxy/web/api/tools.py @@ -3,7 +3,7 @@ from galaxy.web.base.controller import BaseAPIController, UsesHistoryDatasetAsso from galaxy.visualization.genome.visual_analytics import get_dataset_job from galaxy.visualization.genomes import GenomeRegion from galaxy.util.json import to_json_string, from_json_string -from galaxy.visualization.genome.data_providers import * +from galaxy.visualization.data_providers.genome import * class ToolsController( BaseAPIController, UsesVisualizationMixin ): """ @@ -199,11 +199,12 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): # If running tool on region, convert input datasets (create indices) so # that can regions of data can be quickly extracted. # + data_provider_registry = trans.app.data_provider_registry messages_list = [] if run_on_regions: for jida in original_job.input_datasets: input_dataset = jida.dataset - if get_data_provider( original_dataset=input_dataset ): + if data_provider_registry.get_data_provider( original_dataset=input_dataset ): # Can index dataset. track_type, data_sources = input_dataset.datatype.get_track_type() # Convert to datasource that provides 'data' because we need to @@ -325,7 +326,7 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): trans.app.security_agent.set_all_dataset_permissions( new_dataset.dataset, hda_permissions ) # Write subset of data to new dataset - data_provider_class = get_data_provider( original_dataset=input_dataset ) + data_provider_class = data_provider_registry.get_data_provider( original_dataset=input_dataset ) data_provider = data_provider_class( original_dataset=input_dataset, converted_dataset=converted_dataset, dependencies=deps ) diff --git a/lib/galaxy/web/base/controller.py b/lib/galaxy/web/base/controller.py index 5d60fb45d36..bea851abdaf 100644 --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -14,7 +14,6 @@ from galaxy.workflow.modules import * from galaxy.web.framework import simplejson from galaxy.web.form_builder import AddressField, CheckboxField, SelectField, TextArea, TextField from galaxy.web.form_builder import WorkflowField, WorkflowMappingField, HistoryField, PasswordField, build_select_field -from galaxy.visualization.genome.data_providers import get_data_provider from galaxy.visualization.genome.visual_analytics import get_tool_def from galaxy.security.validate_user_input import validate_publicname from paste.httpexceptions import * @@ -488,7 +487,7 @@ class UsesVisualizationMixin( UsesHistoryDatasetAssociationMixin, prefs = {} track_type, _ = dataset.datatype.get_track_type() - track_data_provider_class = get_data_provider( original_dataset=dataset ) + track_data_provider_class = trans.app.data_provider_registry.get_data_provider( original_dataset=dataset ) track_data_provider = track_data_provider_class( original_dataset=dataset ) return { @@ -564,7 +563,7 @@ class UsesVisualizationMixin( UsesHistoryDatasetAssociationMixin, """ # Get data provider. track_type, _ = dataset.datatype.get_track_type() - track_data_provider_class = get_data_provider( original_dataset=dataset ) + track_data_provider_class = trans.app.data_provider_registry.get_data_provider( original_dataset=dataset ) track_data_provider = track_data_provider_class( original_dataset=dataset ) if isinstance( dataset, trans.app.model.HistoryDatasetAssociation ): diff --git a/lib/galaxy/web/controllers/visualization.py b/lib/galaxy/web/controllers/visualization.py index 715b592de75..bd98e1878b0 100644 --- a/lib/galaxy/web/controllers/visualization.py +++ b/lib/galaxy/web/controllers/visualization.py @@ -752,14 +752,15 @@ class VisualizationController( BaseUIController, SharableMixin, UsesAnnotations, # Get dataset and indexed datatype. dataset = self.get_hda_or_ldda( trans, track[ 'hda_ldda'], track[ 'dataset_id' ] ) data_sources = self._get_datasources( trans, dataset ) + data_provider_registry = trans.app.data_provider_registry if 'data_standalone' in data_sources: indexed_type = data_sources['data_standalone']['name'] - data_provider = get_data_provider( indexed_type )( dataset ) + data_provider = data_provider_registry.get_data_provider( indexed_type )( dataset ) else: indexed_type = data_sources['index']['name'] # Get converted dataset and append track's genome data. converted_dataset = dataset.get_converted_dataset( trans, indexed_type ) - data_provider = get_data_provider( indexed_type )( converted_dataset, dataset ) + data_provider = data_provider_registry.get_data_provider( indexed_type )( converted_dataset, dataset ) # HACK: pass in additional params, which are only used for summary tree data, not BBI data. track[ 'genome_wide_data' ] = { 'data': data_provider.get_genome_data( chroms_info, level=4, detail_cutoff=0, draw_cutoff=0 ) }