Data providers framework enhancements: (a) add registry object; (b) move all provider lookup code into registry; and (c) integrate ColumnDataProvider into raw_data requests.

lib/galaxy/app.py

@@ -13,6 +13,7 @@ from galaxy.objectstore import build_object_store_from_config
 import galaxy.quota
 from galaxy.tags.tag_handler import GalaxyTagHandler
 from galaxy.visualization.genomes import Genomes
+from galaxy.visualization.data_providers.registry import DataProviderRegistry
 from galaxy.tools.imp_exp import load_history_imp_exp_tools
 from galaxy.tools.genome_index import load_genome_index_tools
 from galaxy.sample_tracking import external_service_types
@@ -73,6 +74,8 @@ class UniverseApplication( object ):
         self.tag_handler = GalaxyTagHandler()
         # Genomes
         self.genomes = Genomes( self )
+        # Data providers registry.
+        self.data_provider_registry = DataProviderRegistry()
         # Tool data tables
         self.tool_data_tables = galaxy.tools.data.ToolDataTableManager( self.config.tool_data_path, self.config.tool_data_table_config_path )
         # Initialize the tools, making sure the list of tool configs includes the reserved migrated_tools_conf.xml file.

lib/galaxy/visualization/data_providers/__init__.py

@@ -0,0 +1,3 @@
+"""
+Galaxy visualization/visual analysis data providers.
+"""

lib/galaxy/visualization/data_providers/basic.py

@@ -1,4 +1,6 @@
 import sys
+from galaxy.datatypes.tabular import Tabular
+from galaxy.util.json import from_json_string
 
 class BaseDataProvider( object ):
     """
@@ -60,11 +62,13 @@ class ColumnDataProvider( BaseDataProvider ):
         # Attribute init.
         self.original_dataset = original_dataset
         
-    def get_data( self, cols, start_val=0, max_vals=sys.maxint ):
+    def get_data( self, cols, start_val=0, max_vals=sys.maxint, **kwargs ):
         """
         Returns data from specified columns in dataset. Format is list of lists 
         where each list is a line of data.
         """
+
+        cols = from_json_string( cols )
         
         def cast_val( val, type ):
             """ Cast value based on type. """
@@ -91,3 +95,62 @@ class ColumnDataProvider( BaseDataProvider ):
         f.close()
             
         return data
+
+class DataProviderRegistry( object ):
+    """
+    Registry for data providers that enables listing and lookup.
+    """
+
+    def __init__( self ):
+        # Mapping from dataset type name to a class that can fetch data from a file of that
+        # type. First key is converted dataset type; if result is another dict, second key
+        # is original dataset type. TODO: This needs to be more flexible.
+        self.dataset_type_name_to_data_provider = {
+            "tabix": { 
+                Vcf: VcfTabixDataProvider,
+                Bed: BedTabixDataProvider,
+                Gtf: GtfTabixDataProvider,
+                ENCODEPeak: ENCODEPeakTabixDataProvider,
+                Interval: IntervalTabixDataProvider,
+                ChromatinInteractions: ChromatinInteractionsTabixDataProvider, 
+                "default" : TabixDataProvider 
+            },
+            "interval_index": IntervalIndexDataProvider,
+            "bai": BamDataProvider,
+            "bam": SamDataProvider,
+            "summary_tree": SummaryTreeDataProvider,
+            "bigwig": BigWigDataProvider,
+            "bigbed": BigBedDataProvider
+        }
+
+    def get_data_provider( name=None, original_dataset=None ):
+        """
+        Returns data provider class by name and/or original dataset.
+        """
+        data_provider = None
+        if name:
+            value = dataset_type_name_to_data_provider[ name ]
+            if isinstance( value, dict ):
+                # Get converter by dataset extension; if there is no data provider,
+                # get the default.
+                data_provider = value.get( original_dataset.datatype.__class__, value.get( "default" ) )
+            else:
+                data_provider = value
+        elif original_dataset:
+            # Look up data provider from datatype's informaton.
+            try:
+                # Get data provider mapping and data provider for 'data'. If 
+                # provider available, use it; otherwise use generic provider.
+                _ , data_provider_mapping = original_dataset.datatype.get_track_type()
+                if 'data_standalone' in data_provider_mapping:
+                    data_provider_name = data_provider_mapping[ 'data_standalone' ]
+                else:
+                    data_provider_name = data_provider_mapping[ 'data' ]
+                if data_provider_name:
+                    data_provider = self.get_data_provider( name=data_provider_name, original_dataset=original_dataset )
+                else: 
+                    data_provider = GenomeDataProvider
+            except:
+                pass
+        return data_provider
+        

lib/galaxy/visualization/data_providers/genome.py

@@ -17,7 +17,7 @@ from bx.interval_index_file import Indexes
 from bx.bbi.bigwig_file import BigWigFile
 from galaxy.util.lrucache import LRUCache
 from galaxy.visualization.tracks.summary import *
-from galaxy.visualization.data_providers import BaseDataProvider
+from galaxy.visualization.data_providers.basic import BaseDataProvider
 import galaxy_utils.sequence.vcf
 from galaxy.datatypes.tabular import Tabular, Vcf
 from galaxy.datatypes.interval import Interval, Bed, Gff, Gtf, ENCODEPeak, ChromatinInteractions
@@ -160,7 +160,7 @@ class GenomeDataProvider( BaseDataProvider ):
         """
         raise Exception( "Unimplemented Function" )        
         
-    def get_data( self, chrom, start, end, start_val=0, max_vals=sys.maxint, **kwargs ):
+    def get_data( self, chrom=None, low=None, high=None, start_val=0, max_vals=sys.maxint, **kwargs ):
         """ 
         Returns data in region defined by chrom, start, and end. start_val and
         max_vals are used to denote the data to return: start_val is the first element to 
@@ -169,6 +169,7 @@ class GenomeDataProvider( BaseDataProvider ):
         Return value must be a dictionary with the following attributes:
             dataset_type, data
         """
+        start, end = int( low ), int( high )
         iterator = self.get_iterator( chrom, start, end )
         return self.process_data( iterator, start_val, max_vals, **kwargs )
 
@@ -1452,58 +1453,6 @@ class ChromatinInteractionsTabixDataProvider( TabixDataProvider, ChromatinIntera
 # -- Helper methods. --
 #
 
-# Mapping from dataset type name to a class that can fetch data from a file of that
-# type. First key is converted dataset type; if result is another dict, second key
-# is original dataset type. TODO: This needs to be more flexible.
-dataset_type_name_to_data_provider = {
-    "tabix": { 
-        Vcf: VcfTabixDataProvider,
-        Bed: BedTabixDataProvider,
-        Gtf: GtfTabixDataProvider,
-        ENCODEPeak: ENCODEPeakTabixDataProvider,
-        Interval: IntervalTabixDataProvider,
-        ChromatinInteractions: ChromatinInteractionsTabixDataProvider, 
-        "default" : TabixDataProvider 
-    },
-    "interval_index": IntervalIndexDataProvider,
-    "bai": BamDataProvider,
-    "bam": SamDataProvider,
-    "summary_tree": SummaryTreeDataProvider,
-    "bigwig": BigWigDataProvider,
-    "bigbed": BigBedDataProvider
-}
-
-def get_data_provider( name=None, original_dataset=None ):
-    """
-    Returns data provider class by name and/or original dataset.
-    """
-    data_provider = None
-    if name:
-        value = dataset_type_name_to_data_provider[ name ]
-        if isinstance( value, dict ):
-            # Get converter by dataset extension; if there is no data provider,
-            # get the default.
-            data_provider = value.get( original_dataset.datatype.__class__, value.get( "default" ) )
-        else:
-            data_provider = value
-    elif original_dataset:
-        # Look up data provider from datatype's informaton.
-        try:
-            # Get data provider mapping and data provider for 'data'. If 
-            # provider available, use it; otherwise use generic provider.
-            _ , data_provider_mapping = original_dataset.datatype.get_track_type()
-            if 'data_standalone' in data_provider_mapping:
-                data_provider_name = data_provider_mapping[ 'data_standalone' ]
-            else:
-                data_provider_name = data_provider_mapping[ 'data' ]
-            if data_provider_name:
-                data_provider = get_data_provider( name=data_provider_name, original_dataset=original_dataset )
-            else: 
-                data_provider = GenomeDataProvider
-        except:
-            pass
-    return data_provider
-
 def package_gff_feature( feature, no_detail=False, filter_cols=[] ):
     """ Package a GFF feature in an array for data providers. """
     feature = convert_gff_coords_to_bed( feature )

lib/galaxy/visualization/data_providers/registry.py

@@ -0,0 +1,73 @@
+from galaxy.visualization.data_providers.basic import ColumnDataProvider
+from galaxy.visualization.data_providers.genome import *
+
+class DataProviderRegistry( object ):
+    """
+    Registry for data providers that enables listing and lookup.
+    """
+
+    def __init__( self ):
+        # Mapping from dataset type name to a class that can fetch data from a file of that
+        # type. First key is converted dataset type; if result is another dict, second key
+        # is original dataset type.
+        self.dataset_type_name_to_data_provider = {
+            "tabix": { 
+                Vcf: VcfTabixDataProvider,
+                Bed: BedTabixDataProvider,
+                Gtf: GtfTabixDataProvider,
+                ENCODEPeak: ENCODEPeakTabixDataProvider,
+                Interval: IntervalTabixDataProvider,
+                ChromatinInteractions: ChromatinInteractionsTabixDataProvider, 
+                "default" : TabixDataProvider 
+            },
+            "interval_index": IntervalIndexDataProvider,
+            "bai": BamDataProvider,
+            "bam": SamDataProvider,
+            "summary_tree": SummaryTreeDataProvider,
+            "bigwig": BigWigDataProvider,
+            "bigbed": BigBedDataProvider
+        }
+
+    def get_data_provider( self, name=None, raw=False, original_dataset=None ):
+        """
+        Returns data provider class by name and/or original dataset.
+        """
+
+        # If getting raw data, use original dataset type to get data provider.
+        if raw:
+            if isinstance( original_dataset.datatype, Gff ):
+                return RawGFFDataProvider
+            elif isinstance( original_dataset.datatype, Bed ):
+                return RawBedDataProvider
+            elif isinstance( original_dataset.datatype, Vcf ):
+                return RawVcfDataProvider
+            elif isinstance( original_dataset.datatype, Tabular ):
+                return ColumnDataProvider
+
+        # Using converted dataset, so get corrsponding data provider.
+        data_provider = None
+        if name:
+            value = self.dataset_type_name_to_data_provider[ name ]
+            if isinstance( value, dict ):
+                # Get converter by dataset extension; if there is no data provider,
+                # get the default.
+                data_provider = value.get( original_dataset.datatype.__class__, value.get( "default" ) )
+            else:
+                data_provider = value
+        elif original_dataset:
+            # Look up data provider from datatype's informaton.
+            try:
+                # Get data provider mapping and data provider for 'data'. If 
+                # provider available, use it; otherwise use generic provider.
+                _ , data_provider_mapping = original_dataset.datatype.get_track_type()
+                if 'data_standalone' in data_provider_mapping:
+                    data_provider_name = data_provider_mapping[ 'data_standalone' ]
+                else:
+                    data_provider_name = data_provider_mapping[ 'data' ]
+                if data_provider_name:
+                    data_provider = self.get_data_provider( name=data_provider_name, original_dataset=original_dataset )
+                else: 
+                    data_provider = GenomeDataProvider
+            except:
+                pass
+        return data_provider

lib/galaxy/visualization/phyloviz/phyloviz_dataprovider.py

@@ -1,7 +1,7 @@
 from newickparser import Newick_Parser
 from nexusparser import Nexus_Parser
 from phyloxmlparser import Phyloxml_Parser
-from galaxy.visualization.data_providers import BaseDataProvider
+from galaxy.visualization.data_providers.basic import BaseDataProvider
 
 # TODO: bring this class into line with BaseDataProvider by 
 # using BaseDataProvider.init() and providing original dataset

lib/galaxy/web/api/datasets.py

@@ -7,9 +7,9 @@ from galaxy import util, datatypes, jobs, web, util
 from galaxy.web.base.controller import *
 from galaxy.util.sanitize_html import sanitize_html
 from galaxy.model.orm import *
-from galaxy.datatypes.interval import Gff, Bed
+from galaxy.visualization.data_providers.genome import *
+from galaxy.visualization.data_providers.basic import ColumnDataProvider
 from galaxy.datatypes.tabular import Vcf
-from galaxy.visualization.genome.data_providers import *
 from galaxy.model import NoConverterException, ConverterDependencyException
 
 log = logging.getLogger( __name__ )
@@ -90,16 +90,17 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin ):
         # the client.
         valid_chroms = None
         # Check for data in the genome window.
+        data_provider_registry = trans.app.data_provider_registry
         if data_sources.get( 'index' ):
             tracks_dataset_type = data_sources['index']['name']
             converted_dataset = dataset.get_converted_dataset( trans, tracks_dataset_type )
-            indexer = get_data_provider( tracks_dataset_type )( converted_dataset, dataset )
+            indexer = data_provider_registry.get_data_provider( tracks_dataset_type )( converted_dataset, dataset )
             if not indexer.has_data( chrom ):
                 return messages.NO_DATA
             #valid_chroms = indexer.valid_chroms()
         else:
             # Standalone data provider
-            standalone_provider = get_data_provider( data_sources['data_standalone']['name'] )( dataset )
+            standalone_provider = data_provider_registry.get_data_provider( data_sources['data_standalone']['name'] )( dataset )
             kwargs = {"stats": True}
             if not standalone_provider.has_data( chrom ):
                 return messages.NO_DATA
@@ -147,11 +148,12 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin ):
         extra_info = None
         mode = kwargs.get( "mode", "Auto" )
         # Handle histogram mode uniquely for now:
+        data_provider_registry = trans.app.data_provider_registry
         if mode == "Coverage":
             # Get summary using minimal cutoffs.
             tracks_dataset_type = data_sources['index']['name']
             converted_dataset = dataset.get_converted_dataset( trans, tracks_dataset_type )
-            indexer = get_data_provider( tracks_dataset_type )( converted_dataset, dataset )
+            indexer = data_provider_registry.get_data_provider( tracks_dataset_type )( converted_dataset, dataset )
             summary = indexer.get_data( chrom, low, high, resolution=kwargs[ 'resolution' ], detail_cutoff=0, draw_cutoff=0 )
             if summary == "detail":
                 # Use maximum level of detail--2--to get summary data no matter the resolution.
@@ -165,7 +167,7 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin ):
             # Have to choose between indexer and data provider
             tracks_dataset_type = data_sources['index']['name']
             converted_dataset = dataset.get_converted_dataset( trans, tracks_dataset_type )
-            indexer = get_data_provider( tracks_dataset_type )( converted_dataset, dataset )
+            indexer = data_provider_registry.get_data_provider( tracks_dataset_type )( converted_dataset, dataset )
             summary = indexer.get_data( chrom, low, high, resolution=kwargs[ 'resolution' ] )
             if summary is None:
                 return { 'dataset_type': tracks_dataset_type, 'data': None }
@@ -180,11 +182,11 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin ):
         # Get data provider.
         if "data_standalone" in data_sources:
             tracks_dataset_type = data_sources['data_standalone']['name']
-            data_provider_class = get_data_provider( name=tracks_dataset_type, original_dataset=dataset )
+            data_provider_class = data_provider_registry.get_data_provider( name=tracks_dataset_type, original_dataset=dataset )
             data_provider = data_provider_class( original_dataset=dataset )
         else:
             tracks_dataset_type = data_sources['data']['name']
-            data_provider_class = get_data_provider( name=tracks_dataset_type, original_dataset=dataset )
+            data_provider_class = data_provider_registry.get_data_provider( name=tracks_dataset_type, original_dataset=dataset )
             converted_dataset = dataset.get_converted_dataset( trans, tracks_dataset_type )
             deps = dataset.get_converted_dataset_deps( trans, tracks_dataset_type )
             data_provider = data_provider_class( converted_dataset=converted_dataset, original_dataset=dataset, dependencies=deps )
@@ -198,7 +200,7 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin ):
         result.update( { 'dataset_type': tracks_dataset_type, 'extra_info': extra_info } )
         return result
 
-    def _raw_data( self, trans, dataset, chrom, low, high, **kwargs ):
+    def _raw_data( self, trans, dataset, **kwargs ):
         """
         Uses original (raw) dataset to return data. This method is useful 
         when the dataset is not yet indexed and hence using data would
@@ -209,22 +211,19 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin ):
         msg = self.check_dataset_state( trans, dataset )
         if msg:
             return msg
-
-        low, high = int( low ), int( high )
-            
+    
         # Return data.
         data = None
-        # TODO: for raw data requests, map dataset type to provider using dict in data_providers.py
-        if isinstance( dataset.datatype, Gff ):
-            data = RawGFFDataProvider( original_dataset=dataset ).get_data( chrom, low, high, **kwargs )
+        data_provider = trans.app.data_provider_registry.get_data_provider( raw=True, original_dataset=dataset )
+        if data_provider == ColumnDataProvider:
+            data = data_provider( original_dataset=dataset ).get_data( **kwargs )
+        else:
+            # Default to genomic data.
+            # FIXME: need better way to set dataset_type.
+            low, high = int( kwargs.get( 'low' ) ), int( kwargs.get( 'high' ) )
+            data = data_provider( original_dataset=dataset ).get_data( start=low, end=high, **kwargs )
             data[ 'dataset_type' ] = 'interval_index'
             data[ 'extra_info' ] = None
-        elif isinstance( dataset.datatype, Bed ):
-            data = RawBedDataProvider( original_dataset=dataset ).get_data( chrom, low, high, **kwargs )
-            data[ 'dataset_type' ] = 'interval_index'
-            data[ 'extra_info' ] = None
-        elif isinstance( dataset.datatype, Vcf ):
-            data = RawVcfDataProvider( original_dataset=dataset ).get_data( chrom, low, high, **kwargs )
-            data[ 'dataset_type' ] = 'tabix'
-            data[ 'extra_info' ] = None
+            if isinstance( dataset.datatype, Vcf ):
+                data[ 'dataset_type' ] = 'tabix'
         return data

lib/galaxy/web/api/tools.py

@@ -3,7 +3,7 @@ from galaxy.web.base.controller import BaseAPIController, UsesHistoryDatasetAsso
 from galaxy.visualization.genome.visual_analytics import get_dataset_job
 from galaxy.visualization.genomes import GenomeRegion
 from galaxy.util.json import to_json_string, from_json_string
-from galaxy.visualization.genome.data_providers import *
+from galaxy.visualization.data_providers.genome import *
 
 class ToolsController( BaseAPIController, UsesVisualizationMixin ):
     """
@@ -199,11 +199,12 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ):
         # If running tool on region, convert input datasets (create indices) so
         # that can regions of data can be quickly extracted.
         # 
+        data_provider_registry = trans.app.data_provider_registry
         messages_list = []
         if run_on_regions:
             for jida in original_job.input_datasets:
                 input_dataset = jida.dataset
-                if get_data_provider( original_dataset=input_dataset ):
+                if data_provider_registry.get_data_provider( original_dataset=input_dataset ):
                     # Can index dataset.
                     track_type, data_sources = input_dataset.datatype.get_track_type()
                     # Convert to datasource that provides 'data' because we need to
@@ -325,7 +326,7 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ):
                     trans.app.security_agent.set_all_dataset_permissions( new_dataset.dataset, hda_permissions )
 
                     # Write subset of data to new dataset
-                    data_provider_class = get_data_provider( original_dataset=input_dataset )
+                    data_provider_class = data_provider_registry.get_data_provider( original_dataset=input_dataset )
                     data_provider = data_provider_class( original_dataset=input_dataset, 
                                                          converted_dataset=converted_dataset,
                                                          dependencies=deps )

lib/galaxy/web/base/controller.py

@@ -14,7 +14,6 @@ from galaxy.workflow.modules import *
 from galaxy.web.framework import simplejson
 from galaxy.web.form_builder import AddressField, CheckboxField, SelectField, TextArea, TextField
 from galaxy.web.form_builder import WorkflowField, WorkflowMappingField, HistoryField, PasswordField, build_select_field
-from galaxy.visualization.genome.data_providers import get_data_provider
 from galaxy.visualization.genome.visual_analytics import get_tool_def
 from galaxy.security.validate_user_input import validate_publicname
 from paste.httpexceptions import *
@@ -488,7 +487,7 @@ class UsesVisualizationMixin( UsesHistoryDatasetAssociationMixin,
                     prefs = {}
 
                 track_type, _ = dataset.datatype.get_track_type()
-                track_data_provider_class = get_data_provider( original_dataset=dataset )
+                track_data_provider_class = trans.app.data_provider_registry.get_data_provider( original_dataset=dataset )
                 track_data_provider = track_data_provider_class( original_dataset=dataset )
                 
                 return {
@@ -564,7 +563,7 @@ class UsesVisualizationMixin( UsesHistoryDatasetAssociationMixin,
         """
         # Get data provider.
         track_type, _ = dataset.datatype.get_track_type()
-        track_data_provider_class = get_data_provider( original_dataset=dataset )
+        track_data_provider_class = trans.app.data_provider_registry.get_data_provider( original_dataset=dataset )
         track_data_provider = track_data_provider_class( original_dataset=dataset )
         
         if isinstance( dataset, trans.app.model.HistoryDatasetAssociation ):

lib/galaxy/web/controllers/visualization.py

@@ -752,14 +752,15 @@ class VisualizationController( BaseUIController, SharableMixin, UsesAnnotations,
             # Get dataset and indexed datatype.
             dataset = self.get_hda_or_ldda( trans, track[ 'hda_ldda'], track[ 'dataset_id' ] )
             data_sources = self._get_datasources( trans, dataset )
+            data_provider_registry = trans.app.data_provider_registry
             if 'data_standalone' in data_sources:
                 indexed_type = data_sources['data_standalone']['name']
-                data_provider = get_data_provider( indexed_type )( dataset )
+                data_provider = data_provider_registry.get_data_provider( indexed_type )( dataset )
             else:
                 indexed_type = data_sources['index']['name']
                 # Get converted dataset and append track's genome data.
                 converted_dataset = dataset.get_converted_dataset( trans, indexed_type )
-                data_provider = get_data_provider( indexed_type )( converted_dataset, dataset )
+                data_provider = data_provider_registry.get_data_provider( indexed_type )( converted_dataset, dataset )
             # HACK: pass in additional params, which are only used for summary tree data, not BBI data.
             track[ 'genome_wide_data' ] = { 'data': data_provider.get_genome_data( chroms_info, level=4, detail_cutoff=0, draw_cutoff=0 ) }