Automated merge with https://bitbucket.org/galaxy/galaxy-central/

lib/galaxy/app.py

@@ -2,6 +2,7 @@ import sys, os, atexit
 
 from galaxy import config, jobs, util, tools, web
 import galaxy.tools.search
+import galaxy.tools.data
 from galaxy.web import security
 import galaxy.model
 import galaxy.datatypes.registry
@@ -36,6 +37,8 @@ class UniverseApplication( object ):
         self.security = security.SecurityHelper( id_secret=self.config.id_secret )
         # Tag handler
         self.tag_handler = GalaxyTagHandler()
+        # Tool data tables
+        self.tool_data_tables = galaxy.tools.data.ToolDataTableManager( self.config.tool_data_table_config_path )
         # Initialize the tools
         self.toolbox = tools.ToolBox( self.config.tool_config, self.config.tool_path, self )
         # Search support for tools

lib/galaxy/config.py

@@ -48,6 +48,7 @@ class Configuration( object ):
         self.tool_data_path = resolve_path( kwargs.get( "tool_data_path", "tool-data" ), os.getcwd() )
         self.test_conf = resolve_path( kwargs.get( "test_conf", "" ), self.root )
         self.tool_config = resolve_path( kwargs.get( 'tool_config_file', 'tool_conf.xml' ), self.root )
+        self.tool_data_table_config_path = resolve_path( kwargs.get( 'tool_data_table_config_path', 'tool_data_table_conf.xml' ), self.root )
         self.tool_secret = kwargs.get( "tool_secret", "" )
         self.id_secret = kwargs.get( "id_secret", "USING THE DEFAULT IS NOT SECURE!" )
         self.set_metadata_externally = string_as_bool( kwargs.get( "set_metadata_externally", "False" ) )

lib/galaxy/tools/data/__init__.py

@@ -0,0 +1,131 @@
+"""
+Manage tool data tables, which store (at the application level) data that is
+used by tools, for example in the generation of dynamic options. Tables are
+loaded and stored by names which tools use to refer to them. This allows
+users to configure data tables for a local Galaxy instance without needing
+to modify the tool configurations. 
+"""
+
+import logging, sys, os.path
+from galaxy import util
+
+log = logging.getLogger( __name__ )
+
+class ToolDataTableManager( object ):
+    """
+    Manages a collection of tool data tables
+    """
+    
+    def __init__( self, config_filename=None ):
+        self.data_tables = {}
+        if config_filename:
+            self.add_from_config_file( config_filename )
+        
+    def __getitem__( self, key ):
+        return self.data_tables.__getitem__( key )
+        
+    def __contains__( self, key ):
+        return self.data_tables.__contains__( key )
+        
+    def add_from_config_file( self, config_filename ):
+        tree = util.parse_xml( config_filename )
+        root = tree.getroot()
+        for table_elem in root.findall( 'table' ):
+            type = table_elem.get( 'type', 'tabular' )
+            assert type in tool_data_table_types, "Unknown data table type '%s'" % type
+            table = tool_data_table_types[ type ]( table_elem )
+            self.data_tables[ table.name ] = table
+            log.debug( "Loaded tool data table '%s", table.name )
+        print >> sys.stderr, repr( self.data_tables )
+    
+class ToolDataTable( object ):
+    def __init__( self, config_element ):
+        self.name = config_element.get( 'name' )
+    
+class TabularToolDataTable( ToolDataTable ):
+    """
+    Data stored in a tabular / separated value format on disk, allows multiple
+    files to be merged but all must have the same column definitions.
+    
+    <table type="tabular" name="test">
+        <column name='...' index = '...' />
+        <file path="..." />
+        <file path="..." />
+    </table>
+    """
+    
+    type_key = 'tabular'
+    
+    def __init__( self, config_element ):
+        super( TabularToolDataTable, self ).__init__( config_element )
+        self.configure_and_load( config_element )
+    
+    def configure_and_load( self, config_element ):
+        """
+        Configure and load table from an XML element.
+        """
+        self.separator = config_element.get( 'separator', '\t' )
+        self.comment_char = config_element.get( 'comment_char', '#' )
+        # Configure columns
+        self.parse_column_spec( config_element )
+        # Read every file
+        all_rows = []
+        for file_element in config_element.findall( 'file' ):
+            filename = file_element.get( 'path' )
+            assert os.path.exists( filename ), \
+                "Cannot find index file '%s' for tool data table '%s'" % ( filename, self.name )
+            all_rows.extend( self.parse_file_fields( open( filename ) ) )
+        self.data = all_rows
+        
+    def get_fields( self ):
+        return self.data
+            
+    def parse_column_spec( self, config_element ):
+        """
+        Parse column definitions, which can either be a set of 'column' elements
+        with a name and index (as in dynamic options config), or a shorthand
+        comma separated list of names in order as the text of a 'column_names'
+        element.
+        
+        A column named 'value' is required. 
+        """
+        self.columns = {}
+        if config_element.find( 'columns' ) is not None:
+            column_names = util.xml_text( config_element.find( 'columns' ) )
+            column_names = [ n.strip() for n in column_names.split( ',' ) ]
+            for index, name in enumerate( column_names ):
+                self.columns[ name ] = index
+                self.largest_index = index
+        else:
+            for column_elem in config_element.findall( 'column' ):
+                name = column_elem.get( 'name', None )
+                assert name is not None, "Required 'name' attribute missing from column def"
+                index = column_elem.get( 'index', None )
+                assert index is not None, "Required 'index' attribute missing from column def"
+                index = int( index )
+                self.columns[name] = index
+                if index > self.largest_index:
+                    self.largest_index = index
+        assert 'value' in self.columns, "Required 'value' column missing from column def"
+        if 'name' not in self.columns:
+            self.columns['name'] = self.columns['value']
+        
+    def parse_file_fields( self, reader ):
+        """
+        Parse separated lines from file and return a list of tuples.
+        
+        TODO: Allow named access to fields using the column names.
+        """
+        rval = []
+        for line in reader:
+            if line.lstrip().startswith( self.comment_char ):
+                continue
+            line = line.rstrip( "\n\r" )
+            if line:
+                fields = line.split( self.separator )
+                if self.largest_index < len( fields ):
+                    rval.append( fields )
+        return rval        
+
+# Registry of tool data types by type_key
+tool_data_table_types = dict( [ ( cls.type_key, cls ) for cls in [ TabularToolDataTable ] ] )

lib/galaxy/tools/parameters/dynamic_options.py

@@ -46,9 +46,9 @@ class StaticValueFilter( Filter ):
         Filter.__init__( self, d_option, elem )
         self.value = elem.get( "value", None )
         assert self.value is not None, "Required 'value' attribute missing from filter"
-        self.column = elem.get( "column", None )
-        assert self.column is not None, "Required 'column' attribute missing from filter, when loading from file"
-        self.column = int ( self.column )
+        column = elem.get( "column", None )
+        assert column is not None, "Required 'column' attribute missing from filter, when loading from file"
+        self.column = d_option.column_spec_to_index( column )
         self.keep = string_as_bool( elem.get( "keep", 'True' ) )
     def filter_options( self, options, trans, other_values ):
         rval = []
@@ -81,11 +81,11 @@ class DataMetaFilter( Filter ):
         d_option.has_dataset_dependencies = True
         self.key = elem.get( "key", None )
         assert self.key is not None, "Required 'key' attribute missing from filter"
-        self.column = elem.get( "column", None )
-        if self.column is None:
+        column = elem.get( "column", None )
+        if column is None:
             assert self.dynamic_option.file_fields is None and self.dynamic_option.dataset_ref_name is None, "Required 'column' attribute missing from filter, when loading from file"
         else:
-            self.column = int ( self.column )
+            self.column = d_option.column_spec_to_index( column )
         self.multiple = string_as_bool( elem.get( "multiple", "False" ) )
         self.separator = elem.get( "separator", "," )
     def get_dependency_name( self ):
@@ -142,9 +142,9 @@ class ParamValueFilter( Filter ):
         Filter.__init__( self, d_option, elem )
         self.ref_name = elem.get( "ref", None )
         assert self.ref_name is not None, "Required 'ref' attribute missing from filter"
-        self.column = elem.get( "column", None )
-        assert self.column is not None, "Required 'column' attribute missing from filter"
-        self.column = int ( self.column )
+        column = elem.get( "column", None )
+        assert column is not None, "Required 'column' attribute missing from filter"
+        self.column = d_option.column_spec_to_index( column )
         self.keep = string_as_bool( elem.get( "keep", 'True' ) )
         self.ref_attribute = elem.get( "ref_attribute", None )
         if self.ref_attribute:
@@ -177,9 +177,9 @@ class UniqueValueFilter( Filter ):
     """
     def __init__( self, d_option, elem ):
         Filter.__init__( self, d_option, elem )
-        self.column = elem.get( "column", None )
-        assert self.column is not None, "Required 'column' attribute missing from filter"
-        self.column = int ( self.column )
+        column = elem.get( "column", None )
+        assert column is not None, "Required 'column' attribute missing from filter"
+        self.column = d_option.column_spec_to_index( column )
     def get_dependency_name( self ):
         return self.dynamic_option.dataset_ref_name
     def filter_options( self, options, trans, other_values ):
@@ -205,9 +205,9 @@ class MultipleSplitterFilter( Filter ):
     def __init__( self, d_option, elem ):
         Filter.__init__( self, d_option, elem )
         self.separator = elem.get( "separator", "," )
-        self.columns = elem.get( "column", None )
-        assert self.columns is not None, "Required 'columns' attribute missing from filter"
-        self.columns = [ int ( column ) for column in self.columns.split( "," ) ]
+        columns = elem.get( "column", None )
+        assert columns is not None, "Required 'columns' attribute missing from filter"
+        self.columns = [ d_option.column_spec_to_index( column ) for column in columns.split( "," ) ]
     def filter_options( self, options, trans, other_values ):
         rval = []
         for fields in options:
@@ -345,9 +345,9 @@ class SortByColumnFilter( Filter ):
     """
     def __init__( self, d_option, elem ):
         Filter.__init__( self, d_option, elem )
-        self.column = elem.get( "column", None )
-        assert self.column is not None, "Required 'column' attribute missing from filter"
-        self.column = int( self.column )
+        column = elem.get( "column", None )
+        assert column is not None, "Required 'column' attribute missing from filter"
+        self.column = d_option.column_spec_to_index( column )
     def filter_options( self, options, trans, other_values ):
         rval = []
         for i, fields in enumerate( options ):
@@ -398,20 +398,25 @@ class DynamicOptions( object ):
         data_file = elem.get( 'from_file', None )
         dataset_file = elem.get( 'from_dataset', None )
         from_parameter = elem.get( 'from_parameter', None )
-        if data_file is not None or dataset_file is not None or from_parameter is not None:
-            for column_elem in elem.findall( 'column' ):
-                name = column_elem.get( 'name', None )
-                assert name is not None, "Required 'name' attribute missing from column def"
-                index = column_elem.get( 'index', None )
-                assert index is not None, "Required 'index' attribute missing from column def"
-                index = int( index )
-                self.columns[name] = index
-                if index > self.largest_index:
-                    self.largest_index = index
-            assert 'value' in self.columns, "Required 'value' column missing from column def"
-            if 'name' not in self.columns:
-                self.columns['name'] = self.columns['value']
+        tool_data_table_name = elem.get( 'from_data_table', None )
+        
+        # Options are defined from a data table loaded by the app
+        self.tool_data_table = None
+        if tool_data_table_name:
+            app = tool_param.tool.app
+            assert tool_data_table_name in app.tool_data_tables, \
+                "Data table named '%s' is required by tool but not configured" % tool_data_table_name
+            self.tool_data_table = app.tool_data_tables[ tool_data_table_name ]
+            # Column definitions are optional, but if provided override those from the table
+            if elem.find( "column" ) is not None:
+                self.parse_column_definitions( elem )
+            else:
+                self.columns = self.tool_data_table.columns
             
+        # Options are defined by parsing tabular text data from an data file
+        # on disk, a dataset, or the value of another parameter
+        elif data_file is not None or dataset_file is not None or from_parameter is not None:
+            self.parse_column_definitions( elem )
             if data_file is not None:
                 data_file = data_file.strip()
                 if not os.path.isabs( data_file ):
@@ -432,6 +437,20 @@ class DynamicOptions( object ):
         # Load Validators
         for validator in elem.findall( 'validator' ):
             self.validators.append( validation.Validator.from_element( self.tool_param, validator ) )
+            
+    def parse_column_definitions( self, elem ):
+        for column_elem in elem.findall( 'column' ):
+            name = column_elem.get( 'name', None )
+            assert name is not None, "Required 'name' attribute missing from column def"
+            index = column_elem.get( 'index', None )
+            assert index is not None, "Required 'index' attribute missing from column def"
+            index = int( index )
+            self.columns[name] = index
+            if index > self.largest_index:
+                self.largest_index = index
+        assert 'value' in self.columns, "Required 'value' column missing from column def"
+        if 'name' not in self.columns:
+            self.columns['name'] = self.columns['value']
     
     def parse_file_fields( self, reader ):
         rval = []
@@ -465,6 +484,8 @@ class DynamicOptions( object ):
             assert dataset is not None, "Required dataset '%s' missing from input" % self.dataset_ref_name
             if not dataset: return [] #no valid dataset in history
             options = self.parse_file_fields( open( dataset.file_name ) )
+        elif self.tool_data_table:
+            options = self.tool_data_table.get_fields()
         else:
             options = list( self.file_fields )
         for filter in self.filters:
@@ -473,7 +494,7 @@ class DynamicOptions( object ):
     
     def get_options( self, trans, other_values ):
         rval = []
-        if self.file_fields is not None or self.dataset_ref_name is not None:
+        if self.file_fields is not None or self.tool_data_table is not None or self.dataset_ref_name is not None:
             options = self.get_fields( trans, other_values )
             for fields in options:
                 rval.append( ( fields[self.columns['name']], fields[self.columns['value']], False ) )
@@ -481,3 +502,15 @@ class DynamicOptions( object ):
             for filter in self.filters:
                 rval = filter.filter_options( rval, trans, other_values )
         return rval
+    
+    def column_spec_to_index( self, column_spec ):
+        """
+        Convert a column specification (as read from the config file), to an
+        index. A column specification can just be a number, a column name, or
+        a column alias.
+        """
+        # Name?
+        if column_spec in self.columns:
+            return self.columns[column_spec]
+        # Int?
+        return int( column_spec )

lib/galaxy/util/__init__.py

@@ -231,13 +231,17 @@ def rst_to_html( s ):
                 log.warn( str )
     return docutils.core.publish_string( s, writer=HTMLFragWriter(), settings_overrides=dict( warning_stream=FakeStream() ) )
 
-def xml_text(root, name):
+def xml_text(root, name=None):
     """Returns the text inside an element"""
-    # Try attribute first
-    val = root.get(name)
-    if val: return val
-    # Then try as element
-    elem = root.find(name)    
+    if name is not None:
+        # Try attribute first
+        val = root.get(name)
+        if val:
+            return val
+        # Then try as element
+        elem = root.find(name)
+    else:
+        elem = root
     if elem is not None and elem.text:
         text = ''.join(elem.text.splitlines())
         return text.strip()

setup.sh

@@ -7,6 +7,7 @@ SAMPLES="
 datatypes_conf.xml.sample
 reports_wsgi.ini.sample
 tool_conf.xml.sample
+tool_data_table_conf.xml.sample
 universe_wsgi.ini.sample
 tool-data/alignseq.loc.sample
 tool-data/annotation_profiler_options.xml.sample

tool_data_table_conf.xml.sample

@@ -0,0 +1,17 @@
+<tables>
+    <!-- Locations of MAF files that have been indexed with bx-python -->    
+    <table name="indexed_maf_files">
+        <columns>name, value, dbkey, species</columns>
+        <file path="tool-data/maf_index.loc" />
+    </table>
+    <!-- Locations of indexes in the BWA mapper format -->
+    <table name="bwa_indexes">
+        <columns>name, value</columns>
+        <file path="tool-data/bwa_index.loc" />
+    </table>
+    <!-- Locations of indexes in the Bowtie mapper format -->
+    <table name="bowtie_indexes">
+        <columns>name, value</columns>
+        <file path="tool-data/bowtie_indices.loc" />
+    </table>
+</tables>

tools/maf/interval2maf.xml

@@ -32,22 +32,24 @@
       </when>
       <when value="cached">
         <param name="mafType" type="select" label="Choose alignments">
-          <options from_file="maf_index.loc">
+          <options from_data_table="indexed_maf_files">
+            <!--
             <column name="name" index="0"/>
             <column name="value" index="1"/>
             <column name="dbkey" index="2"/>
             <column name="species" index="3"/>
-            <filter type="data_meta" ref="input1" key="dbkey" column="2" multiple="True" separator=","/>
+            -->
+            <filter type="data_meta" ref="input1" key="dbkey" column="dbkey" multiple="True" separator=","/>
             <validator type="no_options" message="No alignments are available for the build associated with the selected interval file"/>
           </options>
         </param>
         <param name="species" type="select" display="checkboxes" multiple="true" label="Choose species" help="Select species to be included in the final alignment">
-          <options from_file="maf_index.loc">
+          <options from_data_table="indexed_maf_files">
             <column name="uid" index="1"/>
             <column name="value" index="3"/>
             <column name="name" index="3"/>
-            <filter type="param_value" ref="mafType" name="uid" column="1"/>
-            <filter type="multiple_splitter" column="3" separator=","/>
+            <filter type="param_value" ref="mafType" column="uid"/>
+            <filter type="multiple_splitter" column="name" separator=","/>
           </options>
         </param>
       </when>

tools/sr_mapping/bowtie_wrapper.xml

@@ -192,10 +192,13 @@
       </param>
       <when value="indexed">
         <param name="index" type="select" label="Select a reference genome" help="if your genome of interest is not listed - contact Galaxy team">
+          <options from_data_table="bowtie_indexes"/>
+          <!--
           <options from_file="bowtie_indices.loc">
             <column name="value" index="1" />
             <column name="name" index="0" />
           </options>
+          -->
         </param>
       </when>
       <when value="history">

tools/sr_mapping/bwa_wrapper.xml

@@ -34,10 +34,13 @@
       </param>
       <when value="indexed">
         <param name="indices" type="select" label="Select a reference genome">
+          <options from_data_table="bwa_indexes"/>
+          <!--
           <options from_file="bwa_index.loc">
             <column name="value" index="1" />
             <column name="name" index="0" />
           </options>
+          -->
         </param>
       </when>
       <when value="history">