From e190e6e2b02fa4b53c3a46b1c392c3ab276df0e9 Mon Sep 17 00:00:00 2001
From: Greg Von Kuster <greg@bx.psu.edu>
Date: Fri, 19 Oct 2007 20:42:11 +0000
Subject: [PATCH] Enhancements to the way dynamic_options work in order to
 enable tools using dynemic_options to function in workdflow. We're still
 supporting the current dynamic_options functionality.  This is a first pass
 and will be cleaned up as we continue to evolve more tools to use this new
 approach. Within tool configs, SelectToolParameter types now include a
 <select_options> tag set.  Several tools have been altered to use this new
 approach.  Other tools we enhanced to use the ColumnListParameter type.

---
 lib/galaxy/tools/actions/__init__.py          |  14 +-
 lib/galaxy/tools/dynamic_options.py           | 155 ++++++++++++++++++
 lib/galaxy/tools/parameters.py                |  87 +++++-----
 .../encode_import_all_latest_datasets.xml     |  18 +-
 ...ncode_import_chromatin_and_chromosomes.xml |  19 ++-
 tools/data_source/encode_import_code.py       |  98 +----------
 tools/data_source/encode_import_gencode.xml   |  18 +-
 .../encode_import_genes_and_transcripts.xml   |  18 +-
 ...import_multi-species_sequence_analysis.xml |  18 +-
 ...encode_import_transcription_regulation.xml |  18 +-
 tools/extract/phastOdds/phastOdds_tool.py     |  12 --
 tools/extract/phastOdds/phastOdds_tool.xml    |  31 ++--
 tools/new_operations/subtract_query.xml       |  35 +---
 tools/new_operations/subtract_query_code.py   |  16 --
 .../aggregate_binned_scores_in_intervals.xml  |  54 +++---
 ...gregate_binned_scores_in_intervals_code.py |  13 --
 16 files changed, 334 insertions(+), 290 deletions(-)
 create mode 100644 lib/galaxy/tools/dynamic_options.py
 delete mode 100644 tools/extract/phastOdds/phastOdds_tool.py
 delete mode 100644 tools/new_operations/subtract_query_code.py
 delete mode 100644 tools/stats/aggregate_binned_scores_in_intervals_code.py

diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py
index 9e6a99704af..3169afe09a4 100644
--- a/lib/galaxy/tools/actions/__init__.py
+++ b/lib/galaxy/tools/actions/__init__.py
@@ -1,6 +1,9 @@
 from galaxy.util.bunch import Bunch
 from galaxy.tools.parameters import *
 
+import logging
+log = logging.getLogger( __name__ )
+
 class ToolAction( object ):
     """
     The actions to be taken when a tool is run (after parameters have
@@ -10,9 +13,7 @@ class ToolAction( object ):
         raise TypeError("Abstract method")
     
 class DefaultToolAction( object ):
-    """
-    Default tool action is to run an external command
-    """
+    """Default tool action is to run an external command"""
     
     def collect_input_datasets( self, tool, param_values ):
         """
@@ -102,16 +103,11 @@ class DefaultToolAction( object ):
             # the type should match the input
             if ext == "input":
                 ext = input_ext
-            # FIXME: What does this flush?
-            trans.app.model.flush()
             data = trans.app.model.Dataset(extension=ext)
-            # Commit the dataset immediately so it gets database assigned 
-            # unique id
+            # Commit the dataset immediately so it gets database assigned unique id
             data.flush()
             # Create an empty file immediately
             open( data.file_name, "w" ).close()
-            # FIXME: What does this flush?
-            trans.app.model.flush()
             # This may not be neccesary with the new parent/child associations
             data.designation = name
             # Set the extension / datatype
diff --git a/lib/galaxy/tools/dynamic_options.py b/lib/galaxy/tools/dynamic_options.py
new file mode 100644
index 00000000000..d8d31b9c0db
--- /dev/null
+++ b/lib/galaxy/tools/dynamic_options.py
@@ -0,0 +1,155 @@
+
+import sys, os, logging
+
+log = logging.getLogger(__name__)
+
+class DynamicOptions( object ):
+    """Handles dynamically generated SelectToolParameter options"""
+    def __init__( self, elem  ):
+        self.data_ref = elem.get( 'data_ref', None)
+        self.from_file = elem.get( 'from_file', None )
+        assert self.from_file is not None, "Value for option data file not found"
+        self.func = elem.get( 'func', None )
+        assert self.func is not None, "Value for option generator function not found"
+        self.func_params = elem.findall( 'func_param' )
+    def get_dataset( self, trans, other_values ):
+        # No value indicates a configuration error, the named DataToolParameter must preceed this parameter in the tool config
+        assert self.data_ref in other_values, "Value for associated DataToolParameter not found"
+        # Get the value of the associated DataToolParameter (a dataset)
+        dataset = other_values[ self.data_ref ]
+        if dataset is None or dataset == '':
+            """
+            Both of these values indicate that no dataset is selected.  However, 'None' indicates that the dataset is optional 
+            while '' indicates that it is not. Currently column parameters do not work well with optional datasets.
+            """
+            return None
+        # TODO: this can be eliminated after Dan's script is run.
+        dataset.set_meta()
+        return dataset
+    #TODO: the following functions should be generalized so that they are not specific to
+    #certain tools (e.g., encode).  We may need to standardize data file formats to be able to do this.
+    def load_from_file_for_build( self ):
+        dict = {}
+        for line in open( self.from_file ):
+            if line and not line.startswith( '#' ):
+                try:
+                    fields = line.rstrip('\r\n').split( "\t" )
+                    if not fields[0] in dict: 
+                        dict[ fields[0] ] = []
+                    dict[ fields[0] ].append( (fields[1], fields[2]) )
+                except:
+                    continue
+        return dict
+    def get_options_for_build( self, trans, other_values ):
+        legal_values = set()
+        options = []
+        dataset = self.get_dataset( trans, other_values )
+        if dataset is None:
+            return legal_values, options
+        dict = self.load_from_file_for_build()
+        if dataset.dbkey in dict:
+            for (descript, scorefile) in dict[ dataset.dbkey ]:
+                options.append( (descript, scorefile, False) )
+                legal_values.add( scorefile )
+        return legal_values, options
+    def load_from_file_for_encode( self ):
+        encode_sets= {}
+        legal_values = set()
+        try:
+            for line in open( self.from_file ):
+                if line and not line.startswith( '#' ):
+                    try:
+                        fields = line.rstrip('\r\n').split( "\t" )
+                        encode_group = fields[0]
+                        build = fields[1]
+                        description = fields[2]
+                        uid = fields[3]
+                        path = fields[4]
+                        try: file_type = fields[5]
+                        except: file_type = "bed"
+                        #TODO: will remove this later, when galaxy can handle gff files
+                        if file_type != "bed": continue
+                        #verify that file exists before making it an option
+                        if not os.path.isfile(path):
+                            continue
+                    except:
+                        continue
+                    #check if group is initialized, if not inititalize
+                    try: temp = encode_sets[encode_group]
+                    except: encode_sets[encode_group] = {}
+                    #add data to group in proper build
+                    try:
+                        encode_sets[encode_group][build].append((description, uid, False))
+                        legal_values.add( uid )
+                    except:
+                        encode_sets[encode_group][build]=[]
+                        encode_sets[encode_group][build].append((description, uid, False))
+                        legal_values.add( uid )
+            #Order by description and date, highest date on top and bold
+            for group in encode_sets:
+                for build in encode_sets[ group ]:
+                    ordered_build = []
+                    for description, uid, selected in encode_sets[ group ][ build ]:
+                        item = {}
+                        item['date']=0
+                        item['description'] = ""
+                        item['uid']=uid
+                        item['selected']=selected
+                        item['partitioned']=False
+                        
+                        if description[-21:]=='[gencode_partitioned]':
+                            item['date'] = description[-31:-23]
+                            item['description'] = description[0:-32]
+                            item['partitioned']=True
+                        else:
+                            item['date'] = description[-9:-1]
+                            item['description'] = description[0:-10]
+                            
+                        for i in range(len(ordered_build)):
+                            ordered_description, ordered_uid, ordered_selected, ordered_item = ordered_build[i]
+                            if item['description'] < ordered_item['description']:
+                                ordered_build.insert(i, (description, uid, selected, item) )
+                                break
+                            if item['description'] == ordered_item['description'] and item['partitioned'] == ordered_item['partitioned']:
+                                if int(item['date']) > int(ordered_item['date']):
+                                    ordered_build.insert(i, (description, uid, selected, item) )
+                                    break
+                        else:
+                            ordered_build.append( (description, uid, selected, item) )
+                    
+                    last_desc = None
+                    last_partitioned = None
+                    for i in range(len(ordered_build)) :
+                        description, uid, selected, item = ordered_build[i]
+                        if item['partitioned'] != last_partitioned or last_desc != item['description']:
+                            last_desc = item['description']
+                            description = "<b>"+description+"</b>"
+                        else:
+                            last_desc = item['description']
+                        last_partitioned = item['partitioned']
+                        encode_sets[group][build][i] = (description, uid, selected)        
+        except Exception, exc:
+            #TODO: Fix this...
+            print >>sys.stdout, 'load_from_file_for_encode: initialization error -> %s' % exc
+        return legal_values, encode_sets
+    #return available datasets for group and build, set None option as selected for hg16
+    def get_options_for_encode( self, trans, other_values ):
+        assert len( self.func_params ) == 2, "Values for 'build' and 'encode group' not found"
+        for func_param in self.func_params:
+            if func_param.get( 'name' ).strip() == 'build':
+                build = func_param.get( 'value' ).strip()
+            elif func_param.get( 'name' ).strip() == 'encode_group':
+                encode_group = func_param.get( 'value' ).strip()
+        legal_values = set()
+        options = []
+        legal_values, dict = self.load_from_file_for_encode()
+        if len( dict ) < 1:
+            options.append(('No data available for this build','None',True))
+            legal_values.add( 'None' )
+        else:
+            try: 
+                options = dict[encode_group][build][0:]
+            except:
+                options.append(('No data available for this build','None',True))
+                legal_values.add( 'None' )
+        return legal_values, options
diff --git a/lib/galaxy/tools/parameters.py b/lib/galaxy/tools/parameters.py
index cd8d776e887..829ab97771b 100644
--- a/lib/galaxy/tools/parameters.py
+++ b/lib/galaxy/tools/parameters.py
@@ -5,7 +5,7 @@ Classes encapsulating tool parameters
 import logging, string, sys
 from galaxy import config, datatypes, util
 from galaxy.datatypes.tabular import *
-import validation
+import validation, dynamic_options
 from elementtree.ElementTree import XML, Element
 
 # For BaseURLToolParameter
@@ -298,14 +298,7 @@ class HiddenToolParameter( ToolParameter ):
         ToolParameter.__init__( self, tool, elem )
         self.name = elem.get( 'name' )
         self.value = elem.get( 'value' )
-        self.dynamic_options = elem.get( "dynamic_options", None )
     def get_html_field( self, trans=None, value=None, other_values={} ):
-        if self.dynamic_options:
-            # Add GALAXY_TOOL_PARAMS to locals for backward compatibility
-            locals = dict( other_values )
-            locals['GALAXY_TOOL_PARAMS'] = other_values
-            options = eval( self.dynamic_options, self.tool.code_namespace, locals )
-            self.value = options
         return form_builder.HiddenField( self.name, self.value )
     def get_initial_value( self, trans, context ):
         return self.value
@@ -424,13 +417,26 @@ class SelectToolParameter( ToolParameter ):
                 self.legal_values.add( value )
                 selected = ( option.get( "selected", None ) == "true" )
                 self.options.append( ( option.text, value, selected ) )
+        select_options = elem.find( 'select_options' )
+        if select_options is not None:
+            self.select_options = dynamic_options.DynamicOptions( select_options )
+        else:
+            self.select_options = None
     def get_options( self, trans, other_values ):
-        if self.dynamic_options:
+        if self.select_options:
+            func = '''self.select_options.%s( trans, other_values )''' %self.select_options.func
+            legal_values, options = eval( func )
+            for value in legal_values:
+                self.legal_values.add( value )
+            return options
+        elif self.dynamic_options:
             return eval( self.dynamic_options, self.tool.code_namespace, other_values )
         else:
             return self.options
     def get_legal_values( self, trans, other_values ):
-        if self.dynamic_options:
+        if self.select_options:
+            return self.legal_values
+        elif self.dynamic_options:
             return set( v for _, v, _ in eval( self.dynamic_options, self.tool.code_namespace, other_values ) )
         else:
             return self.legal_values
@@ -480,6 +486,11 @@ class SelectToolParameter( ToolParameter ):
         elif len( value ) == 1:
             value = value[0]
         return value
+    def get_dependencies( self ):
+        try: 
+            if self.select_options.data_ref is None: return []
+            else: return [ self.select_options.data_ref ]
+        except: return []
 
 class GenomeBuildParameter( SelectToolParameter ):
     """
@@ -545,7 +556,7 @@ class ColumnListParameter( SelectToolParameter ):
     >>> dtp =  DataToolParameter( None, XML( '<param name="blah" type="data" format="interval"/>' ) )
     >>> print dtp.name
     blah
-    >>> clp = ColumnListParameter ( None, XML( '<param name="numerical_column" type="columnlist" assoc_dataset="blah" numerical="true"/>' ) )
+    >>> clp = ColumnListParameter ( None, XML( '<param name="numerical_column" type="data_column" data_ref="blah" numerical="true"/>' ) )
     >>> print clp.name
     numerical_column
     """
@@ -553,9 +564,8 @@ class ColumnListParameter( SelectToolParameter ):
         SelectToolParameter.__init__( self, tool, elem )
         self.tool = tool
         self.numerical = str_bool( elem.get( "numerical", False ))
+        self.force_select = str_bool( elem.get( "force_select", True ))
         self.data_ref = elem.get( "data_ref", None )
-        if self.data_ref is None:
-            self.data_ref = elem.get( "assoc_dataset", None )
     def get_column_list( self, trans, other_values ):
         """
         Generate a select list containing the columns of the associated 
@@ -577,6 +587,8 @@ class ColumnListParameter( SelectToolParameter ):
         # Just to be safe... (FIXME: Is this still neccesary?)
         dataset.set_meta()
         # Generate options
+        if not dataset.metadata.columns: 
+            return column_list
         if self.numerical:
             # If numerical was requsted, filter columns based on metadata
             for i, col in enumerate( dataset.metadata.column_types ):
@@ -587,9 +599,17 @@ class ColumnListParameter( SelectToolParameter ):
         return column_list
     def get_options( self, trans, other_values ):
         column_list = self.get_column_list( trans, other_values )
-        return [ ( "c" + col, col, False ) for col in column_list ]
+        options = []
+        if len( column_list ) > 0 and not self.force_select:
+            options.append( ('?', 'None', False) )
+        for col in column_list:
+            options.append( ( "c" + col, col, False ) )
+        return options
     def get_legal_values( self, trans, other_values ):
-        return set( self.get_column_list( trans, other_values ) )
+        legal_values = set( self.get_column_list( trans, other_values ) )
+        if not self.force_select:
+            legal_values.add( 'None' )
+        return legal_values
     def get_dependencies( self ):
         return [ self.data_ref ]
 
@@ -636,8 +656,6 @@ class DataToolParameter( ToolParameter ):
         self.formats = tuple( formats )
         self.multiple = str_bool( elem.get( 'multiple', False ) )
         self.optional = str_bool( elem.get( 'optional', False ) )
-        self.refresh_on_change = str_bool( elem.get( "refresh_on_change", False ))
-        self.dynamic_options = elem.get( "dynamic_options", None )
 
     def get_html_field( self, trans=None, value=None, other_values={} ):
         assert trans is not None, "DataToolParameter requires a trans"
@@ -646,10 +664,6 @@ class DataToolParameter( ToolParameter ):
         if value is not None:
             if type( value ) != list: value = [ value ]
         field = form_builder.SelectField( self.name, self.multiple, None, self.refresh_on_change )
-        if self.dynamic_options:
-            # Dynamic options for a DataToolParameter specify limits on acceptrable build, id, or extension
-            option_build, option_id, option_extension = \
-                eval( self.dynamic_options, self.tool.code_namespace, other_values )
         # CRUCIAL: the dataset_collector function needs to be local to DataToolParameter.get_html_field()
         def dataset_collector( datasets, parent_hid ):
             for i, data in enumerate( datasets ):
@@ -657,18 +671,9 @@ class DataToolParameter( ToolParameter ):
                     hid = "%s.%d" % ( parent_hid, i + 1 )
                 else:
                     hid = str( data.hid )
-                if self.dynamic_options:
-                    if ( isinstance( data.datatype, self.formats )
-                         and (data.dbkey == option_build) and (data.id != option_id) 
-                         and (data.extension in option_extension) 
-                         and not data.deleted 
-                         and data.state not in [data.states.FAKE, data.states.ERROR] ): 
-                        selected = ( value and ( data in value ) )
-                        field.add_option( "%s: %s" % ( hid, data.name[:30] ), data.id, selected )
-                else:
-                    if isinstance( data.datatype, self.formats) and not data.deleted and data.state not in [data.states.FAKE, data.states.ERROR]:
-                        selected = ( value and ( data in value ) )
-                        field.add_option( "%s: %s" % ( hid, data.name[:30] ), data.id, selected )
+                if isinstance( data.datatype, self.formats) and not data.deleted and data.state not in [data.states.FAKE, data.states.ERROR]:
+                    selected = ( value and ( data in value ) )
+                    field.add_option( "%s: %s" % ( hid, data.name[:30] ), data.id, selected )
                 # Also collect children via association object
                 dataset_collector( [ assoc.child for assoc in data.children ], hid )
         dataset_collector( history.datasets, None )
@@ -695,22 +700,11 @@ class DataToolParameter( ToolParameter ):
         if trans is None or trans.history is None:
             return None
         history = trans.history
-        if self.dynamic_options:
-            # Dynamic options for a DataToolParameter specify limits on acceptrable build, id, or extension
-            option_build, option_id, option_extension = \
-                eval( self.dynamic_options, self.tool.code_namespace, other_values )
         most_recent_dataset = [None]
         def dataset_collector( datasets ):
             for i, data in enumerate( datasets ):
-                if self.dynamic_options:
-                    if ( isinstance( data.datatype, self.formats )
-                         and (data.dbkey == option_build) and (data.id != option_id) 
-                         and (data.extension in option_extension) 
-                         and not data.deleted ): 
-                        most_recent_dataset[0] = data
-                else:
-                    if isinstance( data.datatype, self.formats) and not data.deleted:
-                        most_recent_dataset[0] = data
+                if isinstance( data.datatype, self.formats) and not data.deleted and data.state not in [data.states.FAKE, data.states.ERROR]:
+                    most_recent_dataset[0] = data
                 # Also collect children via association object
                 dataset_collector( [ assoc.child for assoc in data.children ] )
         dataset_collector( history.datasets )
@@ -813,7 +807,6 @@ parameter_types = dict( text        = TextToolParameter,
                         boolean     = BooleanToolParameter,
                         genomebuild = GenomeBuildParameter,
                         select      = SelectToolParameter,
-                        columnlist  = ColumnListParameter,
                         data_column = ColumnListParameter,
                         hidden      = HiddenToolParameter,
                         baseurl     = BaseURLToolParameter,
diff --git a/tools/data_source/encode_import_all_latest_datasets.xml b/tools/data_source/encode_import_all_latest_datasets.xml
index df7b643ddf0..76d2772268e 100644
--- a/tools/data_source/encode_import_all_latest_datasets.xml
+++ b/tools/data_source/encode_import_all_latest_datasets.xml
@@ -2,11 +2,21 @@
   <command interpreter="python">encode_import.py $hg17,$hg16 $output</command>
   <inputs>
     <display>
-    <p><div class="toolFormTitle">hg1 (most recent datasets in bold)</div>$hg17</p>
-    <p><div class="toolFormTitle">hg16 (most recent datasets in bold)</div>$hg16</p>
+      <p><div class="toolFormTitle">hg1 (most recent datasets in bold)</div>$hg17</p>
+      <p><div class="toolFormTitle">hg16 (most recent datasets in bold)</div>$hg16</p>
     </display>
-    <param name="hg17" type="select" display="checkboxes" multiple="true" dynamic_options="get_available_data( 'ALD','hg17' )"/>
-    <param name="hg16" type="select" display="checkboxes" multiple="true" dynamic_options="get_available_data( 'ALD','hg16' )"/>
+    <param name="hg17" type="select" display="checkboxes" multiple="true">
+      <select_options from_file="/depot/data2/galaxy/encode_datasets.loc" func="get_options_for_encode">
+        <func_param name="encode_group" value="ALD" />
+        <func_param name="build" value="hg17" />
+      </select_options>
+    </param>
+    <param name="hg16" type="select" display="checkboxes" multiple="true">
+      <select_options from_file="/depot/data2/galaxy/encode_datasets.loc" func="get_options_for_encode">
+        <func_param name="encode_group" value="ALD" />
+        <func_param name="build" value="hg16" />
+      </select_options>
+    </param>
   </inputs>
   <outputs>
     <data format="bed" name="output"/>
diff --git a/tools/data_source/encode_import_chromatin_and_chromosomes.xml b/tools/data_source/encode_import_chromatin_and_chromosomes.xml
index 0d6a200b4ae..a2a14362f91 100644
--- a/tools/data_source/encode_import_chromatin_and_chromosomes.xml
+++ b/tools/data_source/encode_import_chromatin_and_chromosomes.xml
@@ -1,13 +1,22 @@
 <tool id="encode_import_chromatin_and_chromosomes1" name="Chromatin and Chromosomes">
-<!--  <description>ENCODE Data Sets</description> -->
   <command interpreter="python">encode_import.py $hg17,$hg16 $output</command>
   <inputs>
     <display>
-    <p><div class="toolFormTitle">hg17 (most recent datasets in bold)</div>$hg17</p>
-    <p><div class="toolFormTitle">hg16 (most recent datasets in bold)</div>$hg16</p>
+      <p><div class="toolFormTitle">hg17 (most recent datasets in bold)</div>$hg17</p>
+      <p><div class="toolFormTitle">hg16 (most recent datasets in bold)</div>$hg16</p>
     </display>
-    <param name="hg17" type="select" display="checkboxes" multiple="true" dynamic_options="get_available_data( 'CC','hg17' )"/>
-    <param name="hg16" type="select" display="checkboxes" multiple="true" dynamic_options="get_available_data( 'CC','hg16' )"/>
+     <param name="hg17" type="select" display="checkboxes" multiple="true">
+      <select_options from_file="/depot/data2/galaxy/encode_datasets.loc" func="get_options_for_encode">
+        <func_param name="encode_group" value="CC" />
+        <func_param name="build" value="hg17" />
+      </select_options>
+    </param>
+    <param name="hg16" type="select" display="checkboxes" multiple="true">
+      <select_options from_file="/depot/data2/galaxy/encode_datasets.loc" func="get_options_for_encode">
+        <func_param name="encode_group" value="CC" />
+        <func_param name="build" value="hg16" />
+      </select_options>
+    </param>
   </inputs>
   <outputs>
     <data format="bed" name="output"/>
diff --git a/tools/data_source/encode_import_code.py b/tools/data_source/encode_import_code.py
index 647272e6531..b67d6831c08 100644
--- a/tools/data_source/encode_import_code.py
+++ b/tools/data_source/encode_import_code.py
@@ -1,103 +1,9 @@
-#build list of available data
 import os, sys
-encode_sets= {}
 
-try:
-    for line in open( "/depot/data2/galaxy/encode_datasets.loc" ):
-        if line[0:1] == "#" : continue
-        
-        fields = line.split('\t')
-        #read each line, if not enough fields, go to next line
-        try:
-            encode_group = fields[0]
-            build = fields[1]
-            description = fields[2]
-            uid = fields[3]
-            path = fields[4].replace("\n","").replace("\r","")
-            try:
-                file_type = fields[5].replace("\n","").replace("\r","")
-            except:
-                file_type = "bed"
-            #will remove this later, when galaxy can handle gff files
-            if file_type != "bed":
-                continue
-            #verify that file exists before making it an option
-            if not os.path.isfile(path):
-                continue
-        except:
-            continue
-        #check if group is initialized, if not inititalize
-        try:
-            temp = encode_sets[encode_group]
-        except:
-            encode_sets[encode_group] = {}
-        #add data to group in proper build
-        try:
-            encode_sets[encode_group][build].append((description, uid, False))
-        except:
-            encode_sets[encode_group][build]=[]
-            encode_sets[encode_group][build].append((description, uid, False))
-            
-    #Order by description and date, highest date on top and bold
-    for group in encode_sets:
-        for build in encode_sets[group]:
-            ordered_build = []
-            for description, uid, selected in encode_sets[group][build]:
-                item = {}
-                item['date']=0
-                item['description'] = ""
-                item['uid']=uid
-                item['selected']=selected
-                item['partitioned']=False
-                
-                if description[-21:]=='[gencode_partitioned]':
-                    item['date'] = description[-31:-23]
-                    item['description'] = description[0:-32]
-                    item['partitioned']=True
-                else:
-                    item['date'] = description[-9:-1]
-                    item['description'] = description[0:-10]
-                    
-                for i in range(len(ordered_build)):
-                    ordered_description, ordered_uid, ordered_selected, ordered_item = ordered_build[i]
-                    if item['description'] < ordered_item['description']:
-                        ordered_build.insert(i, (description, uid, selected, item) )
-                        break
-                    if item['description'] == ordered_item['description'] and item['partitioned'] == ordered_item['partitioned']:
-                        if int(item['date']) > int(ordered_item['date']):
-                            ordered_build.insert(i, (description, uid, selected, item) )
-                            break
-                else:
-                    ordered_build.append( (description, uid, selected, item) )
-            
-            last_desc = None
-            last_partitioned = None
-            for i in range(len(ordered_build)) :
-                description, uid, selected, item = ordered_build[i]
-                if item['partitioned'] != last_partitioned or last_desc != item['description']:
-                    last_desc = item['description']
-                    description = "<b>"+description+"</b>"
-                else:
-                    last_desc = item['description']
-                last_partitioned = item['partitioned']
-                encode_sets[group][build][i] = (description, uid, selected)
-                    
-except Exception, exc:
-    print >>sys.stdout, 'encode_import_code.py initialization error -> %s' % exc 
-
-#return available datasets for group and build, set None option as selected for hg16
-def get_available_data( encode_group, build ):
-    try:
-        available_options = encode_sets[encode_group][build][0:]
-    except:
-        available_options = []
-    if len(available_options) < 1:
-        available_options.append(('No data available for this build','None',True))
-    return available_options
-
-#post processing, set build for data and add additional data to history
 from galaxy import datatypes, config, jobs 
 from shutil import copyfile
+
+#post processing, set build for data and add additional data to history
 def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr):
     history = out_data.items()[0][1].history
     if history == None:
diff --git a/tools/data_source/encode_import_gencode.xml b/tools/data_source/encode_import_gencode.xml
index d76be0ab8e0..57421d901f7 100644
--- a/tools/data_source/encode_import_gencode.xml
+++ b/tools/data_source/encode_import_gencode.xml
@@ -2,11 +2,21 @@
   <command interpreter="python">encode_import.py $hg17,$hg16 $output</command>
   <inputs>
     <display>
-    <p><div class="toolFormTitle">hg17 (most recent datasets in bold)</div>$hg17</p>
-    <p><div class="toolFormTitle">hg16 (most recent datasets in bold)</div>$hg16</p>
+      <p><div class="toolFormTitle">hg17 (most recent datasets in bold)</div>$hg17</p>
+      <p><div class="toolFormTitle">hg16 (most recent datasets in bold)</div>$hg16</p>
     </display>
-    <param name="hg17" type="select" display="checkboxes" multiple="true" dynamic_options="get_available_data( 'GENCODE','hg17' )"/>
-    <param name="hg16" type="select" display="checkboxes" multiple="true" dynamic_options="get_available_data( 'GENCODE','hg16' )"/>
+    <param name="hg17" type="select" display="checkboxes" multiple="true">
+      <select_options from_file="/depot/data2/galaxy/encode_datasets.loc" func="get_options_for_encode">
+        <func_param name="encode_group" value="GENCODE" />
+        <func_param name="build" value="hg17" />
+      </select_options>
+    </param>
+    <param name="hg16" type="select" display="checkboxes" multiple="true">
+      <select_options from_file="/depot/data2/galaxy/encode_datasets.loc" func="get_options_for_encode">
+        <func_param name="encode_group" value="GENCODE" />
+        <func_param name="build" value="hg16" />
+      </select_options>
+    </param>
   </inputs>
   <outputs>
     <data format="bed" name="output"/>
diff --git a/tools/data_source/encode_import_genes_and_transcripts.xml b/tools/data_source/encode_import_genes_and_transcripts.xml
index 725d44ceb03..54e7845f95b 100644
--- a/tools/data_source/encode_import_genes_and_transcripts.xml
+++ b/tools/data_source/encode_import_genes_and_transcripts.xml
@@ -2,11 +2,21 @@
   <command interpreter="python">encode_import.py $hg17,$hg16 $output</command>
   <inputs>
     <display>
-    <p><div class="toolFormTitle">hg17 (most recent datasets in bold)</div>$hg17</p>
-    <p><div class="toolFormTitle">hg16 (most recent datasets in bold)</div>$hg16</p>
+      <p><div class="toolFormTitle">hg17 (most recent datasets in bold)</div>$hg17</p>
+      <p><div class="toolFormTitle">hg16 (most recent datasets in bold)</div>$hg16</p>
     </display>
-    <param name="hg17" type="select" display="checkboxes" multiple="true" dynamic_options="get_available_data( 'GT','hg17' )"/>
-    <param name="hg16" type="select" display="checkboxes" multiple="true" dynamic_options="get_available_data( 'GT','hg16' )"/>
+    <param name="hg17" type="select" display="checkboxes" multiple="true">
+      <select_options from_file="/depot/data2/galaxy/encode_datasets.loc" func="get_options_for_encode">
+        <func_param name="encode_group" value="GT" />
+        <func_param name="build" value="hg17" />
+      </select_options>
+    </param>
+    <param name="hg16" type="select" display="checkboxes" multiple="true">
+      <select_options from_file="/depot/data2/galaxy/encode_datasets.loc" func="get_options_for_encode">
+        <func_param name="encode_group" value="GT" />
+        <func_param name="build" value="hg16" />
+      </select_options>
+    </param>
   </inputs>
   <outputs>
     <data format="bed" name="output"/>
diff --git a/tools/data_source/encode_import_multi-species_sequence_analysis.xml b/tools/data_source/encode_import_multi-species_sequence_analysis.xml
index 04224ae6824..15145d44fd7 100644
--- a/tools/data_source/encode_import_multi-species_sequence_analysis.xml
+++ b/tools/data_source/encode_import_multi-species_sequence_analysis.xml
@@ -2,11 +2,21 @@
   <command interpreter="python">encode_import.py $hg17,$hg16 $output</command>
   <inputs>
     <display>
-    <p><div class="toolFormTitle">hg17 (most recent datasets in bold)</div>$hg17</p>
-    <p><div class="toolFormTitle">hg16 (most recent datasets in bold)</div>$hg16</p>
+      <p><div class="toolFormTitle">hg17 (most recent datasets in bold)</div>$hg17</p>
+      <p><div class="toolFormTitle">hg16 (most recent datasets in bold)</div>$hg16</p>
     </display>
-    <param name="hg17" type="select" display="checkboxes" multiple="true" dynamic_options="get_available_data( 'MSA','hg17' )"/>
-    <param name="hg16" type="select" display="checkboxes" multiple="true" dynamic_options="get_available_data( 'MSA','hg16' )"/>
+    <param name="hg17" type="select" display="checkboxes" multiple="true">
+      <select_options from_file="/depot/data2/galaxy/encode_datasets.loc" func="get_options_for_encode">
+        <func_param name="encode_group" value="MSA" />
+        <func_param name="build" value="hg17" />
+      </select_options>
+    </param>
+    <param name="hg16" type="select" display="checkboxes" multiple="true">
+      <select_options from_file="/depot/data2/galaxy/encode_datasets.loc" func="get_options_for_encode">
+        <func_param name="encode_group" value="MSA" />
+        <func_param name="build" value="hg16" />
+      </select_options>
+    </param>
   </inputs>
   <outputs>
     <data format="bed" name="output"/>
diff --git a/tools/data_source/encode_import_transcription_regulation.xml b/tools/data_source/encode_import_transcription_regulation.xml
index e346fdbb1a5..dffdd6ec3fc 100644
--- a/tools/data_source/encode_import_transcription_regulation.xml
+++ b/tools/data_source/encode_import_transcription_regulation.xml
@@ -2,11 +2,21 @@
   <command interpreter="python">encode_import.py $hg17,$hg16 $output</command>
   <inputs>
     <display>
-    <p><div class="toolFormTitle">hg17 (most recent datasets in bold)</div>$hg17</p>
-    <p><div class="toolFormTitle">hg16 (most recent datasets in bold)</div>$hg16</p>
+      <p><div class="toolFormTitle">hg17 (most recent datasets in bold)</div>$hg17</p>
+      <p><div class="toolFormTitle">hg16 (most recent datasets in bold)</div>$hg16</p>
     </display>
-    <param name="hg17" type="select" display="checkboxes" multiple="true" dynamic_options="get_available_data( 'TR','hg17' )"/>
-    <param name="hg16" type="select" display="checkboxes" multiple="true" dynamic_options="get_available_data( 'TR','hg16' )"/>
+    <param name="hg17" type="select" display="checkboxes" multiple="true">
+      <select_options from_file="/depot/data2/galaxy/encode_datasets.loc" func="get_options_for_encode">
+        <func_param name="encode_group" value="TR" />
+        <func_param name="build" value="hg17" />
+      </select_options>
+    </param>
+    <param name="hg16" type="select" display="checkboxes" multiple="true">
+      <select_options from_file="/depot/data2/galaxy/encode_datasets.loc" func="get_options_for_encode">
+        <func_param name="encode_group" value="TR" />
+        <func_param name="build" value="hg16" />
+      </select_options>
+    </param>
   </inputs>
   <outputs>
     <data format="bed" name="output"/>
diff --git a/tools/extract/phastOdds/phastOdds_tool.py b/tools/extract/phastOdds/phastOdds_tool.py
deleted file mode 100644
index 049c6c72b27..00000000000
--- a/tools/extract/phastOdds/phastOdds_tool.py
+++ /dev/null
@@ -1,12 +0,0 @@
-scores = {}
-for line in open( '/depot/data2/galaxy/phastOdds.loc' ):
-    fields = line.strip().split( "\t" )
-    if not fields[0] in scores: scores[fields[0]] = []
-    scores[ fields[0] ].append( (fields[1],fields[2]) )
-
-def get_scores_for_build( build ):
-    rval = []
-    if build in scores:
-        for (descript,scorefile) in scores[build]:
-            rval.append( (descript,scorefile, False) )
-    return rval        
diff --git a/tools/extract/phastOdds/phastOdds_tool.xml b/tools/extract/phastOdds/phastOdds_tool.xml
index 7b3084b1b30..b1c720a0866 100644
--- a/tools/extract/phastOdds/phastOdds_tool.xml
+++ b/tools/extract/phastOdds/phastOdds_tool.xml
@@ -1,20 +1,17 @@
 <tool id="phastOdds_for_intervals" name="Compute phastOdds score">
-    <description>for each interval</description>
-    <command interpreter="python2.4">get_scores_galaxy.py $per_col ${score_file}.h5 ${score_file}.mapping.bed $input $output $input_chromCol $input_startCol $input_endCol</command>
-    <code file="phastOdds_tool.py"/>
-    <inputs>
-        <page>
-            <param format="interval" name="input" type="data" label="Interval file"/>
-        </page>
-        <page>
-            <param name="score_file" type="select" label="Available datasets" dynamic_options="get_scores_for_build( input.dbkey )"/>
-			<param name="per_col" type="boolean" label="Standardize" help="Standardizes the score to be per alignment column" checked="yes" truevalue="-p" falsevalue=""/>
-        </page>
-    </inputs>
-    <outputs>
-        <data format="interval" name="output" metadata_source="input"/>
-    </outputs>
-    <help>
+  <description>for each interval</description>
+  <command interpreter="python2.4">get_scores_galaxy.py $per_col ${score_file}.h5 ${score_file}.mapping.bed $input $output $input_chromCol $input_startCol $input_endCol</command>
+  <inputs>
+    <param format="interval" name="input" type="data" label="Interval file"/>
+    <param name="score_file" type="select" label="Available datasets">
+      <select_options from_file="/depot/data2/galaxy/phastOdds.loc" data_ref="input" func="get_options_for_build" />
+    </param>
+	<param name="per_col" type="boolean" label="Standardize" help="Standardizes the score to be per alignment column" checked="yes" truevalue="-p" falsevalue=""/>
+  </inputs>
+  <outputs>
+    <data format="interval" name="output" metadata_source="input"/>
+  </outputs>
+  <help>
     
 .. class:: warningmark
 
@@ -46,5 +43,5 @@ and you choose to compute phastOdds scores, your output will look like this:
 |chrom|start|end|score|
 +-----+-----+---+-----+
 
-    </help>
+  </help>
 </tool>
diff --git a/tools/new_operations/subtract_query.xml b/tools/new_operations/subtract_query.xml
index c4245aef716..f8daf40eb4a 100644
--- a/tools/new_operations/subtract_query.xml
+++ b/tools/new_operations/subtract_query.xml
@@ -2,26 +2,16 @@
   <description>from another query</description>
   <command interpreter="python2.4">subtract_query.py $input1 $input2 $begin_col $end_col $output</command>
   <inputs>
-    <page>
-      <param format="txt" name="input2" type="data" help="Second query">
-        <label>Subtract</label>
-      </param>
-      <param format="txt" name="input1" type="data" help="First query">
-        <label>from</label>
-      </param>
-    </page>
-    <page>
-      <param name="begin_col" label="If both queries are tabular format, restrict subtraction between 'begin column'" type="select" dynamic_options="get_columns( input1, input2 )"/>
-      <param name="end_col" label="and 'end column'" type="select" help="Specifying columns for restricting subtraction is available only for tabular formatted queries" dynamic_options="get_columns( input1, input2 )"/>
-    </page>
+    <param format="txt" name="input2" type="data" label="Subtract" help="Second query" />
+    <param format="txt" name="input1" type="data" label="from" help="First query" />
+    <param name="begin_col" type="data_column" data_ref="input1" force_select="False" label="If both queries are tabular format, restrict subtraction between 'begin column'" />
+    <param name="end_col" type="data_column" data_ref="input1" force_select="False" label="and 'end column'" help="Specifying columns for restricting subtraction is available only for tabular formatted queries" />
   </inputs>
   <outputs>
     <data format="input" name="output" metadata_source="input1" />
   </outputs>
   <tests>
-  	<!--
-  	Subtract 2 non-tabular files with no column restrictions.
-  	-->
+  	<!-- Subtract 2 non-tabular files with no column restrictions. -->
     <test>
       <param name="input1" value="1.txt" />
       <param name="input2" value="2.txt" />
@@ -29,9 +19,7 @@
       <param name="end_col" value="None" />
       <output name="output" file="subtract-query-1.dat" />
     </test>
-  	<!--
-  	Subtract 2 tabular files with no column restrictions.
-  	-->
+  	<!-- Subtract 2 tabular files with no column restrictions. -->
     <test>
       <param name="input1" value="eq-showbeginning.dat" />
       <param name="input2" value="eq-showtail.dat" />
@@ -39,9 +27,7 @@
       <param name="end_col" value="None" />
       <output name="output" file="subtract-query-2.dat" />
     </test>
-  	<!--
-  	Subtract 2 tabular files with column restrictions.
-  	-->
+  	<!-- Subtract 2 tabular files with column restrictions. -->
     <test>
       <param name="input1" value="eq-showbeginning.dat" />
       <param name="input2" value="eq-removebeginning.dat" />
@@ -49,9 +35,7 @@
       <param name="end_col" value="c3" />
       <output name="output" file="subtract-query-3.dat" />
     </test>
-  	<!--
-  	Subtract a non-tabular file from a tabular file with no column restrictions.
-  	-->
+  	<!-- Subtract a non-tabular file from a tabular file with no column restrictions. -->
     <test>
       <param name="input1" value="eq-showbeginning.dat" />
       <param name="input2" value="2.txt" />
@@ -120,6 +104,5 @@ Subtracting the **Second query** from the **First query** (restricting to column
   chr10           7
   chr10           2
 
-</help>
-  <code file="subtract_query_code.py"/>
+  </help>
 </tool>
\ No newline at end of file
diff --git a/tools/new_operations/subtract_query_code.py b/tools/new_operations/subtract_query_code.py
deleted file mode 100644
index ef50f362a34..00000000000
--- a/tools/new_operations/subtract_query_code.py
+++ /dev/null
@@ -1,16 +0,0 @@
-from galaxy.datatypes import *
-
-#return set of columns contained in both input datasets
-def get_columns( input1, input2 ):
-    columns = []
-    """
-    Placing a '?' in the first option will keep 'c1' from being automatically
-    selected if the user does nothing.  Not sure why this is the behavior...
-    """
-    columns.append(('?','None',False))
-    if isinstance(input1.datatype, tabular.Tabular().__class__) and isinstance(input2.datatype, tabular.Tabular().__class__):
-        num_columns = min(input1.metadata.columns, input2.metadata.columns)
-        for col in range(1, num_columns+1):
-            option = "c" + str(col)
-            columns.append((option,str(col),False))
-    return columns
\ No newline at end of file
diff --git a/tools/stats/aggregate_binned_scores_in_intervals.xml b/tools/stats/aggregate_binned_scores_in_intervals.xml
index 40fc325ea6b..fcb3eed00d4 100644
--- a/tools/stats/aggregate_binned_scores_in_intervals.xml
+++ b/tools/stats/aggregate_binned_scores_in_intervals.xml
@@ -1,32 +1,28 @@
 <tool id="aggregate_scores_in_intervals2" description="such as phastCons, GERP, binCons, and others for a set of genomic intervals" name="Aggregate datapoints">
-    <description>Appends the average, min, max of datapoints per interval</description>
-    <command interpreter="python2.4">aggregate_scores_in_intervals.py $datasets $input1 $input1_chromCol $input1_startCol $input1_endCol $out_file1 -b</command>
-    <code file="aggregate_binned_scores_in_intervals_code.py"/>
-    <inputs>
-        <page>
-            <param format="interval" name="input1" type="data" label="Interval file"/>
-        </page>
-        <page>
-            <param name="datasets" type="select" label="Available datasets" dynamic_options="get_scores_for_build( input1.dbkey )" display="radio" multiple="false">
-            </param>
-        </page>
-    </inputs>
-    <outputs>
-        <data format="interval" name="out_file1" metadata_source="input1"/>
-    </outputs>
-    <tests>
-      <test>
-        <param name="input1" value="6.bed" dbkey="hg17" ftype="bed"/>
-        <param name="datasets" value="/depot/data2/galaxy/binned_scores/hg17/phastcons_encode_sep2005_tba" />
-        <output name="out_file1" file="aggregate_binned_scores_in_intervals.out" />
-      </test>
-      <test>
-        <param name="input1" value="9_hg18.bed" dbkey="hg18" ftype="bed"/>
-        <param name="datasets" value="/depot/data2/galaxy/binned_scores/hg18/phastCons17way/ba" />
-        <output name="out_file1" file="aggregate_binned_scores_in_intervals2.out" />
-      </test>
-    </tests>
-    <help>
+  <description>Appends the average, min, max of datapoints per interval</description>
+  <command interpreter="python2.4">aggregate_scores_in_intervals.py $datasets $input1 $input1_chromCol $input1_startCol $input1_endCol $out_file1 -b</command>
+  <inputs>
+    <param format="interval" name="input1" type="data" label="Interval file"/>
+    <param name="datasets" type="select" label="Available datasets" display="radio">
+      <select_options from_file="/depot/data2/galaxy/binned_scores.loc" data_ref="input1" func="get_options_for_build" />
+    </param>
+  </inputs>
+  <outputs>
+    <data format="interval" name="out_file1" metadata_source="input1"/>
+  </outputs>
+  <tests>
+    <test>
+      <param name="input1" value="6.bed" dbkey="hg17" ftype="bed"/>
+      <param name="datasets" value="/depot/data2/galaxy/binned_scores/hg17/phastcons_encode_sep2005_tba" />
+      <output name="out_file1" file="aggregate_binned_scores_in_intervals.out" />
+    </test>
+    <test>
+      <param name="input1" value="9_hg18.bed" dbkey="hg18" ftype="bed"/>
+      <param name="datasets" value="/depot/data2/galaxy/binned_scores/hg18/phastCons17way/ba" />
+      <output name="out_file1" file="aggregate_binned_scores_in_intervals2.out" />
+    </test>
+  </tests>
+  <help>
 
 .. class:: warningmark
 
@@ -81,5 +77,5 @@ where:
 * **min** - minimum phastCons score for each region
 * **max** - maximum phastCons score for each region
 
-    </help>
+  </help>
 </tool>
diff --git a/tools/stats/aggregate_binned_scores_in_intervals_code.py b/tools/stats/aggregate_binned_scores_in_intervals_code.py
deleted file mode 100644
index ddf80c7cf47..00000000000
--- a/tools/stats/aggregate_binned_scores_in_intervals_code.py
+++ /dev/null
@@ -1,13 +0,0 @@
-scores = {}
-for line in open( '/depot/data2/galaxy/binned_scores.loc' ):
-    fields = line.strip().split( "\t" )
-    if not fields[0] in scores: 
-        scores[fields[0]] = []
-    scores[ fields[0] ].append( (fields[1], fields[2]) )
-
-def get_scores_for_build( build ):
-    rval = []
-    if build in scores:
-        for (descript, scorefile) in scores[build]:
-            rval.append( (descript, scorefile, False) )
-    return rval