Many changes related to tool interfaces and jobs (DATABASE CHANGE, READ)

#) An "xy_plot" tool and a new "build_ucsc_custom_track" tool that
   demonstrate the various features added here.

#) Grouping constructs for tool parameters:
   - "repeat" element allows for a set of parameters to be repeated an
     arbitrary number of times
   - "conditional" element allows choosing a set of parameters to display
     based on the value of another parameter
   These constructs can be arbitrarily nested. Their values are structured
   and can be used in hooks, validation, et cetera

#) Support for generating arbitrary config files to pass to a tool

#) Command lines are now full Cheetah templates

#) Better job error reporting, includes tracebacks for internal errors
   preparing the job (database change required, see below!)

#) Preparation of command line, config files, exec_before_job hook moved
   into job execution stage

#) Parameter values now jsonified before storing in database (this is much
   more rigorous that before, and restoring from the database now works
   properly)

DATABASE CHANGE:

alter table job add column traceback text;

lib/galaxy/config.py

@@ -30,6 +30,7 @@ class Configuration( object ):
         self.template_path = resolve_path( kwargs.get( "template_path", "templates" ), self.root )
         self.template_cache = resolve_path( kwargs.get( "template_cache_path", "database/compiled_templates" ), self.root )
         self.job_queue_workers = int( kwargs.get( "job_queue_workers", "10" ) )
+        self.job_working_directory = resolve_path( kwargs.get( "job_working_directory", "database/job_working_directory" ), self.root )
         self.admin_pass = kwargs.get('admin_pass',"galaxy")
         self.sendmail_path = kwargs.get('sendmail_path',"/usr/sbin/sendmail")
         self.mailing_join_addr = kwargs.get('mailing_join_addr',"galaxy-user-join@bx.psu.edu")
@@ -47,7 +48,7 @@ class Configuration( object ):
         return self.config_dict.get( key, default )
     def check( self ):
         # Check that required directories exist
-        for path in self.root, self.file_path, self.tool_path, self.template_path:
+        for path in self.root, self.file_path, self.tool_path, self.template_path, self.job_working_directory:
             if not os.path.isdir( path ):
                 raise ConfigurationError("Directory does not exist: %s" % path )
         # Check that required files exist

lib/galaxy/form_builder.py

@@ -7,7 +7,7 @@ import logging, util
 log = logging.getLogger(__name__)
 
 class BaseField(object):
-    def get_html( self ):
+    def get_html( self, prefix="" ):
         """Returns the html widget corresponding to the parameter"""
         raise TypeError( "Abstract Method" )
         
@@ -24,18 +24,18 @@ class TextField(BaseField):
         self.name = name
         self.size = int( size or 10 )
         self.value = value or ""
-    def get_html( self ):
-        return '<input type="text" name="%s" size="%d" value="%s">' \
-            % ( self.name, self.size, self.value )
+    def get_html( self, prefix="" ):
+        return '<input type="text" name="%s%s" size="%d" value="%s">' \
+            % ( prefix, self.name, self.size, self.value )
 
 class TextArea(BaseField):
     """
     A standard text area box.
     
     >>> print TextArea( "foo" ).get_html()
-    <textarea name="foo" rows="5" cols="25"></TEXTAREA>
+    <textarea name="foo" rows="5" cols="25"></textarea>
     >>> print TextArea( "bins", size="4x5", value="default" ).get_html()
-    <textarea name="bins" rows="4" cols="5">default</TEXTAREA>
+    <textarea name="bins" rows="4" cols="5">default</textarea>
     """
     def __init__( self, name, size="5x25", value=None ):
         self.name = name
@@ -43,9 +43,9 @@ class TextArea(BaseField):
         self.rows = int(self.size[0])
         self.cols = int(self.size[-1])
         self.value = value or ""
-    def get_html( self ):
-        return '<textarea name="%s" rows="%d" cols="%d">%s</TEXTAREA>' \
-            % (self.name, self.rows, self.cols, self.value)
+    def get_html( self, prefix="" ):
+        return '<textarea name="%s%s" rows="%d" cols="%d">%s</textarea>' \
+            % ( prefix, self.name, self.rows, self.cols, self.value )
 
 class CheckboxField(BaseField):
     """
@@ -60,10 +60,11 @@ class CheckboxField(BaseField):
         if checked is None: checked = False
         self.name = name
         self.checked = ( checked in ( True, "yes", "true", "on" ) )
-    def get_html( self ):
+    def get_html( self, prefix="" ):
         if self.checked: checked_text = " checked"
         else: checked_text = ""
-        return '<input type="checkbox" name="%s" value="true"%s><input type="hidden" name="%s" value="true">' % ( self.name, checked_text, self.name )
+        return '<input type="checkbox" name="%s%s" value="true"%s><input type="hidden" name="%s" value="true">' \
+            % ( prefix, self.name, checked_text, self.name )
     @staticmethod
     def is_checked( value ):
         if type( value ) == list and len( value ) == 2:
@@ -80,8 +81,8 @@ class FileField(BaseField):
     """
     def __init__( self, name ):
         self.name = name
-    def get_html( self ):
-        return '<input type="file" name="%s">' % self.name
+    def get_html( self, prefix="" ):
+        return '<input type="file" name="%s%s">' % ( prefix, self.name )
 
 class HiddenField(BaseField):
     """
@@ -93,8 +94,8 @@ class HiddenField(BaseField):
     def __init__( self, name, value=None ):
         self.name = name
         self.value = value or ""
-    def get_html( self ):
-        return '<input type="hidden" name="%s" value="%s">' % ( self.name, self.value )
+    def get_html( self, prefix="" ):
+        return '<input type="hidden" name="%s%s" value="%s">' % ( prefix, self.name, self.value )
 
 class SelectField(BaseField):
     """
@@ -132,7 +133,7 @@ class SelectField(BaseField):
     <div><input type="checkbox" name="bar" value="3">automatic</div>
     <div><input type="checkbox" name="bar" value="4" checked>bazooty</div>
     """
-    def __init__( self, name, multiple=None, display=None ):
+    def __init__( self, name, multiple=None, display=None, refresh_on_change=False ):
         self.name = name
         self.multiple = multiple or False
         self.options = list()
@@ -143,16 +144,17 @@ class SelectField(BaseField):
         elif display is not None:
             raise Exception, "Unknown display type: %s" % display
         self.display = display
+        self.refresh_on_change = refresh_on_change
     def add_option( self, text, value, selected = False ):
         self.options.append( ( text, value, selected ) )
-    def get_html( self ):
+    def get_html( self, prefix="" ):
         if self.display == "checkboxes":
-            return self.get_html_checkboxes()
+            return self.get_html_checkboxes( prefix )
         elif self.display == "radio":
-            return self.get_html_radio()
+            return self.get_html_radio( prefix)
         else:
-            return self.get_html_default()
-    def get_html_checkboxes( self ):
+            return self.get_html_default( prefix )
+    def get_html_checkboxes( self, prefix="" ):
         rval = []
         ctr = 0
         for text, value, selected in self.options:
@@ -160,12 +162,12 @@ class SelectField(BaseField):
             if len(self.options) > 2 and ctr % 2 == 1:
                 style = " class=\"odd_row\""
             if selected:
-                rval.append( '<div%s><input type="checkbox" name="%s" value="%s" checked>%s</div>' % ( style, self.name, value, text) )
+                rval.append( '<div%s><input type="checkbox" name="%s%s" value="%s" checked>%s</div>' % ( style, prefix, self.name, value, text) )
             else:
-                rval.append( '<div%s><input type="checkbox" name="%s" value="%s">%s</div>' % ( style, self.name, value, text) )
+                rval.append( '<div%s><input type="checkbox" name="%s%s" value="%s">%s</div>' % ( style, prefix, self.name, value, text) )
             ctr += 1
         return "\n".join( rval )
-    def get_html_radio( self ):
+    def get_html_radio( self, prefix="" ):
         rval = []
         ctr = 0
         for text, value, selected in self.options:
@@ -173,15 +175,18 @@ class SelectField(BaseField):
             if len(self.options) > 2 and ctr % 2 == 1:
                 style = " class=\"odd_row\""
             if selected:
-                rval.append( '<div%s><input type="radio" name="%s" value="%s" checked>%s</div>' % ( style, self.name, value, text) )
+                rval.append( '<div%s><input type="radio" name="%s%s" value="%s" checked>%s</div>' % ( style, prefix, self.name, value, text) )
             else:
-                rval.append( '<div%s><input type="radio" name="%s" value="%s">%s</div>' % ( style, self.name, value, text) )
+                rval.append( '<div%s><input type="radio" name="%s%s" value="%s">%s</div>' % ( style, prefix, self.name, value, text) )
             ctr += 1
         return "\n".join( rval )    
-    def get_html_default( self ):
+    def get_html_default( self, prefix="" ):
         if self.multiple: multiple = " multiple"
         else: multiple = ""
-        rval = [ '<select name="%s"%s>' % ( self.name, multiple ) ]
+        if self.refresh_on_change:
+            rval = [ '<select name="%s%s"%s onchange="document.forms[0].submit();">' % ( prefix, self.name, multiple ) ]
+        else:
+            rval = [ '<select name="%s%s"%s>' % ( prefix, self.name, multiple ) ]
         for text, value, selected in self.options:
             if selected: selected_text = " selected"
             else: selected_text = ""

lib/galaxy/jobs/__init__.py

@@ -1,4 +1,4 @@
-import logging, threading, sys, os, time, subprocess, string, tempfile, re
+import logging, threading, sys, os, time, subprocess, string, tempfile, re, traceback
 
 from galaxy import util, model
 from galaxy.model import mapping
@@ -141,8 +141,54 @@ class JobWrapper( object ):
         self.tool = tool
         self.queue = queue
         self.app = queue.app
+        self.extra_filenames = []
         
-    def fail( self, message ):
+    def get_param_dict( self ):
+        """
+        Restore the dictionary of parameters from the database.
+        """
+        job = model.Job.get( self.job_id )
+        param_dict = dict( [ ( p.name, p.value ) for p in job.parameters ] )
+        param_dict = self.tool.params_from_strings( param_dict, self.app )
+        return param_dict
+        
+    def prepare( self ):
+        """
+        Prepare the job to run by creating the working directory and the
+        config files.
+        """
+        # Create the working directory
+        self.working_directory = \
+            os.path.join( self.app.config.job_working_directory, str( self.job_id ) )
+        os.mkdir( self.working_directory )
+        # Restore parameters from the database
+        job = model.Job.get( self.job_id )
+        incoming = dict( [ ( p.name, p.value ) for p in job.parameters ] )
+        incoming = self.tool.params_from_strings( incoming, self.app )
+        # Resore input / output data lists
+        inp_data = dict( [ ( da.name, da.dataset ) for da in job.input_datasets ] )
+        out_data = dict( [ ( da.name, da.dataset ) for da in job.output_datasets ] )
+        # Build params, done before hook so hook can use
+        param_dict = self.tool.build_param_dict( incoming, inp_data, out_data )
+        # Run the before queue ("exec_before_job") hook
+        self.tool.call_hook( 'exec_before_job', trans=None, inp_data=inp_data, 
+                             out_data=out_data, tool=self.tool, param_dict=incoming )
+        mapping.context.current.flush()
+        # Build any required config files
+        config_filenames = self.tool.build_config_files( param_dict, self.working_directory )
+        # FIXME: Build the param file (might return None, DEPRECATED)
+        param_filename = self.tool.build_param_file( param_dict, self.working_directory )
+        # Build the job's command line
+        self.command_line = self.tool.build_command_line( param_dict )
+        # Return list of all extra files
+        extra_filenames = config_filenames
+        if param_filename is not None:
+            extra_filenames.append( param_filename )
+        self.param_dict = param_dict
+        self.extra_filenames = extra_filenames
+        return extra_filenames
+        
+    def fail( self, message, exception=False ):
         """
         Indicate job failure by setting state and message on all output 
         datasets.
@@ -154,10 +200,15 @@ class JobWrapper( object ):
             dataset.refresh()
             dataset.state = dataset.states.ERROR
             dataset.blurb = 'tool error'
-            dataset.info = "ERROR: " + message
+            dataset.info = message
             dataset.flush()
         job.state = model.Job.states.ERROR
+        # If the failure is due to a Galaxy framework exception, save 
+        # the traceback
+        if exception:
+            job.traceback = traceback.format_exc()
         job.flush()
+        self.cleanup()
         
     def change_state( self, state ):
         job = model.Job.get( self.job_id )
@@ -219,12 +270,10 @@ class JobWrapper( object ):
         inp_data = dict( [ ( da.name, da.dataset ) for da in job.input_datasets ] )
         out_data = dict( [ ( da.name, da.dataset ) for da in job.output_datasets ] )
         param_dict = dict( [ ( p.name, p.value ) for p in job.parameters ] )
+        param_dict = self.tool.params_from_strings( param_dict, self.app )
         self.tool.call_hook( 'exec_after_process', self.queue.app, inp_data=inp_data, 
                              out_data=out_data, param_dict=param_dict, 
                              tool=self.tool, stdout=stdout, stderr=stderr )
-        # remove temporary file
-        if job.param_filename: 
-            os.remove( job.param_filename )
         # remove 'fake' datasets 
         for dataset_assoc in job.input_datasets:
             data = dataset_assoc.dataset
@@ -235,10 +284,16 @@ class JobWrapper( object ):
                         
         mapping.context.current.flush()
         log.debug('job ended, id: %d' % self.job_id )
+        self.cleanup()
+        
+    def cleanup( self ):
+        # remove temporary files
+        for fname in self.extra_filenames: 
+            os.remove( fname )
+        os.rmdir( self.working_directory ) 
         
     def get_command_line( self ):
-        job = model.Job.get( self.job_id )
-        return job.command_line
+        return self.command_line
     
     def get_session_id( self ):
         job = model.Job.get( self.job_id )

lib/galaxy/jobs/runners/local.py

@@ -32,7 +32,16 @@ class LocalJobRunner( object ):
             if job_wrapper is self.STOP_SIGNAL:
                 return
             job_wrapper.change_state( 'running' )
-            command_line = job_wrapper.get_command_line()
+            stderr = stdout = command_line = ''
+            # Prepare the job to run
+            try:
+                job_wrapper.prepare()
+                command_line = job_wrapper.get_command_line()
+            except:
+                job_wrapper.fail( "failure preparing job", exception=True )
+                log.exception( "failure running job id: %d" % job_wrapper.job_id  )
+                continue
+            # If we were able to get a command line, run the job
             if command_line:
                 try:
                     log.debug( 'executing: %s' % command_line )
@@ -46,12 +55,12 @@ class LocalJobRunner( object ):
                     proc.stderr.close()
                     log.debug('execution finished: %s' % command_line)
                 except Exception, e:
-                    job_wrapper.fail( "failure running job" )
+                    job_wrapper.fail( "failure running job", exception=True )
                     log.exception( "failure running job id: %d" % job_wrapper.job_id  )
-            else:
-                stderr = stdout = ''
+                    continue
+            # Finish the job                
             job_wrapper.finish( stdout, stderr )
-            
+
     def put( self, job_wrapper ):
         """Add a job to the queue (by job identifier)"""
         self.queue.put( job_wrapper )
@@ -61,4 +70,4 @@ class LocalJobRunner( object ):
         log.info( "sending stop signal to worker threads" )
         for i in range( len( self.threads ) ):
             self.queue.put( self.STOP_SIGNAL )
-        log.info( "local job runner stopped" )
+        log.info( "local job runner stopped" )

lib/galaxy/jobs/runners/pbs.py

@@ -8,9 +8,12 @@ from Queue import Queue, Empty
 from galaxy import model
 
 import pkg_resources
-pkg_resources.require( "pbs_python" )
-pbs = __import__( "pbs" )
 
+try:
+    pkg_resources.require( "pbs_python" )
+    pbs = __import__( "pbs" )
+except:
+    pbs = None
 
 log = logging.getLogger( __name__ )
 
@@ -43,6 +46,9 @@ class PBSJobRunner( object ):
     STOP_SIGNAL = object()
     def __init__( self, app ):
         """Initialize this job runner and start the monitor thread"""
+        # Check if PBS was importable, fail if not
+        if pbs is None:
+            raise Exception( "PBSJobRunner requires pbs-python which was not found" )
         self.app = app
         # 'watched' and 'queue' are both used to keep track of jobs to watch.
         # 'queue' is used to add new watched jobs, and can be called from
@@ -67,6 +73,7 @@ class PBSJobRunner( object ):
 
     def queue_job( self, job_wrapper ):
         """Create PBS script for a job and submit it to the PBS queue"""
+        job_wrapper.prepare()
         command_line = job_wrapper.get_command_line()
         
         # This is silly, why would we queue a job with no command line?

lib/galaxy/model/mapping.py

@@ -97,6 +97,7 @@ Job.table = Table( "job", metadata,
     Column( "runner_name", String( 255 ) ),
     Column( "stdout", String() ),
     Column( "stderr", String() ),
+    Column( "traceback", String() ),
     Column( "session_id", Integer, ForeignKey( "galaxy_session.id" ), nullable=True ) )
     
 JobParameter.table = Table( "job_parameter", metadata,

lib/galaxy/tools/actions/__init__.py

@@ -0,0 +1,176 @@
+from cookbook.patterns import Bunch
+from galaxy.tools.parameters import *
+
+class ToolAction( object ):
+    """
+    The actions to be taken when a tool is run (after parameters have
+    been converted and validated).
+    """
+    def execute( self, tool, trans, incoming={} ):
+        raise TypeError("Abstract method")
+    
+class DefaultToolAction( object ):
+    """
+    Default tool action is to run an external command
+    """
+    
+    def collect_input_datasets( self, tool, param_values ):
+        """
+        Collect any dataset inputs from incoming. Returns a mapping from 
+        parameter name to Dataset instance for each tool parameter that is
+        of the DataToolParameter type.
+        """
+        input_datasets = dict()
+        def visitor( prefix, input, value ):
+            if isinstance( input, DataToolParameter ):
+                if isinstance( value, list ):
+                    # If there are multiple inputs with the same name, they
+                    # are stored as name1, name2, ...
+                    for i, v in enumerate( value ):
+                        input_datasets[ prefix + input.name + str( i + 1 ) ] = v
+                else:
+                    input_datasets[ prefix + input.name ] = value
+        tool.visit_inputs( param_values, visitor )
+        return input_datasets
+    
+    def execute(self, tool, trans, incoming={} ):
+        out_data   = {}
+        
+        # Collect any input datasets from the incoming parameters
+        inp_data = self.collect_input_datasets( tool, incoming )
+        
+        # Deal with input metadata, 'dbkey', names, and types
+        
+        # FIXME: does this need to modify 'incoming' or should this be 
+        #        moved into 'build_param_dict'? Is this just about getting the
+        #        metadata into the command line? 
+        input_names = []
+        input_ext = 'data'
+        input_dbkey = incoming.get( "dbkey", "?" )
+        input_meta = Bunch()
+        for name, data in inp_data.items():
+            # Hack for fake incoming data
+            if data == None:
+                data = trans.app.model.Dataset()
+                data.state = data.states.FAKE
+            input_names.append( 'data %s' % data.hid )
+            input_ext = data.ext
+            if data.dbkey not in [None, '?']:
+                input_dbkey = data.dbkey
+            for meta_key, meta_value in data.metadata.items():
+                if meta_value is not None:
+                    meta_key = '%s_%s' % (name, meta_key)
+                    incoming[meta_key] = meta_value
+
+        # Build name for output datasets based on tool name and input names
+        output_base_name = tool.name
+        if len( input_names ) == 1:
+            output_base_name += ' on ' + input_names[0]
+        elif len( input_names ) == 2:
+            output_base_name += ' on %s and %s' % tuple(input_names[0:2])
+        elif len( input_names ) == 3:
+            output_base_name += ' on %s, %s, and %s' % tuple(input_names[0:3])
+        elif len( input_names ) > 3:
+            output_base_name += ' on %s, %s, and others' % tuple(input_names[0:2])
+        
+        # Add the dbkey to the incoming parameters
+        incoming[ "dbkey" ] = input_dbkey
+        
+        # Keep track of parent / child relationships, we'll create all the 
+        # datasets first, then create the associations
+        parent_to_child_pairs = []
+        child_dataset_names = set()
+        
+        for name, elems in tool.outputs.items():
+            ( ext, metadata_source, parent ) = elems
+            if parent:
+                parent_to_child_pairs.append( ( parent, name ) )
+                child_dataset_names.add( name )
+            ## What is the following hack for? Need to document under what 
+            ## conditions can the following occur? (james@bx.psu.edu)
+            # HACK: the output data has already been created
+            if name in incoming:
+                dataid = incoming[name]
+                data = trans.app.model.Dataset.get( dataid )
+                assert data != None
+                out_data[name] = data
+                continue 
+            # the type should match the input
+            if ext == "input":
+                ext = input_ext
+            # FIXME: What does this flush?
+            trans.app.model.flush()
+            data = trans.app.model.Dataset()
+            # Commit the dataset immediately so it gets database assigned 
+            # unique id
+            data.flush()
+            # Create an empty file immediately
+            open( data.file_name, "w" ).close()
+            # FIXME: What does this flush?
+            trans.app.model.flush()
+            # This may not be neccesary with the new parent/child associations
+            data.designation = name
+            # Set the extension / datatype
+            # FIXME: Datatypes -- this propertype has a lot of hidden logic
+            data.extension = ext
+            # Copy metadata from one of the inputs if requested. 
+            # FIXME: init_meta should take a dataset to copy from as an 
+            # argument
+            if metadata_source:
+                data.metadata = Bunch( ** inp_data[metadata_source].metadata.__dict__ )
+            else:
+                data.init_meta()
+            # Take dbkey from LAST input
+            data.dbkey = input_dbkey
+            # Default attributes
+            data.state = data.states.QUEUED
+            data.blurb = "queued"
+            data.name  = output_base_name
+            out_data[ name ] = data
+            # Store all changes to database
+            trans.app.model.flush()
+                        
+        # Add all the top-level (non-child) datasets to the history
+        for name in out_data.keys():
+            if name not in child_dataset_names:
+                data = out_data[ name ]
+                trans.history.add_dataset( data )
+                data.flush()
+                
+        # Add all the children to their parents
+        for parent_name, child_name in parent_to_child_pairs:
+            parent_dataset = out_data[ parent_name ]
+            child_dataset = out_data[ child_name ]
+            assoc = trans.app.model.DatasetChildAssociation()
+            assoc.child = child_dataset
+            assoc.designation = child_dataset.designation
+            parent_dataset.children.append( assoc )
+            # FIXME: Child dataset hid
+            
+        # Store data after custom code runs 
+        trans.app.model.flush()
+        
+        # Create the job object
+        job = trans.app.model.Job()
+        job.session_id = trans.get_galaxy_session( create=True ).id
+        if trans.get_history() is not None:
+            job.history_id = trans.get_history().id
+        job.tool_id = tool.id
+        ## job.command_line = command_line
+        ## job.param_filename = param_filename
+        # FIXME: Don't need all of incoming here, just the defined parameters
+        #        from the tool. We need to deal with tools that pass all post
+        #        parameters to the command as a special case.
+        for name, value in tool.params_to_strings( incoming, trans.app ).iteritems():
+            job.add_parameter( name, value )
+        for name, dataset in inp_data.iteritems():
+            job.add_input_dataset( name, dataset )
+        for name, dataset in out_data.iteritems():
+            job.add_output_dataset( name, dataset )
+        trans.app.model.flush()
+        
+        # Queue the job for execution
+        trans.app.job_queue.put( job.id, tool )
+        # IMPORTANT: keep the following event as is - we parse it for our session activity reports
+        trans.log_event( "Added job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id )
+        return out_data

lib/galaxy/tools/grouping.py

@@ -0,0 +1,101 @@
+"""
+Constructs for grouping tool parameters
+"""
+
+from galaxy.tools.parameters import ToolParameter
+
+class Group( object ):
+    def __init__( self ):
+        self.name = None
+        self.inputs = None
+    def value_to_basic( self, value, app ):
+        """
+        Convert value to a (possibly nested) representation using only basic
+        types (dict, list, tuple, str, unicode, int, long, float, bool, None)
+        """
+        return value
+    def value_from_basic( self, value, app ):
+        """
+        Convert a basic representation as produced by `value_to_basic` back
+        into the preferred value form.
+        """
+        return value
+        
+class Repeat( Group ):
+    type = "repeat"
+    def __init__( self ):
+        self.name = None
+        self.title = None
+        self.inputs = None
+    @property
+    def title_plural( self ):
+        if self.title.endswith( "s" ):
+            return self.title
+        else:
+            return self.title + "s"
+    def value_to_basic( self, value, app ):
+        rval = []
+        for d in value:
+            rval_dict = {}
+            for input in self.inputs.itervalues():
+                rval_dict[ input.name ] = input.value_to_basic( d[input.name], app )
+            rval.append( rval_dict )
+        return rval
+    def value_from_basic( self, value, app, ignore_errors=False ):
+        rval = []
+        for d in value:
+            rval_dict = {}
+            for input in self.inputs.itervalues():
+                rval_dict[ input.name ] = input.value_from_basic( d[input.name], app, ignore_errors )
+            rval.append( rval_dict )
+        return rval 
+    def visit_inputs( self, prefix, value, callback ):
+        for i, d in enumerate( value ):
+            for input in self.inputs.itervalues():
+                new_prefix = prefix + "%s_%d|" % ( self.name, i )
+                if isinstance( input, ToolParameter ):
+                    callback( new_prefix, input, d[input.name] )
+                else:
+                    input.visit_inputs( new_prefix, d[input.name], callback )  
+        
+class Conditional( Group ):
+    type = "conditional"
+    def __init__( self ):
+        self.name = None
+        self.test_param = None
+        self.cases = []
+    def get_current_case( self, value, trans ):
+        # Convert value to user representation
+        str_value = self.test_param.filter_value( value, trans )
+        # Find the matching case
+        for index, case in enumerate( self.cases ):
+            if str_value == case.value:
+                return index
+        raise Exception( "No case matched value" )
+    def value_to_basic( self, value, app ):
+        rval = dict()
+        current_case = rval['__current_case__'] = value['__current_case__']
+        rval[ self.test_param.name ] = self.test_param.value_to_basic( value[ self.test_param.name ], app )
+        for input in self.cases[current_case].inputs.itervalues():
+            rval[ input.name ] = input.value_to_basic( value[ input.name ], app )
+        return rval
+    def value_from_basic( self, value, app, ignore_errors=False ):
+        rval = dict()
+        current_case = rval['__current_case__'] = value['__current_case__']
+        rval[ self.test_param.name ] = self.test_param.value_from_basic( value[ self.test_param.name ], app )
+        for input in self.cases[current_case].inputs.itervalues():
+            rval[ input.name ] = input.value_from_basic( value[ input.name ], app, ignore_errors=False )
+        return rval
+    def visit_inputs( self, prefix, value, callback ):
+        current_case = value['__current_case__']
+        new_prefix = prefix + "%s|" % ( self.name )
+        for input in self.cases[current_case].inputs.itervalues():
+            if isinstance( input, ToolParameter ):
+                callback( prefix, input, value[input.name] )
+            else:
+                input.visit_inputs( prefix, value[input.name], callback )
+                         
+class ConditionalWhen( object ):
+    def __init__( self ):
+        self.value = None
+        self.inputs = None

lib/galaxy/tools/parameters.py

@@ -33,12 +33,28 @@ class ToolParameter( object ):
         if self.label: return self.label
         else: return self.name
 
-    def get_html( self, trans=None, value=None, other_values={} ):
+    def get_html_field( self, trans=None, value=None, other_values={} ):
+        raise TypeError( "Abstract Method" )
+
+    def get_html( self, trans=None, value=None, other_values={}):
         """
         Returns the html widget corresponding to the paramter. 
         Optionally attempt to retain the current value specific by 'value'
         """
-        return self.html
+        return self.get_html_field( trans, value, other_values ).get_html()
+        
+    def from_html( self, value, trans=None, other_values={} ):
+        """
+        Convert a value from an HTML POST into the parameters prefered value 
+        format. 
+        """
+        return value
+        
+    def get_initial_value( self, trans, context ):
+        """
+        Return the starting value of the parameter
+        """
+        return None
 
     def get_required_enctype( self ):
         """
@@ -63,6 +79,24 @@ class ToolParameter( object ):
         """Convert a value created with to_string back to an object representation"""
         return value
         
+    def value_to_basic( self, value, app ):
+        return self.to_string( value, app )
+        
+    def value_from_basic( self, value, app, ignore_errors=False ):
+        # HACK: Some things don't deal with unicode well, psycopg problem?
+        if type( value ) == unicode:
+            value = str( value )
+        if ignore_errors:
+            try:
+                return self.to_python( value, app )
+            except:
+                return value
+        else:
+            return self.to_python( value, app )
+        
+    def to_param_dict_string( self, value ):
+        return str( value )
+        
     def validate( self, value, history=None ):
         for validator in self.validators:
             validator.validate( value, history )
@@ -94,10 +128,13 @@ class TextToolParameter( ToolParameter ):
         self.size = elem.get( 'size' )
         self.value = elem.get( 'value' )
         self.area = str_bool( elem.get( 'area', False ) )
-    def get_html( self, trans=None, value=None, other_values={} ):
+    def get_html_field( self, trans=None, value=None, other_values={} ):
         if self.area:
-            return form_builder.TextArea( self.name, self.size, value or self.value ).get_html()
-        return form_builder.TextField( self.name, self.size, value or self.value ).get_html()
+            return form_builder.TextArea( self.name, self.size, value or self.value )
+        else:
+            return form_builder.TextField( self.name, self.size, value or self.value )
+    def get_initial_value( self, trans, context ):
+        return self.value
 
 class IntegerToolParameter( TextToolParameter ):
     """
@@ -108,18 +145,20 @@ class IntegerToolParameter( TextToolParameter ):
     blah
     >>> print p.get_html()
     <input type="text" name="blah" size="4" value="10">
-    >>> type( p.filter_value( "10" ) )
+    >>> type( p.from_html( "10" ) )
     <type 'int'>
-    >>> type( p.filter_value( "bleh" ) )
+    >>> type( p.from_html( "bleh" ) )
     Traceback (most recent call last):
         ...
     ValueError: An integer is required
     """
-    def filter_value( self, value, trans=None, other_values={} ):
+    def from_html( self, value, trans=None, other_values={} ):
         try: return int( value )
         except: raise ValueError( "An integer is required" )
     def to_python( self, value, app ):
         return int( value )
+    def get_initial_value( self, trans, context ):
+        return int( self.value )
             
 class FloatToolParameter( TextToolParameter ):
     """
@@ -130,18 +169,20 @@ class FloatToolParameter( TextToolParameter ):
     blah
     >>> print p.get_html()
     <input type="text" name="blah" size="4" value="3.141592">
-    >>> type( p.filter_value( "36.1" ) )
+    >>> type( p.from_html( "36.1" ) )
     <type 'float'>
-    >>> type( p.filter_value( "bleh" ) )
+    >>> type( p.from_html( "bleh" ) )
     Traceback (most recent call last):
         ...
     ValueError: A real number is required
     """
-    def filter_value( self, value, trans=None, other_values={} ):
+    def from_html( self, value, trans=None, other_values={} ):
         try: return float( value )
-        except: raise ValueError( "A real number is required")
+        except: raise ValueError( "A real number is required" )
     def to_python( self, value, app ):
         return float( value )
+    def get_initial_value( self, trans, context ):
+        return float( self.value )
 
 class BooleanToolParameter( ToolParameter ):
     """
@@ -152,9 +193,13 @@ class BooleanToolParameter( ToolParameter ):
     blah
     >>> print p.get_html()
     <input type="checkbox" name="blah" value="true" checked><input type="hidden" name="blah" value="true">
-    >>> print p.filter_value( ["true","true"] )
+    >>> print p.from_html( ["true","true"] )
+    True
+    >>> print p.to_param_dict_string( True )
     bulletproof vests
-    >>> print p.filter_value( ["true"] )
+    >>> print p.from_html( ["true"] )
+    False
+    >>> print p.to_param_dict_string( False )
     cellophane chests
     """
     def __init__( self, tool, elem ):
@@ -162,19 +207,24 @@ class BooleanToolParameter( ToolParameter ):
         self.truevalue = elem.get( 'truevalue', 'true' )
         self.falsevalue = elem.get( 'falsevalue', 'false' )
         self.name = elem.get( 'name' )
-        self.checked = elem.get( 'checked' )
-    def get_html( self, trans=None, value=None, other_values={} ):
+        self.checked = str_bool( elem.get( 'checked' ) )
+    def get_html_field( self, trans=None, value=None, other_values={} ):
         checked = self.checked
-        if value: checked = form_builder.CheckboxField.is_checked( value )
-        return form_builder.CheckboxField( self.name, checked ).get_html()
-    def filter_value( self, value, trans=None, other_values={} ):
-        if form_builder.CheckboxField.is_checked( value ):
-            return self.truevalue
-        else: 
-            return self.falsevalue
+        if value: 
+            checked = form_builder.CheckboxField.is_checked( value )
+        return form_builder.CheckboxField( self.name, checked )
+    def from_html( self, value, trans=None, other_values={} ):
+        return form_builder.CheckboxField.is_checked( value )  
     def to_python( self, value, app ):
         return ( value == 'True' )
-    
+    def get_initial_value( self, trans, context ):
+        return self.checked
+    def to_param_dict_string( self, value ):
+        if value:
+            return self.truevalue
+        else:
+            return self.falsevalue
+
 class FileToolParameter( ToolParameter ):
     """
     Parameter that takes an uploaded file as a value.
@@ -190,16 +240,26 @@ class FileToolParameter( ToolParameter ):
         Example: C{<param name="bins" type="file" />}
         """
         ToolParameter.__init__( self, tool, elem )
-        self.html = form_builder.FileField( elem.get( 'name') ).get_html()
+        self.name = elem.get( 'name' )
+    def get_html_field( self, trans=None, value=None, other_values={}  ):
+        return form_builder.FileField( self.name )
     def get_required_enctype( self ):
         """
         File upload elements require the multipart/form-data encoding
         """
         return "multipart/form-data"
     def to_string( self, value, app ):
-        raise Exception( "FileToolParameter cannot be persisted" )
+        if value is None:
+            return None
+        else:
+            raise Exception( "FileToolParameter cannot be persisted" )
     def to_python( self, value, app ):
-        raise Exception( "FileToolParameter cannot be persisted" )
+        if value is None:
+            return None
+        else:
+            raise Exception( "FileToolParameter cannot be persisted" )
+    def get_initial_value( self, trans, context ):
+        return None
         
 class HiddenToolParameter( ToolParameter ):
     """
@@ -219,11 +279,16 @@ class HiddenToolParameter( ToolParameter ):
         self.name = elem.get( 'name' )
         self.value = elem.get( 'value' )
         self.dynamic_options = elem.get( "dynamic_options", None )
-    def get_html( self, trans=None, value=None, other_values={} ):
+    def get_html_field( self, trans=None, value=None, other_values={} ):
         if self.dynamic_options:
-            options = eval( self.dynamic_options, self.tool.code_namespace, other_values )
+            # Add GALAXY_TOOL_PARAMS to locals for backward compatibility
+            locals = dict( other_values )
+            locals['GALAXY_TOOL_PARAMS'] = other_values
+            options = eval( self.dynamic_options, self.tool.code_namespace, locals )
             self.value = options
-        return form_builder.HiddenField( self.name, self.value ).get_html()
+        return form_builder.HiddenField( self.name, self.value )
+    def get_initial_value( self, trans, context ):
+        return self.value
     
 ## This is clearly a HACK, parameters should only be used for things the user
 ## can change, there needs to be a different way to specify this. I'm leaving
@@ -238,8 +303,10 @@ class BaseURLToolParameter( ToolParameter ):
         ToolParameter.__init__( self, tool, elem )
         self.name = elem.get( 'name' )
         self.value = elem.get( 'value', '' )
-    def get_html( self, trans=None, value=None, other_values={} ):
-        return form_builder.HiddenField( self.name, trans.request.base + self.value ).get_html()
+    def get_html_field( self, trans=None, value=None, other_values={} ):
+        return form_builder.HiddenField( self.name, trans.request.base + self.value )
+    def get_initial_value( self, trans, context ):
+        return self.value
 
 class SelectToolParameter( ToolParameter ):
     """
@@ -295,7 +362,7 @@ class SelectToolParameter( ToolParameter ):
     <option value="y" selected>I am Y</option>
     <option value="z">I am Z</option>
     </select>
-    >>> print p.filter_value( ["y", "z"] )
+    >>> print p.to_param_dict_string( ["y", "z"] )
     y,z
     
     >>> p = SelectToolParameter( None, XML( 
@@ -316,7 +383,7 @@ class SelectToolParameter( ToolParameter ):
     <div><input type="checkbox" name="blah" value="x" checked>I am X</div>
     <div class="odd_row"><input type="checkbox" name="blah" value="y" checked>I am Y</div>
     <div><input type="checkbox" name="blah" value="z">I am Z</div>
-    >>> print p.filter_value( ["y", "z"] )
+    >>> print p.to_param_dict_string( ["y", "z"] )
     y,z
     """
     def __init__( self, tool, elem):
@@ -343,7 +410,7 @@ class SelectToolParameter( ToolParameter ):
             return set( v for _, v, _ in eval( self.dynamic_options, self.tool.code_namespace, other_values ) )
         else:
             return self.legal_values
-    def get_html( self, trans=None, value=None, other_values={} ):
+    def get_html_field( self, trans=None, value=None, other_values={} ):
         if value is not None:
             if not isinstance( value, list ): value = [ value ]
         field = form_builder.SelectField( self.name, self.multiple, self.display )
@@ -352,8 +419,8 @@ class SelectToolParameter( ToolParameter ):
             if value: 
                 selected = ( optval in value )
             field.add_option( text, optval, selected )
-        return field.get_html()
-    def filter_value( self, value, trans=None, other_values={} ):
+        return field
+    def from_html( self, value, trans=None, other_values={} ):
         legal_values = self.get_legal_values( other_values )
         if isinstance( value, list ):
             if not(self.repeat):
@@ -363,11 +430,28 @@ class SelectToolParameter( ToolParameter ):
                 v = util.restore_text( v )
                 assert v in legal_values 
                 rval.append( v )
-            return self.separator.join( rval )
+            return rval
         else:
             value = util.restore_text( value )
             assert value in legal_values
+            return value    
+    def to_param_dict_string( self, value ):
+        if isinstance( value, list ):
+            if not(self.repeat):
+                assert self.multiple, "Multiple values provided but parameter is not expecting multiple values"
+            return self.separator.join( value )
+        else:
             return value
+    def value_to_basic( self, value, app ):
+        return value
+    def get_initial_value( self, trans, context ):
+        options = self.get_options( trans, context )
+        value = [ optval for _, optval, selected in options if selected ]
+        if len( value ) == 0:
+            value = None
+        elif len( value ) == 1:
+            value = value[0]
+        return value
 
 class GenomeBuildParameter( SelectToolParameter ):
     """
@@ -445,11 +529,16 @@ class DataToolParameter( ToolParameter ):
     """
     def __init__( self, tool, elem ):
         ToolParameter.__init__( self, tool, elem )
-        self.format = datatypes.get_datatype_by_extension( elem.get( 'format', 'data' ).lower() )
+        # Build tuple of classes for supported data formats
+        formats = []
+        extensions = elem.get( 'format', 'data' ).split( "," )
+        for extension in extensions:
+            formats.append( datatypes.get_datatype_by_extension( extension.lower() ).__class__ )
+        self.formats = tuple( formats )
         self.multiple = str_bool( elem.get( 'multiple', False ) )
         self.optional = str_bool( elem.get( 'optional', False ) )
         self.dynamic_options = elem.get( "dynamic_options", None )
-    def get_html( self, trans=None, value=None, other_values={} ):
+    def get_html_field( self, trans=None, value=None, other_values={} ):
         assert trans is not None, "DataToolParameter requires a trans"
         history = trans.history
         assert history is not None, "DataToolParameter requires a history"
@@ -468,30 +557,31 @@ class DataToolParameter( ToolParameter ):
                 else:
                     hid = str( data.hid )
                 if self.dynamic_options:
-                    if ( isinstance( data.datatype, self.format.__class__ )
+                    if ( isinstance( data.datatype, self.formats )
                          and (data.dbkey == option_build) and (data.id != option_id) 
                          and (data.extension in option_extension) 
                          and not data.deleted ): 
                         selected = ( value and ( data in value ) )
                         field.add_option( "%s: %s" % ( hid, data.name[:30] ), data.id, selected )
                 else:
-                    if isinstance( data.datatype, self.format.__class__ ) and not data.deleted:
+                    if isinstance( data.datatype, self.formats) and not data.deleted:
                         selected = ( value and ( data in value ) )
                         field.add_option( "%s: %s" % ( hid, data.name[:30] ), data.id, selected )
                 # Also collect children via association object
                 dataset_collector( [ assoc.child for assoc in data.children ], hid )
         dataset_collector( history.datasets, None )
         some_data = bool( field.options )
-        if some_data and value is None:
-            # Ensure that the last item is always selected
-            a, b, c = field.options[-1]; field.options[-1] = a, b, True
+        if some_data:
+            if value is None:
+                # Ensure that the last item is always selected
+                a, b, c = field.options[-1]; field.options[-1] = a, b, True
         else:
             # HACK: we should just disable the form or something
             field.add_option( "no data has the proper type", '' )                
         if self.optional == True:
             field.add_option( "Selection is Optional", 'None', True )
-        return field.get_html()
-    def filter_value( self, value, trans, other_values={} ):
+        return field
+    def from_html( self, value, trans, other_values={} ):
         if not value:
             raise ValueError( "A data of the appropriate type is required" )
         if value in [None, "None"]:
@@ -502,31 +592,48 @@ class DataToolParameter( ToolParameter ):
             return [ trans.app.model.Dataset.get( v ) for v in value ]
         else:
             return trans.app.model.Dataset.get( value )
-    def to_string( self, value, app ):
+    def value_to_basic( self, value, app ):
+        if value is None:
+            return None
+        if isinstance( value, str ):
+            return value
         return value.id
-    def to_python( self, value, app ):
-        return app.model.Dataset.get( int( value ) )
+    def value_from_basic( self, value, app, ignore_errors=False ):
+        if value is None:
+            return None
+        try:
+            return app.model.Dataset.get( int( value ) )
+        except:
+            if ignore_errors:
+                return value
+            else:
+                raise
+    def to_param_dict_string( self, value ):
+        return value.file_name
 
-class RawToolParameter( ToolParameter ):
-    """
-    Completely nondescript parameter, HTML representation is provided as text
-    contents. 
-    
-    >>> p = RawToolParameter( None, XML( 
-    ... '''
-    ... <param name="blah" type="raw">
-    ... <![CDATA[<span id="$name">Some random stuff</span>]]>
-    ... </param>
-    ... ''' ) )
-    >>> print p.name
-    blah
-    >>> print p.get_html().strip()
-    <span id="blah">Some random stuff</span>
-    """
-    def __init__( self, tool, elem ):
-        ToolParameter.__init__( self, tool, elem )
-        template = string.Template( elem.text )
-        self.html = template.substitute( self.__dict__ )
+# class RawToolParameter( ToolParameter ):
+#     """
+#     Completely nondescript parameter, HTML representation is provided as text
+#     contents. 
+#     
+#     >>> p = RawToolParameter( None, XML( 
+#     ... '''
+#     ... <param name="blah" type="raw">
+#     ... <![CDATA[<span id="$name">Some random stuff</span>]]>
+#     ... </param>
+#     ... ''' ) )
+#     >>> print p.name
+#     blah
+#     >>> print p.get_html().strip()
+#     <span id="blah">Some random stuff</span>
+#     """
+#     def __init__( self, tool, elem ):
+#         ToolParameter.__init__( self, tool, elem )
+#         self.template = string.Template( elem.text )
+#     def get_html( self, prefix="" ):
+#         context = dict( self.__dict__ )
+#         context.update( dict( prefix=prefix ) )
+#         return self.template.substitute( context )
         
 # class HistoryIDParameter( ToolParameter ):
 #     """
@@ -562,19 +669,14 @@ parameter_types = dict( text        = TextToolParameter,
                         hidden      = HiddenToolParameter,
                         baseurl     = BaseURLToolParameter,
                         file        = FileToolParameter,
-                        data        = DataToolParameter,
-                        raw         = RawToolParameter )
+                        data        = DataToolParameter )
 
-def get_suite():
-    """Get unittest suite for this module"""
-    import doctest, sys
-    return doctest.DocTestSuite( sys.modules[__name__] )
 
 def str_bool(in_str):
     """
     returns true/false of a string, since bool(str), always returns true if string is not empty
     default action is to return false
     """
-    if str(in_str).lower() == 'true':
+    if str(in_str).lower() == 'true' or str(in_str).lower() == 'yes':
         return True
     return False

lib/galaxy/tools/test.py

@@ -17,8 +17,9 @@ class ToolTestBuilder( object ):
         self.error = False
         self.exception = None
     def add_param( self, name, value, extra ):
-        if isinstance( self.tool.param_map[name], parameters.DataToolParameter ):
+        # FIXME: This needs to be updated for parameter grouping support
+        if isinstance( self.tool.inputs[name], parameters.DataToolParameter ):
             self.required_files.append( ( value, extra ) )
         self.inputs.append( ( name, value, extra ) )
     def add_output( self, name, file ):
-        self.outputs.append( ( name, file ) )
+        self.outputs.append( ( name, file ) )

lib/galaxy/tools/validation.py

@@ -102,9 +102,48 @@ class InRangeValidator( Validator ):
         if not( self.min <= float( value ) <= self.max ):
             raise ValueError( self.message )   
         
+class LengthValidator( Validator ):
+    """
+    Validator that ensures a number is in a specific range
+
+    >>> from galaxy.tools.parameters import ToolParameter
+    >>> p = ToolParameter.build( None, XML( '''
+    ... <param name="blah" type="text" size="10" value="foobar">
+    ...     <validator type="length" min="2" max="8"/>
+    ... </param>
+    ... ''' ) )
+    >>> t = p.validate( "foo" )
+    >>> t = p.validate( "bar" )
+    >>> t = p.validate( "f" )
+    Traceback (most recent call last):
+        ...
+    ValueError: Must have length of at least 2
+    >>> t = p.validate( "foobarbaz" )
+    Traceback (most recent call last):
+        ...
+    ValueError: Must have length no more than 8
+    """
+    @classmethod
+    def from_element( cls, elem ):
+        return cls( elem.get( 'message', None ), elem.get( 'min', None ), elem.get( 'max', None ) )
+    def __init__( self, message, min, max ):
+        self.message = message
+        if min is not None: 
+            min = int( min )
+        if max is not None:
+            max = int( max )
+        self.min = min
+        self.max = max
+    def validate( self, value, history=None ):
+        if self.min is not None and len( value ) < self.min:
+            raise ValueError( self.message or ( "Must have length of at least %d" % self.min ) )
+        if self.max is not None and len( value ) > self.max:
+            raise ValueError( self.message or ( "Must have length no more than %d" % self.max ) )
+        
 validator_types = dict( expression=ExpressionValidator,
                         regex=RegexValidator,
-                        in_range=InRangeValidator )
+                        in_range=InRangeValidator,
+                        length=LengthValidator )
                         
 def get_suite():
     """Get unittest suite for this module"""

lib/galaxy/util/expressions.py

@@ -0,0 +1,30 @@
+"""
+Expression evaluation support.
+
+For the moment this depends on python's eval. In the future it should be 
+replaced with a "safe" parser.
+"""
+
+from UserDict import DictMixin
+
+class ExpressionContext( object, DictMixin ):
+    def __init__( self, dict, parent=None ):
+        """
+        Create a new expression context that looks for values in the
+        container object 'dict', and falls back to 'parent'
+        """
+        self.dict = dict
+        self.parent = parent
+    def __getitem__( self, key ):
+        if key in self.dict:
+            return self.dict[key]
+        if self.parent is not None and key in self.parent:
+            return self.parent[key]
+        raise KeyError( key )
+    def __contains__( self, key ):
+        if key in self.dict:
+            return True
+        if self.parent is not None and key in self.parent:
+            return True
+        return False
+        

templates/dataset/errors.tmpl

@@ -22,29 +22,41 @@ pre
 
 <div class="toolForm">
   
-  #set job = $dataset.creating_job_associations[0].job
-  
   <h2>Dataset generation errors</h2>
 
   <p><b>Dataset $dataset.hid: $dataset.display_name</b></p>
+  
+  #if $dataset.creating_job_associations
+  
+    #set job = $dataset.creating_job_associations[0].job
+  
+    #if job.traceback
+      The Galaxy framework encountered the following error while attempting
+      to run the tool:
+      
+      <pre>${job.traceback}</pre>
+  
+    #end if
+  
+    #if $job.stderr
+      Tool execution generated the following error message:
+      <pre>${job.stderr}</pre>
+    #else
+      Tool execution did not generate any error messages.
+    #end if
+
+    #if $job.stdout
+      The tool produced the following additional output:
+      <pre>${job.stdout}</pre>
+    #end if
 
-  #if $job.stderr
-    Tool execution generated the following error message:
-    <pre>
-    ${job.stderr}
-    </pre>
   #else
-    Tool execution did not generate any error messages.
-  #end if
-
-  #if $job.stdout
-    The tool produced the following additional output:
-    <pre>
-    ${job.stdout}
-    </pre>
+  
+    The tool did not create any additional job / error info.
+  
   #end if
   
-  <h2>Report this error to Galaxy Team</h2>
+  <h2>Report this error to the Galaxy Team</h2>
   
   <p>The Galaxy team regularly reviews errors that occur in the application. 
   However, if you would like to provide additional information (such as 

templates/history.tmpl

@@ -184,7 +184,7 @@ main();">
                     	<div>Job is currently running</div>
                     	#*<a href="delete?id=$data.id">delete</a> *#
                     #elif $data_state == "error"
-                    	<div>An error occurred running this job: <i>$data.display_info</i>, <a href="${h.url_for( controller='dataset', action='errors', id=$data.id )}" target="galaxy_main">report this error</a></div>
+                    	<div>An error occurred running this job: <i>$data.display_info.strip()</i>, <a href="${h.url_for( controller='dataset', action='errors', id=$data.id )}" target="galaxy_main">report this error</a></div>
                     	#*<a href="delete?id=$data.id">delete</a> *#
                     #elif $data_state == "empty"
                     	<div>No data: <i>$data.display_info</i></div>

templates/tool_form.tmpl

@@ -15,6 +15,56 @@
    <p></p>
 #end if
 
+#def do_inputs( $inputs, $tool_state, $errors, $prefix )
+  #for $input_index, $input in enumerate( $inputs.itervalues() )
+    #if $input.type == "repeat"
+      #if $input_index > 0
+        <tr><td colspan="2"><hr/></td></tr>
+      #end if
+      <tr><td colspan="2"><b>${input.title_plural}</b></td></tr>
+      #set repeat_state = $tool_state[$input.name]
+      #for i in range( len( $repeat_state ) ):
+        #if $input.name in errors
+          #set rep_errors = $errors[$input.name][$i]
+        #else
+          #set rep_errors = dict()
+        #end if
+        <tr><td colspan="2"><b>${input.title} ${i + 1}</b></td></tr>
+        $do_inputs( $input.inputs, $repeat_state[$i], $rep_errors, $prefix + $input.name + "_" + str(i) + "|" )
+        <tr><td></td><td><input type="submit" name="${input.name}_${i}_remove" value="Remove ${input.title} ${i+1}"></td></tr>
+      #end for
+      <tr><td colspan="2">&nbsp;</td></tr>
+      <tr><td></td><td><input type="submit" name="${input.name}_add" value="Add new ${input.title}"></td></tr>
+      <tr><td colspan="2"><hr/></td></tr>
+    #elif $input.type == "conditional"
+      #set group_state = $tool_state[$input.name]
+      #set group_errors = $errors.get( $input.name, {} )
+      #set current_case = $group_state['__current_case__']
+      #set prefix = $prefix + $input.name + "|"
+      $row_for_param( $prefix, $input.test_param, $group_state, $group_errors, refresh=True )
+      $do_inputs( $input.cases[$current_case].inputs, $group_state, $group_errors, $prefix )
+    #else
+      $row_for_param( $prefix, $input, $tool_state, $errors )
+    #end if
+  #end for  
+#end def
+
+#def row_for_param( $prefix, $param, $parent_state, $parent_errors, $refresh=False )
+  <tr valign="top">
+    <td>$param.get_label():</td>
+    <td>
+      #set field = $param.get_html_field( $caller, $parent_state[ $param.name ], $parent_state )
+      #set $field.refresh_on_change = $refresh
+      <div>$field.get_html( $prefix )</div>
+      #if $parent_errors.has_key( $param.name ):
+      <div style="color: red; font-style: italic; padding-top: 1px; padding-bottom: 3px;">$parent_errors[$param.name]</div>
+      #elif $param.help
+      <div class="toolParamHelp">$param.help</div>
+      #end if
+    </td>
+  </tr>
+#end def
+
 <div class="toolForm" id="$tool.id">
   #if $tool.has_multiple_pages 
   <div class="toolFormTitle">$tool.name (step #echo $tool_state.page+1 # of $tool.npages)</div>
@@ -24,7 +74,7 @@
   <div class="toolFormBody">
   <form name="tool_form" action="$tool.action" enctype="$tool.enctype" target="$tool.target" method="$tool.method">
     <input type="hidden" name="tool_id" value="$tool.id">
-    <input type="hidden" name="tool_state" value="$util.object_to_string( $tool_state )">
+    <input type="hidden" name="tool_state" value="$util.object_to_string( $tool_state.encode( $tool, $app ) )">
     
     #if $tool.display_by_page[$tool_state.page]
         
@@ -33,108 +83,23 @@
         
     #else
     	
-		#set $param_list = $tool.param_map_by_page[$tool_state.page].items()
-        
-        ##Clean-up previously generated repeat parameters, which are identified by a # at the beginning.
-        #for $name, $param in $param_list
-        	#if $name.count('#') != 0:
-        		#del $param_list[$param_list.index(($name,$param))]
-        	#end if
-        #end for
-        
-  		#set $new_list = []
-        #for $name, $param in $param_list 
-			#if $param.repeat:
-				#try
-			  		#set $param.repeat = $int($param.repeat)
-			  	#except
-			  		#pass
-			  	#end try
-			  	#if $isinstance($param.repeat, int)
-				  	#set $numloops = $param.repeat
-				#else if $isinstance($param.repeat, str)
-					#if not($param.repeat.startswith("with_")):
-				  		#set $datasets = []
-				  	  	#if $isinstance($param_values.get($param.repeat), list):
-					  		#for $set in $param_values.get($param.repeat):
-			                    $datasets.append("%s" %($set.name))
-			                #end for
-			                #set $numloops = $len($datasets)
-		                #else:
-		                	$datasets.append("%s" %($param_values.get($param.repeat).name))
-		                	#set $numloops = 1
-		                #end if
-					#end if
-				#end if
-                #if $numloops > 1:
-			  		#set $j = 1
-                	#while $j < $numloops:
-                		$new_list.append(("#%s%s" %($j+1,$name), $param))
-                		#set $j = $j + 1
-                    #end while	
-        		#end if
-        	#end if
-		#end for
-		
-		#if $new_list != []:
-			$new_list.sort()
-			$param_list.extend($new_list)
-		#end if
-		
-       
-		#set $i=0
-		<table>
-			#for $name, $param in $param_list
-				#set $other_values = {}
-				#if $param.condition:
-					#try
-						#set $status = $int($param_values.get($param.condition))
-					#except
-						#set $status = 0					
-					#end try
-				#else:
-					#set $status = 1
-				#end if
-				#if $status:			
-					#if $param.repeat and $isinstance($param.repeat, str):
-						#if not($param.repeat.startswith("with_")) and $datasets != []:
-							<tr>
-								<td> Dataset: </td>
-								<td> $datasets[$i]</td>
-							</tr>
-							#set $i = $i + 1
-						#end if
-					#end if	
-					<tr valign="top">
-		              <td>
-		              		#if $param.type != "hidden":
-		              			$param.get_label():
-		              		#end if
-		              </td>
-		              <td>
-		              	#if $param.type != "hidden":
-		                	<div>$param.get_html( $caller, $param_values.get( $param.name, None ), $param_values )</div>
-		                #else:
-		                	<div>$param.get_html( $caller, None, $param_values )</div>
-		                #end if
-		                #if $errors and $errors.has_key( $param.name ):
-		                <div style="color: red; font-style: italic; padding-top: 1px; padding-bottom: 3px;">$errors[$param.name]</div>
-		                #elif $param.help
-		                <div class="toolParamHelp">$param.help</div>
-		                #end if
-		              </td>
-		            </tr>
-		        #end if
-		  	#end for
-        	
-        <tr><td></td><td>
+      <table width="100%">
+          <tr style="display: none;"><td></td><td>
             #if $tool_state.page == $tool.last_page
             <input type="submit" name="runtool_btn" value="Execute">
             #else
             <input type="submit" name="runtool_btn" value="Next step">
             #end if
-        </td></tr>    
-        </table>
+          </td></tr>
+          $do_inputs( $tool.inputs_by_page[ $tool_state.page ], $tool_state.inputs, $errors, "" )
+          <tr><td></td><td>
+            #if $tool_state.page == $tool.last_page
+            <input type="submit" name="runtool_btn" value="Execute">
+            #else
+            <input type="submit" name="runtool_btn" value="Next step">
+            #end if
+          </td></tr>    
+      </table>
     	
     #end if    
 

test/functional/test_toolbox.py

@@ -24,12 +24,12 @@ class ToolTestCase( TwillTestCase ):
         all_inputs = dict( ( name, value ) for ( name, value, _ ) in self.testdef.inputs )
         # Do the first page
         page_inputs = dict( ( key, all_inputs[key] )
-                            for key in self.testdef.tool.param_map_by_page[0].keys() )        
+                            for key in self.testdef.tool.inputs_by_page[0].keys() )        
         self.run_tool( self.testdef.tool.id, **page_inputs )
         # Do other pages if they exist
         for i in range( 1, self.testdef.tool.npages ):
             page_inputs = dict( ( key, all_inputs[key] )
-                                for key in self.testdef.tool.param_map_by_page[i].keys() )
+                                for key in self.testdef.tool.inputs_by_page[i].keys() )
             self.submit_form( **page_inputs )
         # Check the result
         assert len( self.testdef.outputs ) == 1, "ToolTestCase does not deal with multiple outputs properly yet."
@@ -65,4 +65,4 @@ def setup():
                 for k, testdef in enumerate( tool.tests ):
                     name = "%s > %s > %s" % ( section.name, tool.name, testdef.name )
                     testcase = get_testcase( testdef, name )
-                    G[ 'testcase_%d_%d_%d' % ( i, j, k ) ] = testcase
+                    G[ 'testcase_%d_%d_%d' % ( i, j, k ) ] = testcase

tools/filters/axt_to_lav.xml

@@ -1,14 +1,14 @@
 <tool id="axt_to_lav_1" name="AXT to LAV">
 <description>Converts an AXT formated file to LAV format</description>
 <!--  <command interpreter="python2.4">axt_to_lav.py $align_input $dbkey_1 $dbkey_2 $lav_file $seq_file1 $seq_file2</command> -->
-<command interpreter="python2.4">axt_to_lav.py /depot/data2/galaxy/$dbkey_1/seq/%s.nib:$dbkey_1:./static/ucsc/chrom/$dbkey_1.len /depot/data2/galaxy/$dbkey_2/seq/%s.nib:$dbkey_2:./static/ucsc/chrom/$dbkey_2.len $align_input $lav_file $seq_file1 $seq_file2</command>
+<command interpreter="python2.4">axt_to_lav.py /depot/data2/galaxy/$dbkey_1/seq/%s.nib:$dbkey_1:./static/ucsc/chrom/${dbkey_1}.len /depot/data2/galaxy/$dbkey_2/seq/%s.nib:$dbkey_2:./static/ucsc/chrom/${dbkey_2}.len $align_input $lav_file $seq_file1 $seq_file2</command>
 
   <inputs>
       <page>
       <param name="align_input" type="data" format="axt" label="Alignment File" optional="False"/>
       </page>
       <page>
-      <param label="Genome" name="dbkey_1" type="select" dynamic_options="get_available_builds(build=GALAXY_TOOL_PARAMS.align_input.dbkey)"/>
+      <param label="Genome" name="dbkey_1" type="select" dynamic_options="get_available_builds(build=align_input.dbkey)"/>
       <param label="Genome" name="dbkey_2" type="select" dynamic_options="get_available_builds()"/>
       </page>
   </inputs>
@@ -90,4 +90,4 @@ This tool converts an AXT formated file to the LAV format.
     CACAGTCTTCACATTGAGGTACCAAGTTGTGGATCAGAATGGAAAGCTAGGCTATGATGAGGGACAGTGCGCTGTCACA
     </help>
     <code file="axt_to_lav_code.py"/>
-</tool>
+</tool>

tools/plotting/r_wrapper.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+### Run R providing the R script in $1 as standard input and passing 
+### the remaining arguments on the command line
+
+# Function that writes a message to stderr and exits
+function fail
+{
+    echo "$@" >&2
+    exit 1
+}
+
+# Ensure R executable is found
+which R > /dev/null || fail "'R' is required by this tool but was not found on path" 
+
+# Extract first argument
+infile=$1; shift
+
+# Ensure the file exists
+test -f $infile || fail "R input file '$infile' does not exist"
+
+# Invoke R passing file named by first argument to stdin
+R --vanilla --slave $* < $infile

tools/plotting/xy_plot.xml

@@ -0,0 +1,112 @@
+<tool id="XY_Plot_1" name="XY Plot">
+  <description></description>
+  <command interpreter="bash">r_wrapper.sh $script_file</command>
+
+  <inputs>
+    
+    <param name="main" type="text" 
+           value="" size="30" 
+           label="Plot Title"/>
+    <param name="xlab" type="text" 
+           value="" size="30"
+           label="Label for x axis"/>
+    <param name="ylab" type="text" 
+           value="" size="30"
+           label="Label for y axis"/>
+    
+    <repeat name="series" title="Series">
+      <param name="input"
+             type="data" format="tabular"
+             label="Dataset"/>
+      <param name="xcol" type="integer" 
+             value="1" size="30"
+             label="Column for x axis"/>
+      <param name="ycol" type="integer" 
+             value="1" size="30" 
+             label="Column for y axis"/>
+      <conditional name="series_type">
+        <param name="type" type="select" label="Series Type">
+          <option value="line" selected="true">Line</option>
+          <option value="points">Points</option>
+        </param>
+        <when value="line">
+          <param name="lty" type="select" label="Line Type">
+            <option value="1">Solid</option>
+            <option value="2">Dashed</option>
+            <option value="3">Dotted</option>
+          </param>
+          <param name="col" type="select" label="Line Color">
+            <option value="1">Black</option>
+            <option value="2">Red</option>
+            <option value="3">Green</option>
+            <option value="4">Blue</option>
+            <option value="5">Cyan</option>
+            <option value="6">Magenta</option>
+            <option value="7">Yellow</option>
+            <option value="8">Gray</option>
+          </param>
+          <param name="lwd" type="float" label="Line Width" value="1.0"/>
+        </when>
+        <when value="points">
+          <param name="pch" type="select" label="Point Type">
+            <option value="1">Circle (hollow)</option>
+            <option value="2">Triangle (hollow)</option>
+            <option value="3">Cross</option>
+            <option value="4">Diamond (hollow)</option>
+            <option value="15">Square (filled)</option>
+            <option value="16">Circle (filled)</option>
+            <option value="17">Triangle (filled)</option>  
+          </param>
+          <param name="col" type="select" label="Point Color">
+            <option value="1">Black</option>
+            <option value="2">Red</option>
+            <option value="3">Green</option>
+            <option value="4">Blue</option>
+            <option value="5">Cyan</option>
+            <option value="6">Magenta</option>
+            <option value="7">Yellow</option>
+            <option value="8">Gray</option>
+          </param>
+          <param name="cex" type="float" label="Point Scale" value="1.0"/>
+        </when>
+      </conditional>
+    </repeat>       
+  </inputs>
+
+  <configfiles>
+    <configfile name="script_file">
+      ## Setup R error handling to go to stderr
+      options( show.error.messages=F, 
+               error = function () { cat( geterrmessage(), file=stderr() ); q( "no", 1, F ) } )
+      ## Determine range of all series in the plot
+      xrange = c( NULL, NULL )
+      yrange = c( NULL, NULL )
+      #for $i, $s in enumerate( $series )
+        s${i} = read.table( "${s.input.file_name}" )
+        x${i} = s${i}[,${s.xcol}]
+        y${i} = s${i}[,${s.ycol}]
+        xrange = range( x${i}, xrange )
+        yrange = range( y${i}, yrange )
+      #end for
+      ## Open output PDF file
+      pdf( "${out_file1}" )
+      ## Dummy plot for axis / labels
+      plot( NULL, type="n", xlim=xrange, ylim=yrange, main="${main}", xlab="${xlab}", ylab="${ylab}" )
+      ## Plot each series
+      #for $i, $s in enumerate( $series )
+        #if $s.series_type['type'] == "line"
+          lines( x${i}, y${i}, lty=${s.series_type.lty}, lwd=${s.series_type.lwd}, col=${s.series_type.col} )
+        #elif $s.series_type.type == "points"
+          points( x${i}, y${i}, pch=${s.series_type.pch}, cex=${s.series_type.cex}, col=${s.series_type.col} )
+        #end if
+      #end for    
+      ## Close the PDF file
+      devname = dev.off() 
+    </configfile>
+  </configfiles>
+
+  <outputs>
+    <data format="pdf" name="out_file1" />
+  </outputs>
+
+</tool>

tools/visualization/build_ucsc_custom_track.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python2.4
+"""
+Build a UCSC genome browser custom track file
+"""
+
+import sys, os
+
+args = sys.argv[1:]
+
+out_fname = args.pop(0)
+out = open( out_fname, "w" )
+
+num_tracks = 0
+
+while args:
+    # Suck in one dataset worth of arguments
+    in_fname = args.pop(0)
+    type = args.pop(0)
+    colspec = args.pop(0)
+    name = args.pop(0)
+    description = args.pop(0)
+    color = args.pop(0).replace( '-', ',' )
+    visibility = args.pop(0)
+    # Do the work
+    if type == "wig":
+        print >> out, '''track type=wiggle_0 name="%s" description="%s" color=%s visibility=%s''' \
+                      % ( name, description, color, visibility )
+        for line in open( in_fname ):
+            print >> out, line,
+        print >> out
+    elif type == "bed":
+        print >> out, '''track name="%s" description="%s" color=%s visibility=%s''' \
+                      % ( name, description, color, visibility )
+        for line in open( in_fname ):
+            print >> out, line,
+        print >> out
+    else:
+        # Assume type is interval (don't pass this script anything else!)
+        c, s, e, st = map( int, colspec.split( "," ) )
+        
+        print >> out, '''track name="%s" description="%s" color=%s visibility=%s''' \
+                      % ( name, description, color, visibility )
+        i = 0
+        for line in open( in_fname ):
+            if line.startswith( "#" ):
+                continue
+            fields = line.split( "\t" )
+            if st > 0 and st < len( fields ):
+                print >> out, "%s\t%s\t%s\t%d\t0\t%s" % ( fields[c], fields[s], fields[e], i, fields[st] )
+            else:
+                print >> out, "%s\t%s\t%s" % ( fields[c], fields[s], fields[e] )
+            i += 1
+        print >> out
+    num_tracks += 1
+    
+out.close()
+
+print "Generated a custom track containing %d subtracks." % num_tracks
+
+
+

tools/visualization/build_ucsc_custom_track.xml

@@ -0,0 +1,85 @@
+<tool id="build_ucsc_custom_track_1" name="Build custom track">
+  <description>for UCSC genome browser</description>    
+  <command interpreter="python2.4">
+    build_ucsc_custom_track.py 
+      "$out_file1"
+      #for $t in $tracks
+        "${t.input.file_name}"
+        "${t.input.ext}"
+        #if $t.input.ext == "interval"
+          ${t.input.metadata.chromCol},${t.input.metadata.startCol},${t.input.metadata.endCol},${t.input.metadata.strandCol}
+        #else
+          "NA"
+        #end if
+        "${t.name}"
+        "${t.description}"
+        "${t.color}"
+        "${t.visibility}"
+      #end for
+  </command>
+  <inputs>
+    <repeat name="tracks" title="Track">
+      <param name="input" type="data" format="interval,wig" label="Dataset"/>
+      <param name="name" type="text" size="15" value="User Track">
+        <validator type="length" max="15"/>
+      </param>
+      <param name="description" type="text" value="User Supplied Track (from Galaxy)">
+        <validator type="length" max="60"/>
+      </param>
+  		<param label="Color" name="color" type="select">
+  			<option selected="yes" value="0-0-0">Black</option>
+  			<option value="255-0-0">Red</option>
+  			<option value="0-255-0">Green</option>
+  			<option value="0-0-255">Blue</option>
+  			<option value="255-0-255">Magenta</option>
+  			<option value="0-255-255">Cyan</option>
+  			<option value="255-215-0">Gold</option>
+  			<option value="160-32-240">Purple</option>
+  			<option value="255-140-0">Orange</option>
+  			<option value="255-20-147">Pink</option>
+  			<option value="92-51-23">Dark Chocolate</option>
+  			<option value="85-107-47">Olive green</option>
+  		</param>
+  		<param label="Visibility" name="visibility" type="select">
+  			<option selected="yes" value="1">Dense</option>
+  			<option value="2">Full</option>
+  			<option value="3">Pack</option>
+  			<option value="4">Squish</option>
+  			<option value="0">Hide</option>
+  		</param>
+    </repeat>
+  </inputs>
+	<outputs>
+  	<data format="customtrack" name="out_file1" />
+	</outputs>
+  <!--
+  <tests>
+    <test>
+  		<param name="primary" value="customTrack1.bed" />
+  		<param name="primary_color" value="0-0-0" />
+  		<param name="primary_visib" value="1" />
+  		<param name="primary_name" value="customTrack1.bed" />
+  		<param name="newdata" value="customTrack2.bed" />
+  		<param name="status" value="1" />
+  		<param name="Color" value="255-0-0" />
+  		<param name="Visibility" value="2" />
+  		<param name="other_names" value="customTrack2.bed" />
+  		<output name="out_file1" file="customTrack_output.dat" />
+    </test>
+	</tests>
+  -->
+	<help>
+
+**Info**
+
+This tool displays the selected datasets with their custom track attributes (if any) in the UCSC genome browser. 
+
+This tool allows you to set the **Color** and **Visibility** attributes and you can edit the **Name** attribute of the dataset by clicking on **"edit attributes"** button (pencil icon) next to the dataset name in the history panel. 
+
+Please note that the primary dataset in step 1 of the tool sets the database build for the datasets in step 2.
+
+	</help>
+
+  <code file="build_ucsc_custom_track_code.py" />
+  
+</tool>

tools/visualization/build_ucsc_custom_track_code.py

@@ -0,0 +1,17 @@
+# runs after the job (and after the default post-filter)
+
+from sets import Set as set
+
+def validate_input( trans, error_map, param_values, page_param_map ):
+    dbkeys = set()
+    tracks = param_values['tracks']
+    for track in tracks:
+        if track['input'] is not None:
+            dbkeys.add( track['input'].dbkey )
+    if len( dbkeys ) > 1:
+        # FIXME: Should be able to assume error map structure is created
+        if 'tracks' not in error_map:
+            error_map['tracks'] = [ dict() for t in tracks ]
+            for i in range( len( tracks ) ):
+                error_map['tracks'][i]['input'] = \
+                    "All datasets must belong to same genomic build"