Workflow: Repeat/Conditional now work in workflows, adding/removing/refreshing

properly update the 'terminals' for the nodes in the editor. Many other fixes to saving and loading of workflows (error tracking should now be working properly).
2026-09-24 16:30:27 +08:00 · 2008-05-04 20:10:57 +00:00
parent 20d9cd38d3
commit 95ca08bb99
8 changed files with 181 additions and 82 deletions
@@ -689,20 +689,28 @@ class Tool:
                group_errors = []
                group_old_errors = old_errors.get( input.name, None )
                any_group_errors = False
-                # Check any removals before updating state
-                for i in range( len( group_state ) ):                    
-                    if key + "_" + str(i) + "_remove" in incoming:
+                # Check any removals before updating state -- only one
+                # removal can be performed, others will be ignored
+                for i, rep_state in enumerate( group_state ):
+                    rep_index = rep_state['__index__']
+                    if key + "_" + str(rep_index) + "_remove" in incoming:
                        del group_state[i]
+                        if group_old_errors:
+                            del group_old_errors[i]
+                        break
                # Update state
-                for i in range( len( group_state ) ):
-                    prefix = "%s_%d|" % ( key, i )
+                max_index = -1
+                for i, rep_state in enumerate( group_state ):
+                    rep_index = rep_state['__index__']
+                    max_index = max( max_index, rep_index )
+                    prefix = "%s_%d|" % ( key, rep_index )
                    if group_old_errors:
                        rep_old_errors = group_old_errors[i]
                    else:
                        rep_old_errors = {}
                    rep_errors = self.update_state( trans,
                                                    input.inputs, 
-                                                    group_state[i], 
+                                                    rep_state, 
                                                    incoming, 
                                                    prefix=prefix,
                                                    context=context,
@@ -717,6 +725,7 @@ class Tool:
                # Check for addition
                if key + "_add" in incoming:
                    new_state = {}
+                    new_state['__index__'] = max_index + 1
                    self.fill_in_new_state( trans, input.inputs, new_state, context )
                    group_state.append( new_state )
                    if any_group_errors:
@@ -14,7 +14,7 @@ class Group( object ):
        types (dict, list, tuple, str, unicode, int, long, float, bool, None)
        """
        return value
-    def value_from_basic( self, value, app ):
+    def value_from_basic( self, value, app, ignore_errors=False ):
        """
        Convert a basic representation as produced by `value_to_basic` back
        into the preferred value form.
@@ -37,14 +37,21 @@ class Repeat( Group ):
        rval = []
        for d in value:
            rval_dict = {}
+            # Propogate __index__
+            if '__index__' in d:
+                rval_dict['__index__'] = d['__index__']
            for input in self.inputs.itervalues():
                rval_dict[ input.name ] = input.value_to_basic( d[input.name], app )
            rval.append( rval_dict )
        return rval
    def value_from_basic( self, value, app, ignore_errors=False ):
        rval = []
-        for d in value:
+        for i, d in enumerate( value ):
            rval_dict = {}
+            # If the special __index__ key is not set, create it (for backward
+            # compatibility)
+            rval_dict['__index__'] = d.get( '__index__', i )
+            # Restore child inputs
            for input in self.inputs.itervalues():
                rval_dict[ input.name ] = input.value_from_basic( d[input.name], app, ignore_errors )
            rval.append( rval_dict )
@@ -82,9 +89,9 @@ class Conditional( Group ):
    def value_from_basic( self, value, app, ignore_errors=False ):
        rval = dict()
        current_case = rval['__current_case__'] = value['__current_case__']
-        rval[ self.test_param.name ] = self.test_param.value_from_basic( value[ self.test_param.name ], app )
+        rval[ self.test_param.name ] = self.test_param.value_from_basic( value[ self.test_param.name ], app, ignore_errors )
        for input in self.cases[current_case].inputs.itervalues():
-            rval[ input.name ] = input.value_from_basic( value[ input.name ], app, ignore_errors=False )
+            rval[ input.name ] = input.value_from_basic( value[ input.name ], app, ignore_errors )
        return rval
    def visit_inputs( self, prefix, value, callback ):
        current_case = value['__current_case__']
@@ -100,6 +100,8 @@ class WorkflowController( BaseController ):
            rval['tool_errors'] = errors
        else:
            rval['tool_errors'] = None
+        # Updated data_inputs
+        rval['data_inputs'] = get_data_inputs( tool.inputs, state.inputs )
        rval['state'] = state.encode( tool, trans.app )
        return rval
        
@@ -116,11 +118,6 @@ class WorkflowController( BaseController ):
        rval = {}
        rval['name'] = tool.name
        rval['tool_id'] = tool.id
-        data_inputs = []
-        for name, input in tool.inputs.iteritems():
-            if isinstance( input, DataToolParameter ):
-                data_inputs.append( dict( name=input.name, label=input.label, extensions=input.extensions ) )
-        rval['data_inputs'] = data_inputs
        data_outputs = []
        for name, ( format, metadata_source, parent ) in tool.outputs.iteritems():
            data_outputs.append( dict( name=name, extension=format ) )
@@ -129,6 +126,7 @@ class WorkflowController( BaseController ):
        rval['form_html'] = trans.fill_template( "workflow/editor_tool_form.mako", 
            tool=tool, as_html=as_html, values=state.inputs, errors={} )
        rval['tool_state'] = state.encode( tool, trans.app )
+        rval['data_inputs'] = get_data_inputs( tool.inputs, state.inputs )
        return rval
                
    @web.json
@@ -158,8 +156,10 @@ class WorkflowController( BaseController ):
            tool = trans.app.toolbox.tools_by_id[tool_id]
            # Build a state from the tool_inputs dict
            state = DefaultToolState()
-            state.inputs = tool.params_from_strings( step.tool_inputs, trans.app )
+            state.inputs = tool.params_from_strings( step.tool_inputs, trans.app, ignore_errors=True )
            step_dict['tool_state'] = state.encode( tool, trans.app )
+            # Error messages for the tool
+            step_dict['tool_errors'] = ( step.tool_errors or None )
            # Connections
            input_conn_dict = {}
            for conn in step.input_connections:
@@ -169,11 +169,7 @@ class WorkflowController( BaseController ):
            # Position
            step_dict['position'] = step.position
            # Input and output specs
-            data_inputs = []
-            for name, input in tool.inputs.iteritems():
-                if isinstance( input, DataToolParameter ):
-                    data_inputs.append( dict( name=input.name, label=input.label, extensions=input.extensions ) )
-            step_dict['data_inputs'] = data_inputs
+            step_dict['data_inputs'] = get_data_inputs( tool.inputs, state.inputs )
            data_outputs = []
            for name, ( format, metadata_source, parent ) in tool.outputs.iteritems():
                data_outputs.append( dict( name=name, extension=format ) )
@@ -181,7 +177,7 @@ class WorkflowController( BaseController ):
            # Build the tool form html
            errors = step.tool_errors
            step_dict['form_html'] = trans.fill_template( "workflow/editor_tool_form.mako", 
-                tool=tool, as_html=as_html, values=state.inputs, errors={} )
+                tool=tool, as_html=as_html, values=state.inputs, errors=( step.tool_errors or {} ) )
            step_dict['name'] = tool.name
            data['steps'][step.order_index] = step_dict
        return data
@@ -329,6 +325,7 @@ class WorkflowController( BaseController ):
                for other_hid, input_name in associations:
                    if other_hid in hid_to_output_pair:
                        other_job_id, other_name = hid_to_output_pair[ other_hid ]
+                        print "!!!", input_name, other_job_id, other_name, other_hid
                        # Only create association if the associated output dataset
                        # is being included in this workflow
                        if other_job_id in job_id_to_step_index:
@@ -550,8 +547,8 @@ def cleanup_param_values( inputs, values ):
                values[key] = None
                # HACK: Nested associations are not yet working, but we
                #       still need to clean them up so we can serialize
-                if not( prefix ):
-                    associations.append( ( tmp.hid, prefix + key ) )
+                # if not( prefix ):
+                associations.append( ( tmp.hid, prefix + key ) )
                # Cleanup the other deprecated crap associated with datasets
                # as well. Worse, for nested datasets all the metadata is
                # being pushed into the root. FIXME: MUST REMOVE SOON
@@ -561,8 +558,9 @@ def cleanup_param_values( inputs, values ):
                        del root_values[k]            
            elif isinstance( input, Repeat ):
                group_values = values[key]
-                for i in range( len( group_values ) ):
-                    prefix = "%s_%d|" % ( key, i )
+                for i, rep_values in enumerate( group_values ):
+                    rep_index = rep_values['__index__']
+                    prefix = "%s_%d|" % ( key, rep_index )
                    cleanup( prefix, input.inputs, group_values[i] )
            elif isinstance( input, Conditional ):
                group_values = values[input.name]
@@ -570,4 +568,32 @@ def cleanup_param_values( inputs, values ):
                prefix = "%s|" % ( key )
                cleanup( prefix, input.cases[current_case].inputs, group_values )
    cleanup( "", inputs, values )
-    return associations
+    return associations
+
+def get_data_inputs( inputs, input_values ):
+    """
+    For a set of input definitions and a tool state, find all of the dataset
+    inputs
+    """
+    data_inputs = []
+    def visitor( inputs, input_values, name_prefix, label_prefix ):
+        for input in inputs.itervalues():
+            if isinstance( input, Repeat ):  
+                for i, d in enumerate( input_values[ input.name ] ):
+                    index = d['__index__']
+                    new_name_prefix = name_prefix + "%s_%d|" % ( input.name, index )
+                    new_label_prefix = label_prefix + "%s %d > " % ( input.title, i + 1 )
+                    visitor( input.inputs, d, new_name_prefix, new_label_prefix )
+            elif isinstance( input, Conditional ):
+                values = input_values[ input.name ]
+                current = values["__current_case__"]
+                label_prefix = label_prefix
+                name_prefix = name_prefix + "|" + input.name
+                visitor( input.cases[current].inputs, values, name_prefix, label_prefix )
+            else:
+                if isinstance( input, DataToolParameter ):
+                    data_inputs.append( dict( name=name_prefix+input.name, label=label_prefix+input.label, extensions=input.extensions ) )
+    visitor( inputs, input_values, "", "" )
+    return data_inputs
+    
+    
@@ -1,14 +1,19 @@
-function OutputTerminal( element, datatype ) {
+function Terminal( element ) {
    this.element = element;
    this.connectors = [];
-    this.datatype = datatype
 }
-$.extend( OutputTerminal.prototype, {
+Terminal.prototype = {
    connect: function ( connector ) {
        this.connectors.push( connector );
+        if ( this.node ) {
+            this.node.changed();
+        }
    },
    disconnect: function ( connector ) {
        this.connectors.splice( $.inArray( connector, this.connectors ), 1 );
+        if ( this.node ) {
+            this.node.changed();
+        }
    },
    redraw: function () {
        $.each( this.connectors, function( _, c ) {
@@ -20,23 +25,24 @@ $.extend( OutputTerminal.prototype, {
            c.destroy();
        });
    }
-} )
+}
+
+function OutputTerminal( element, datatype ) {
+    Terminal.call( this, element );
+    this.datatype = datatype;
+}
+
+OutputTerminal.prototype.__proto__ = Terminal.prototype;

 function InputTerminal( element, datatypes ) {
-    this.element = element;
-    this.connectors = [];
-    this.max_connections = 1;
+    Terminal.call( this, element );
    this.datatypes = datatypes;
 }
-$.extend( InputTerminal.prototype, {
-    connect: function ( connector ) {
-        this.connectors.push( connector );
-    },
-    disconnect: function ( connector ) {
-        this.connectors.splice( $.inArray( connector, this.connectors ), 1 );
-    },
+
+InputTerminal.prototype = {
+    __proto__: Terminal.prototype,
    can_accept: function ( other ) {
-        if ( this.connectors.length < this.max_connections ) {
+        if ( this.connectors.length < 1 ) {
            for ( t in this.datatypes ) {
                // FIXME: No idea what to do about this case
                if ( other.datatype == "input" ) { return true; }
@@ -46,18 +52,8 @@ $.extend( InputTerminal.prototype, {
            }
        }
        return false;
-    },
-    redraw: function () {
-        $.each( this.connectors, function( _, c ) {
-            c.redraw();  
-        })
-    },
-    destroy: function () {
-        $.each( this.connectors.slice(), function( _, c ) {
-            c.destroy();
-        });
    }
-} );
+}

 function Connector( handle1, handle2 ) {
    this.canvas = null;
@@ -147,7 +143,7 @@ $.extend( Node.prototype, {
        $(elements).each( function() {
            var terminal = this.terminal = new InputTerminal( this, types );
            terminal.node = node;
-            terminal.name = name
+            terminal.name = name;
            $(this).droppable( {
                tolerance: 'intersect',
                accept: function( draggable ) {
@@ -184,7 +180,9 @@ $.extend( Node.prototype, {
                                        $.each( terminal.connectors, function( _, x ) { x.destroy() } );
                                        t.remove();
                                    })))
-                            .bind( "mouseleave", function() { $(this).fadeOut( "fast", function() { $(this).remove() } ) } );
+                            .bind( "mouseleave", function() {
+                                $(this).fadeOut( "fast", function() { $(this).remove() } )
+                            });
                        // Position it and show
                        t.css( {
                                top: $(this).offset().top - 2,
@@ -235,6 +233,10 @@ $.extend( Node.prototype, {
           node.output_terminals[name] = terminal;
       });
    },
+    redraw : function () {
+        $.each( this.input_terminals, function( _, t ) { t.redraw() } );
+        $.each( this.output_terminals, function( _, t ) { t.redraw() } );
+    },
    destroy : function () {
        $.each( this.input_terminals, function( k, t ) {
            t.destroy();
@@ -257,13 +259,20 @@ $.extend( Node.prototype, {
        var f = this.element;
        this.form_html = data.form_html;
        this.tool_state = data.tool_state;
+        this.tool_errors = data.tool_errors;
+        if ( this.tool_errors ) {
+            f.addClass( "tool-node-error" );
+        } else {
+            f.removeClass( "tool-node-error" );
+        }
        var node = this;
-        b = f.find( ".toolFormBody" );
+        var b = f.find( ".toolFormBody" );
        b.find( "div" ).remove();
+        var ibox = $("<div class='inputs'/>").appendTo( b );
        $.each( data.data_inputs, function( i, input ) {
            t = $("<div class='terminal input-terminal'></div>")
            node.enable_input_terminal( t, input.name, input.extensions );
-            b.append( $("<div class='form-row dataRow'>" + input.name + "</div></div>" ).prepend( t ) );
+            ibox.append( $("<div class='form-row dataRow input-data-row' name='" + input.name + "'>" + input.label + "</div></div>" ).prepend( t ) );
        });
        if ( ( data.data_inputs.length > 0 ) && ( data.data_outputs.length > 0 ) ) {
            b.append( $( "<div class='rule'></div>" ) );
@@ -276,18 +285,45 @@ $.extend( Node.prototype, {
        workflow.node_changed( this );
    },
    update_field_data : function( data ) {
+        var el = $(this.element),
+            node = this;
        this.tool_state = data.state;
        this.form_html = data.form_html;
 	this.tool_errors = data.tool_errors;
        if ( this.tool_errors ) {
-                $(this.element).addClass( "tool-node-error" );
+                el.addClass( "tool-node-error" );
        } else {
-                $(this.element).removeClass( "tool-node-error" );
-        }
-        if ( workflow.active_node == this ) {
-            // Reactive with new form_html
-           workflow.activate_node( this );
+                el.removeClass( "tool-node-error" );
        }
+        // Update input rows
+        var old_body = el.find( "div.inputs" );
+        var new_body = $("<div class='inputs'/>");
+        old = old_body.find( "div.input-data-row")
+        $.each( data.data_inputs, function( i, input ) {
+            var t = $("<div class='terminal input-terminal'></div>");
+            node.enable_input_terminal( t, input.name, input.extensions );
+            // If already connected save old connection
+            old_body.find( "div[name=" + input.name + "]" ).each( function() {
+                $(this).find( ".input-terminal" ).each( function() {
+                    var c = this.terminal.connectors[0];
+                    if ( c ) {
+                        t[0].terminal.connectors[0] = c;
+                        c.handle2 = t[0].terminal;
+                    }
+                });
+                $(this).remove();
+            });
+            // Append to new body
+            new_body.append( $("<div class='form-row dataRow input-data-row' name='" + input.name + "'>" + input.label + "</div></div>" ).prepend( t ) );
+        });
+        old_body.replaceWith( new_body );
+        // Cleanup any leftover terminals
+        old_body.find( "div.input-data-row > .terminal" ).each( function() {
+            this.terminal.destroy();
+        })
+        // If active, reactivate with new form_html
+        this.changed();
+        this.redraw();
    },
    error : function ( text ) {
        var b = $(this.element).find( ".toolFormBody" );
@@ -296,6 +332,9 @@ $.extend( Node.prototype, {
        this.form_html = tmp;
        b.html( tmp );
        workflow.node_changed( this );
+    },
+    changed: function() {
+        workflow.node_changed( this );
    }
 } );

@@ -311,6 +350,7 @@ $.extend( Workflow.prototype, {
        this.id_counter++;
        this.nodes[ node.id ] = node;
        this.has_changes = true;
+        node.workflow = this;
    },
    remove_node : function( node ) {
        if ( this.active_node == node ) {
@@ -393,6 +433,7 @@ $.extend( Workflow.prototype, {
        this.active_node = node;
    },
    node_changed : function ( node ) {
+        this.has_changes = true;
        if ( this.active_node == node ) {
            // Reactive with new form_html
            this.activate_node( node );
@@ -45,9 +45,10 @@ jQuery( function() {
            #else
              #set rep_errors = dict()
            #end if
+            #set index = $repeat_state[$i]['__index__']
            <div class="form-title-row"><b>${input.title} ${i + 1}</b></div>
-            $do_inputs( $input.inputs, $repeat_state[$i], $rep_errors, $prefix + $input.name + "_" + str(i) + "|", $context )
-            <div class="form-row"><input type="submit" name="${prefix}${input.name}_${i}_remove" value="Remove ${input.title} ${i+1}"></div>
+            $do_inputs( $input.inputs, $repeat_state[$i], $rep_errors, $prefix + $input.name + "_" + str($index) + "|", $context )
+            <div class="form-row"><input type="submit" name="${prefix}${input.name}_${index}_remove" value="Remove ${input.title} ${i+1}"></div>
            </div>
          #end for
          <div class="form-row"><input type="submit" name="${prefix}${input.name}_add" value="Add new ${input.title}"></div>
@@ -47,12 +47,9 @@
            body = $( "<img src='${h.url_for('/static/images/yui/rel_interstitial_loading.gif')}'/>" );
        }
        $( ".dialog-box" ).find( ".body" ).html( body );
-        // $( "#overlay").show();
        if ( ! $(".dialog-box-container").is( ":visible" ) ) {
            $("#overlay").show();
            $(".dialog-box-container").fadeIn()
-        } else {
-            // $(".dialog-box").center( "horizontal" );
        }
    };
    </script>
@@ -81,6 +78,8 @@
                success: function( data ) {
                     reset();
                     workflow.from_simple( data );
+                     workflow.has_changes = false;
+                     scroll_to_nodes();
                     hide_modal();
                 },
                 beforeSubmit: function( data ) {
@@ -218,9 +217,18 @@
            workflow.remove_all();
        }
        workflow = new Workflow();
-        // Start at the middle of the canvas
-        // $(window).scrollTop( 2500 );
-        // $(window).scrollLeft( 2500 );
+    }
+    
+    function scroll_to_nodes() {
+        // Scroll to the top left node
+        if ( $("div.toolFormInCanvas").length > 0 ) {
+            var x = 5000, y = 5000;
+            $("div.toolFormInCanvas").each( function() {
+                x = Math.min( x, $(this).position().left );
+                y = Math.min( x, $(this).position().left );
+            });            
+            $("#canvas-container").css( { left: - x + 20, top: - y + 20 } );
+        }
    }
    
    // Add a new step to the workflow by tool id
@@ -255,12 +263,15 @@
        // });
        $("#right-content").html( text );
        $("#right-content").find( "form" ).ajaxForm( {
-            method: 'POST',
+            type: 'POST',
            dataType: 'json',
            success: function( data ) { 
                node.update_field_data( data );
            },
-            beforeSubmit: function( data ) { data.push( { name: 'tool_state', value: node.tool_state } ); }
+            beforeSubmit: function( data ) {
+                data.push( { name: 'tool_state', value: node.tool_state } );
+                data.push( { name: '_', value: "true" } );
+            }
        }).each( function() {
            form = this;
            $(this).find( "select[refresh_on_change='true']").change( function() {
@@ -324,6 +335,7 @@
            success: function( data ) {
                reset();
                workflow.from_simple( data );
+                workflow.has_changes = false;
                show_modal( "Workflow loaded", "Workflow loaded.", {
                    "Ok" : function () { hide_modal(); }
                });
@@ -5,15 +5,17 @@
          <div class="form-title-row"><b>${input.title_plural}</b></div>
          <% repeat_values = values[input.name] %>
          %for i in range( len( repeat_values ) ):
-            %if input.name in errors:
-                <% rep_errors = errors[input.name][i] %>
-            %else:
-                <% rep_errors = dict() %>
-            %endif
+            <%
+            if input.name in errors:
+                rep_errors = errors[input.name][i]
+            else:
+                rep_errors = dict()
+            index = repeat_values[i]['__index__']
+            %>
            <div class="repeat-group-item">
            <div class="form-title-row"><b>${input.title} ${i + 1}</b></div>
-            ${do_inputs( input.inputs, repeat_values[ i ], rep_errors,  prefix + input.name + "_" + str(i) + "|" )}
-            <div class="form-row"><input type="submit" name="${prefix}${input.name}_${i}_remove" value="Remove ${input.title} ${i+1}"></div>
+            ${do_inputs( input.inputs, repeat_values[ i ], rep_errors,  prefix + input.name + "_" + str(index) + "|" )}
+            <div class="form-row"><input type="submit" name="${prefix}${input.name}_${index}_remove" value="Remove ${input.title} ${i+1}"></div>
            </div>
          %endfor
          <div class="form-row"><input type="submit" name="${prefix}${input.name}_add" value="Add new ${input.title}"></div>
@@ -24,7 +26,7 @@
      <% prefix = prefix + input.name + "|" %>
      <% group_errors = errors.get( input.name, {} ) %>
      ${row_for_param( input.test_param, group_values[ input.test_param.name ], group_errors, prefix )}
-      ${do_inputs( input.cases[ current_case ].inputs, group_values, group_errors, prefix + input.name + "|" )}
+      ${do_inputs( input.cases[ current_case ].inputs, group_values, group_errors, prefix )}
    %else:
      ${row_for_param( input, values[ input.name ], errors, prefix )}
    %endif
@@ -51,10 +51,11 @@

    %if workflows:
        <table class="colored" border="0" cellspacing="0" cellpadding="0" width="100%">
-            <tr class="header"><td>Name</td><td>Last Updated</td><td>Actions</td></tr>
+            <tr class="header"><td>Name</td><td># of Steps<td>Last Updated</td><td>Actions</td></tr>
            %for workflow in workflows:
                <tr>
                    <td>${workflow.name}</td>
+                    <td>${len(workflow.latest_workflow.steps)}
                    <td>${str(workflow.update_time)[:19]}</td>
                    <td>
                        <a href="${h.url_for( action='run', id=trans.security.encode_id(workflow.id) )}">run</a>