From 63c6c6dadd329cca4df31064cb17fc820342fcbe Mon Sep 17 00:00:00 2001 From: James Taylor Date: Tue, 15 Jan 2008 00:12:21 +0000 Subject: [PATCH] Workflows can now be run!!! If 'enable_beta_features' is true there will be a link to workflow management from the tool menu. From there new workflows can be created and existing workflows edited / run. Still substantial problems with: - Selects using "dynamic options" of any sort - Repeat/conditional tools (no way to connect inputs in a repeat or conditional) The database / json representation is a real mess. I'm going to completely redo that next. --- lib/galaxy/web/controllers/workflow.py | 123 +++++++++++++----- lib/galaxy/web/framework/__init__.py | 9 ++ lib/galaxy/workflow/__init__.py | 36 +++-- .../scripts/galaxy.workflow_editor.canvas.js | 4 +- templates/tool_menu.tmpl | 20 +++ templates/workflow/editor.mako | 5 +- templates/workflow/index.mako | 45 +++++++ templates/workflow/run.mako | 18 ++- templates/workflow/run_complete.mako | 31 +++++ 9 files changed, 239 insertions(+), 52 deletions(-) create mode 100644 templates/workflow/index.mako create mode 100644 templates/workflow/run_complete.mako diff --git a/lib/galaxy/web/controllers/workflow.py b/lib/galaxy/web/controllers/workflow.py index 8808f17e4e5..906ccc077ea 100644 --- a/lib/galaxy/web/controllers/workflow.py +++ b/lib/galaxy/web/controllers/workflow.py @@ -12,6 +12,15 @@ from galaxy.util.odict import odict class WorkflowController( BaseController ): beta = True + @web.expose + def index( self, trans ): + user = trans.get_user() + if not user: + return trans.show_error_message( "You must be logged in to use Galaxy workflows." ) + workflows = trans.sa_session.query( model.StoredWorkflow ).filter_by( user = user ) + return trans.fill_template( "workflow/index.mako", + workflows = workflows ) + @web.expose def editor( self, trans, workflow_name=None ): user = trans.get_user() @@ -69,7 +78,7 @@ class WorkflowController( BaseController ): for key, node in nodes.iteritems(): decode_state( node, trans.app ) # Create workflow from json data - workflow = Workflow.from_simple( data ) + workflow = Workflow.from_simple( data, trans.app, decode_inputs=False ) workflow.order_nodes() # Store it stored = model.StoredWorkflow.get_by( user = user, name = workflow_name ) @@ -77,7 +86,7 @@ class WorkflowController( BaseController ): stored = model.StoredWorkflow() stored.user = user stored.name = workflow_name - stored.encoded_value = simplejson.dumps( workflow.to_simple() ) + stored.encoded_value = simplejson.dumps( workflow.to_simple( trans.app ) ) stored.flush() # Return something informative errors = [] @@ -99,15 +108,20 @@ class WorkflowController( BaseController ): trans.workflow_building_mode = True # Load encoded workflow from database stored = model.StoredWorkflow.get_by( user = user, name = workflow_name ) - data = simplejson.loads( stored.encoded_value ) + # FIXME: This is a mess (encode/decoded states and inputs are all over + # the place). Fix with more structured database representation. + workflow = Workflow.from_simple( simplejson.loads( stored.encoded_value ), trans.app ) + data = workflow.to_simple( trans.app ) # For each step, rebuild the form and encode the state for step in data['steps'].values(): + step_id = step['id'] + # Load tool tool_id = step['tool_id'] tool = trans.app.toolbox.tools_by_id[tool_id] - # Build a state from the tool_inputs dict - inputs = step['tool_inputs'] + # Build a state from the tool_inputs dict (need to get this from + # the unsimplified Workflow instance since state expects unencoded) state = DefaultToolState() - state.inputs = inputs + state.inputs = workflow.steps[ step_id ].tool_inputs # Replace state in dict with encoded version step['tool_state'] = state.encode( tool, trans.app ) del step['tool_inputs'] @@ -202,7 +216,8 @@ class WorkflowController( BaseController ): workflow.steps[ step_id ] = step # Try to order the nodes workflow.order_nodes() - # And let's try to set up some reasonable locations + # And let's try to set up some reasonable locations on canvas + # (these are pretty arbitrary values) levorder = workflow.order_nodes_levels() base_pos = 2510 for i, steps_at_level in enumerate( levorder ): @@ -216,42 +231,86 @@ class WorkflowController( BaseController ): stored = model.StoredWorkflow() stored.user = user stored.name = workflow_name - stored.encoded_value = simplejson.dumps( workflow.to_simple() ) + stored.encoded_value = simplejson.dumps( workflow.to_simple( trans.app ) ) stored.flush() # return trans.show_ok_message( "Workflow '%s' created.
Click to load in workflow editor" % ( workflow_name, web.url_for( action='editor', workflow_name=workflow_name ) ) ) @web.expose - def run( self, trans, workflow_name ): + def run( self, trans, workflow_name, **kwargs ): user = trans.get_user() - trans.workflow_building_mode = True + ## trans.workflow_building_mode = True # Load encoded workflow from database - stored = model.StoredWorkflow.get_by( user = user, name = workflow_name ) - workflow = Workflow.from_simple( simplejson.loads( stored.encoded_value ) ) + stored = trans.sa_session.query( model.StoredWorkflow ).get_by( user = user, name = workflow_name ) + if not stored: + return trans.show_error_message( "No workflow named '%s'" % workflow_name ) + workflow = Workflow.from_simple( simplejson.loads( stored.encoded_value ), trans.app ) if workflow.has_cycles: return trans.show_error_message( "Workflow cannot be run because it contains cycles" ) if workflow.has_errors: return trans.show_error_message( "Workflow cannot be run because of validation errors in some steps" ) - inputs = [] - # Ensure node_order is set - workflow.order_nodes() - # Prepare steps for template + # Construct the list of steps steps = [] for step_id in workflow.node_order: step = workflow.steps[ step_id ] steps.append( step ) - # Build a tool state for the step - state = DefaultToolState() - state.inputs = step.tool_inputs - # Store state with the step - step.state = state + # Build the state for each step + if kwargs: + # If kwargs were provided, the states for each step should have + # been POSTed + errors = {} + for step in steps: + # Extract just the arguments for this step by prefix + p = "%s|" % step.id + l = len(p) + step_args = dict( ( k[l:], v ) for ( k, v ) in kwargs.iteritems() if k.startswith( p ) ) + # Get the tool + tool = trans.app.toolbox.tools_by_id[ step.tool_id ] + # Get the state + state = DefaultToolState() + state.decode( step_args.pop("tool_state"), tool, trans.app ) + step.state = state + # Get old errors + old_errors = state.inputs.pop( "__errors__", {} ) + # Update the state + step_errors = tool.update_state( trans, tool.inputs, step.state.inputs, step_args, + update_only=True, old_errors=old_errors ) + if step_errors: + errors[step.id] = state.inputs["__errors__"] = step_errors + if not errors: + # Run each step, connecting outputs to inputs + outputs = {} + for step in steps: + tool = trans.app.toolbox.tools_by_id[ step.tool_id ] + inputs = step.state.inputs + # Connect up + for name, conn in step.input_connections.iteritems(): + if conn: + other_id, other_name = conn + inputs[name] = outputs[other_id][other_name] + outputs[ step.id ] = tool.execute( trans, step.state.inputs ) + return trans.fill_template( "workflow/run_complete.mako", + workflow=stored, + outputs=outputs ) + else: + for step in steps: + # Build a new tool state for the step + state = DefaultToolState() + state.inputs = step.tool_inputs + # Store state with the step + step.state = state + # Don't show any errors on the first load -- we can assume this + # at the moment since we don't allow running workflows with errors + errors = {} + # Render the form return trans.fill_template( "workflow/run.mako", - steps=steps ) - + steps=steps, + workflow=stored, + errors=errors ) ## ---- Utility methods ------------------------------------------------------- @@ -271,15 +330,15 @@ def decode_state( data, app ): data['tool_inputs'] = state.inputs del data['tool_state'] -def encode_state( data, app ): - """ - tool_state --> tool_inputs - """ - tool = app.toolbox.tools_by_id[ data['tool_id'] ] - state = DefaultToolState() - state.inputs = data['tool_inputs'] - data['tool_state'] = state.encode( tool, app ) - del data['tool_inputs'] +#def encode_state( data, app ): +# """ +# tool_state --> tool_inputs +# """ +# tool = app.toolbox.tools_by_id[ data['tool_id'] ] +# state = DefaultToolState() +# state.inputs = data['tool_inputs'] +# data['tool_state'] = state.encode( tool, app ) +# del data['tool_inputs'] def get_job_dict( trans ): """ diff --git a/lib/galaxy/web/framework/__init__.py b/lib/galaxy/web/framework/__init__.py index f50faa14464..26d5e10623b 100644 --- a/lib/galaxy/web/framework/__init__.py +++ b/lib/galaxy/web/framework/__init__.py @@ -72,6 +72,15 @@ class UniverseWebTransaction( base.DefaultWebTransaction ): # that the current history should not be used for parameter values # and such). self.workflow_building_mode = False + + @property + def sa_session( self ): + """ + Returns a SQLAlchemy session -- currently just gets the current + session from the threadlocal session context, but this is provided + to allow migration toward a more SQLAlchemy 0.4 style of use. + """ + return self.app.model.context.current def log_event( self, message, tool_id=None, **kwargs ): """ diff --git a/lib/galaxy/workflow/__init__.py b/lib/galaxy/workflow/__init__.py index bcd89b01ad8..663ffe823b6 100644 --- a/lib/galaxy/workflow/__init__.py +++ b/lib/galaxy/workflow/__init__.py @@ -10,29 +10,40 @@ class Workflow( object ): self.node_order = None @staticmethod - def from_simple( data ): + def from_simple( data, app, decode_inputs=True ): """ Create from simple (list/dict only) representation + + FIXME: This understands data both as it is passed from the web + interface and as it is encoded in the database, messy? """ workflow = Workflow() workflow.has_errors = False for id, step_data in data['steps'].iteritems(): - step = WorkflowStep.from_simple( step_data ) + step = WorkflowStep.from_simple( step_data, app, decode_inputs ) if step.has_errors: workflow.has_errors = True workflow.steps[ int(id) ] = step + # If reading from the database, we will also have these fields + if 'has_errors' in data: + assert workflow.has_errors == data['has_errors'] + if 'has_cycles' in data: + workflow.has_cycles = data['has_cycles'] + if 'node_order' in data: + workflow.node_order = data['node_order'] return workflow - def to_simple( self ): + def to_simple( self, app ): """ Convert to simple (list/dict only) representation """ steps = {} for id, step in self.steps.iteritems(): - steps[ id ] = step.to_simple() + steps[ id ] = step.to_simple( app ) return dict( steps=steps, has_cycles=self.has_cycles, - has_errors=self.has_errors ) + has_errors=self.has_errors, + node_order=self.node_order ) def edge_list( self ): edges = [] @@ -53,8 +64,6 @@ class Workflow( object ): edges = self.edge_list() try: node_order = topsort( edges ) - #node_order_set = set( node_order ) - #node_order.extend( [ id for id in all_ids if id not in node_order ] ) self.node_order = node_order return self.node_order except CycleError: @@ -81,12 +90,15 @@ class WorkflowStep( object ): self.position = None @staticmethod - def from_simple( data ): + def from_simple( data, app, decode_inputs=True ): step = WorkflowStep() - step.id = data['id'] + step.id = int( data['id'] ) step.tool_id = data['tool_id'] step.has_errors = data['has_errors'] step.tool_inputs = data['tool_inputs'] + # Decode inputs *if* neccesary (HACK) + if decode_inputs: + step.tool_inputs = app.toolbox.tools_by_id[ step.tool_id ].params_from_strings( step.tool_inputs, app ) # Position step.position = data.get( 'position', None ) # Connections @@ -97,16 +109,18 @@ class WorkflowStep( object ): step.input_connections[ input_name ] = ( conn['node_id'], conn['output_name' ] ) return step - def to_simple( self ): + def to_simple( self, app ): input_connections = {} for name, conn in self.input_connections.iteritems(): if conn is None: input_connections[ name ] = None else: input_connections[ name ] = dict( node_id = conn[0], output_name = conn[1] ) + # Convert input values to simple representation + tool_inputs = app.toolbox.tools_by_id[ self.tool_id ].params_to_strings( self.tool_inputs, app ) return dict( id = self.id, tool_id = self.tool_id, has_errors = self.has_errors, - tool_inputs = self.tool_inputs, + tool_inputs = tool_inputs, position = self.position, input_connections = input_connections ) \ No newline at end of file diff --git a/static/scripts/galaxy.workflow_editor.canvas.js b/static/scripts/galaxy.workflow_editor.canvas.js index 0a671781fa7..b359b031cee 100644 --- a/static/scripts/galaxy.workflow_editor.canvas.js +++ b/static/scripts/galaxy.workflow_editor.canvas.js @@ -282,7 +282,7 @@ function Workflow() { } $.extend( Workflow.prototype, { add_node : function( node ) { - node.id = String( this.id_counter ); + node.id = this.id_counter; this.id_counter++; this.nodes[ node.id ] = node; this.has_changes = true; @@ -336,7 +336,7 @@ $.extend( Workflow.prototype, { if ( step.position ) { node.element.css( { top: step.position.top, left: step.position.left } ); } - node.id = id; + node.id = step.id; wf.nodes[ node.id ] = node; max_id = Math.max( max_id, parseInt( id ) ) }); diff --git a/templates/tool_menu.tmpl b/templates/tool_menu.tmpl index a229190a2ec..98760e85e54 100644 --- a/templates/tool_menu.tmpl +++ b/templates/tool_menu.tmpl @@ -100,6 +100,26 @@ div#footer { #end for + +## Link to workflow management. The location of this may change, but eventually +## at least some workflows will appear here (the user should be able to +## configure which of their stored workflows appear in the tools menu). + +#if $app.config.enable_beta_features +
+
+
+ Workflow (beta) +
+
+
+
+ Manage workflows +
+
+
+#end if + diff --git a/templates/workflow/editor.mako b/templates/workflow/editor.mako index 8c805b7f7da..22ae77a64f1 100644 --- a/templates/workflow/editor.mako +++ b/templates/workflow/editor.mako @@ -91,7 +91,10 @@ $( function() { $(document).ajaxError( function ( e, x ) { // console.error( "AJAX:", e, ", ", x ); - $("#error-display").html( x.responseText ).show(); + $("#error-display").empty() + .append( $("
").html( x.responseText ) ) + .append( $("
close
" ).click( function() { $("#error-display").hide(); } ) ) + .show(); return false; }); diff --git a/templates/workflow/index.mako b/templates/workflow/index.mako new file mode 100644 index 00000000000..80f8c249045 --- /dev/null +++ b/templates/workflow/index.mako @@ -0,0 +1,45 @@ + + + + Galaxy Workflows + + + +
+ Workflow support is currently in beta testing. + Workflows may not work with all tools, may fail unexpectedly, and may + not be compatible with future updates to Galaxy. +
+ +

Workflow home

+ +

+ Create new workflow using + the Galaxy workflow editor +

+ + +

Stored workflows

+ + %if workflows: + + + %for workflow in workflows: + + + + + %endfor +
NameLast UpdatedActions
${workflow.name}${str(workflow.update_time)[:19]} + run + | edit + | delete +
+ %else: + + You have no stored workflows. + + %endif + + + diff --git a/templates/workflow/run.mako b/templates/workflow/run.mako index 75f34a8114b..256a3e2d292 100644 --- a/templates/workflow/run.mako +++ b/templates/workflow/run.mako @@ -31,10 +31,10 @@ from galaxy.tools.parameters import DataToolParameter
${input.title} ${i + 1}
${do_inputs( input.inputs, repeat_values[ i ], rep_errors, prefix + input.name + "_" + str(i) + "|", step )} -
+
%endfor -
+
%elif input.type == "conditional": <% group_values = values[input.name] %> @@ -60,7 +60,7 @@ from galaxy.tools.parameters import DataToolParameter
%if isinstance( param, DataToolParameter ): %if step.input_connections[ prefix + param.name ] is None: - ${param.get_html_field( t, dict(), dict() ).get_html( prefix )} + ${param.get_html_field( t, dict(), dict() ).get_html( str(step.id) + "|" + prefix )} %else: <% id, name = step.input_connections[ prefix + param.name ] %> Output dataset '${name}' from step ${int(id)+1} @@ -79,14 +79,20 @@ from galaxy.tools.parameters import DataToolParameter - %for step in steps: +

Running workflow "${workflow.name}"

+
+ ## + + %for i, step in enumerate( steps ): <% tool = app.toolbox.tools_by_id[step.tool_id] %> +
-
${int(step.id) + 1}: ${tool.name}
+
Step ${i+1}: ${tool.name}
- ${do_inputs( tool.inputs, step.state.inputs, dict(), "", step )} + ${do_inputs( tool.inputs, step.state.inputs, errors.get( step.id, dict() ), "", step )}
%endfor +
\ No newline at end of file diff --git a/templates/workflow/run_complete.mako b/templates/workflow/run_complete.mako new file mode 100644 index 00000000000..93efdd29650 --- /dev/null +++ b/templates/workflow/run_complete.mako @@ -0,0 +1,31 @@ + + + + + + + + + + +
+

+ Sucesfully ran workflow "${workflow.name}", the following datasets have + been added to the queue. +

+ +
+ %for step_outputs in outputs.itervalues(): + %for data in step_outputs.itervalues(): +

${data.hid}: ${data.name}

+ %endfor + %endfor +
+
+ + + \ No newline at end of file