mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-21 13:50:20 +08:00
Workflows can now be run!!! If 'enable_beta_features' is true there will be
a link to workflow management from the tool menu. From there new workflows can be created and existing workflows edited / run. Still substantial problems with: - Selects using "dynamic options" of any sort - Repeat/conditional tools (no way to connect inputs in a repeat or conditional) The database / json representation is a real mess. I'm going to completely redo that next.
This commit is contained in:
James Taylor
@@ -12,6 +12,15 @@ from galaxy.util.odict import odict
|
||||
class WorkflowController( BaseController ):
|
||||
beta = True
|
||||
|
||||
@web.expose
|
||||
def index( self, trans ):
|
||||
user = trans.get_user()
|
||||
if not user:
|
||||
return trans.show_error_message( "You must be logged in to use <b>Galaxy</b> workflows." )
|
||||
workflows = trans.sa_session.query( model.StoredWorkflow ).filter_by( user = user )
|
||||
return trans.fill_template( "workflow/index.mako",
|
||||
workflows = workflows )
|
||||
|
||||
@web.expose
|
||||
def editor( self, trans, workflow_name=None ):
|
||||
user = trans.get_user()
|
||||
@@ -69,7 +78,7 @@ class WorkflowController( BaseController ):
|
||||
for key, node in nodes.iteritems():
|
||||
decode_state( node, trans.app )
|
||||
# Create workflow from json data
|
||||
workflow = Workflow.from_simple( data )
|
||||
workflow = Workflow.from_simple( data, trans.app, decode_inputs=False )
|
||||
workflow.order_nodes()
|
||||
# Store it
|
||||
stored = model.StoredWorkflow.get_by( user = user, name = workflow_name )
|
||||
@@ -77,7 +86,7 @@ class WorkflowController( BaseController ):
|
||||
stored = model.StoredWorkflow()
|
||||
stored.user = user
|
||||
stored.name = workflow_name
|
||||
stored.encoded_value = simplejson.dumps( workflow.to_simple() )
|
||||
stored.encoded_value = simplejson.dumps( workflow.to_simple( trans.app ) )
|
||||
stored.flush()
|
||||
# Return something informative
|
||||
errors = []
|
||||
@@ -99,15 +108,20 @@ class WorkflowController( BaseController ):
|
||||
trans.workflow_building_mode = True
|
||||
# Load encoded workflow from database
|
||||
stored = model.StoredWorkflow.get_by( user = user, name = workflow_name )
|
||||
data = simplejson.loads( stored.encoded_value )
|
||||
# FIXME: This is a mess (encode/decoded states and inputs are all over
|
||||
# the place). Fix with more structured database representation.
|
||||
workflow = Workflow.from_simple( simplejson.loads( stored.encoded_value ), trans.app )
|
||||
data = workflow.to_simple( trans.app )
|
||||
# For each step, rebuild the form and encode the state
|
||||
for step in data['steps'].values():
|
||||
step_id = step['id']
|
||||
# Load tool
|
||||
tool_id = step['tool_id']
|
||||
tool = trans.app.toolbox.tools_by_id[tool_id]
|
||||
# Build a state from the tool_inputs dict
|
||||
inputs = step['tool_inputs']
|
||||
# Build a state from the tool_inputs dict (need to get this from
|
||||
# the unsimplified Workflow instance since state expects unencoded)
|
||||
state = DefaultToolState()
|
||||
state.inputs = inputs
|
||||
state.inputs = workflow.steps[ step_id ].tool_inputs
|
||||
# Replace state in dict with encoded version
|
||||
step['tool_state'] = state.encode( tool, trans.app )
|
||||
del step['tool_inputs']
|
||||
@@ -202,7 +216,8 @@ class WorkflowController( BaseController ):
|
||||
workflow.steps[ step_id ] = step
|
||||
# Try to order the nodes
|
||||
workflow.order_nodes()
|
||||
# And let's try to set up some reasonable locations
|
||||
# And let's try to set up some reasonable locations on canvas
|
||||
# (these are pretty arbitrary values)
|
||||
levorder = workflow.order_nodes_levels()
|
||||
base_pos = 2510
|
||||
for i, steps_at_level in enumerate( levorder ):
|
||||
@@ -216,42 +231,86 @@ class WorkflowController( BaseController ):
|
||||
stored = model.StoredWorkflow()
|
||||
stored.user = user
|
||||
stored.name = workflow_name
|
||||
stored.encoded_value = simplejson.dumps( workflow.to_simple() )
|
||||
stored.encoded_value = simplejson.dumps( workflow.to_simple( trans.app ) )
|
||||
stored.flush()
|
||||
#
|
||||
return trans.show_ok_message( "Workflow '%s' created.<br/><a target='_top' href='%s'>Click to load in workflow editor</a>"
|
||||
% ( workflow_name, web.url_for( action='editor', workflow_name=workflow_name ) ) )
|
||||
|
||||
@web.expose
|
||||
def run( self, trans, workflow_name ):
|
||||
def run( self, trans, workflow_name, **kwargs ):
|
||||
user = trans.get_user()
|
||||
trans.workflow_building_mode = True
|
||||
## trans.workflow_building_mode = True
|
||||
# Load encoded workflow from database
|
||||
stored = model.StoredWorkflow.get_by( user = user, name = workflow_name )
|
||||
workflow = Workflow.from_simple( simplejson.loads( stored.encoded_value ) )
|
||||
stored = trans.sa_session.query( model.StoredWorkflow ).get_by( user = user, name = workflow_name )
|
||||
if not stored:
|
||||
return trans.show_error_message( "No workflow named '%s'" % workflow_name )
|
||||
workflow = Workflow.from_simple( simplejson.loads( stored.encoded_value ), trans.app )
|
||||
if workflow.has_cycles:
|
||||
return trans.show_error_message(
|
||||
"Workflow cannot be run because it contains cycles" )
|
||||
if workflow.has_errors:
|
||||
return trans.show_error_message(
|
||||
"Workflow cannot be run because of validation errors in some steps" )
|
||||
inputs = []
|
||||
# Ensure node_order is set
|
||||
workflow.order_nodes()
|
||||
# Prepare steps for template
|
||||
# Construct the list of steps
|
||||
steps = []
|
||||
for step_id in workflow.node_order:
|
||||
step = workflow.steps[ step_id ]
|
||||
steps.append( step )
|
||||
# Build a tool state for the step
|
||||
state = DefaultToolState()
|
||||
state.inputs = step.tool_inputs
|
||||
# Store state with the step
|
||||
step.state = state
|
||||
# Build the state for each step
|
||||
if kwargs:
|
||||
# If kwargs were provided, the states for each step should have
|
||||
# been POSTed
|
||||
errors = {}
|
||||
for step in steps:
|
||||
# Extract just the arguments for this step by prefix
|
||||
p = "%s|" % step.id
|
||||
l = len(p)
|
||||
step_args = dict( ( k[l:], v ) for ( k, v ) in kwargs.iteritems() if k.startswith( p ) )
|
||||
# Get the tool
|
||||
tool = trans.app.toolbox.tools_by_id[ step.tool_id ]
|
||||
# Get the state
|
||||
state = DefaultToolState()
|
||||
state.decode( step_args.pop("tool_state"), tool, trans.app )
|
||||
step.state = state
|
||||
# Get old errors
|
||||
old_errors = state.inputs.pop( "__errors__", {} )
|
||||
# Update the state
|
||||
step_errors = tool.update_state( trans, tool.inputs, step.state.inputs, step_args,
|
||||
update_only=True, old_errors=old_errors )
|
||||
if step_errors:
|
||||
errors[step.id] = state.inputs["__errors__"] = step_errors
|
||||
if not errors:
|
||||
# Run each step, connecting outputs to inputs
|
||||
outputs = {}
|
||||
for step in steps:
|
||||
tool = trans.app.toolbox.tools_by_id[ step.tool_id ]
|
||||
inputs = step.state.inputs
|
||||
# Connect up
|
||||
for name, conn in step.input_connections.iteritems():
|
||||
if conn:
|
||||
other_id, other_name = conn
|
||||
inputs[name] = outputs[other_id][other_name]
|
||||
outputs[ step.id ] = tool.execute( trans, step.state.inputs )
|
||||
return trans.fill_template( "workflow/run_complete.mako",
|
||||
workflow=stored,
|
||||
outputs=outputs )
|
||||
else:
|
||||
for step in steps:
|
||||
# Build a new tool state for the step
|
||||
state = DefaultToolState()
|
||||
state.inputs = step.tool_inputs
|
||||
# Store state with the step
|
||||
step.state = state
|
||||
# Don't show any errors on the first load -- we can assume this
|
||||
# at the moment since we don't allow running workflows with errors
|
||||
errors = {}
|
||||
# Render the form
|
||||
return trans.fill_template(
|
||||
"workflow/run.mako",
|
||||
steps=steps )
|
||||
|
||||
steps=steps,
|
||||
workflow=stored,
|
||||
errors=errors )
|
||||
|
||||
## ---- Utility methods -------------------------------------------------------
|
||||
|
||||
@@ -271,15 +330,15 @@ def decode_state( data, app ):
|
||||
data['tool_inputs'] = state.inputs
|
||||
del data['tool_state']
|
||||
|
||||
def encode_state( data, app ):
|
||||
"""
|
||||
tool_state --> tool_inputs
|
||||
"""
|
||||
tool = app.toolbox.tools_by_id[ data['tool_id'] ]
|
||||
state = DefaultToolState()
|
||||
state.inputs = data['tool_inputs']
|
||||
data['tool_state'] = state.encode( tool, app )
|
||||
del data['tool_inputs']
|
||||
#def encode_state( data, app ):
|
||||
# """
|
||||
# tool_state --> tool_inputs
|
||||
# """
|
||||
# tool = app.toolbox.tools_by_id[ data['tool_id'] ]
|
||||
# state = DefaultToolState()
|
||||
# state.inputs = data['tool_inputs']
|
||||
# data['tool_state'] = state.encode( tool, app )
|
||||
# del data['tool_inputs']
|
||||
|
||||
def get_job_dict( trans ):
|
||||
"""
|
||||
|
||||
@@ -72,6 +72,15 @@ class UniverseWebTransaction( base.DefaultWebTransaction ):
|
||||
# that the current history should not be used for parameter values
|
||||
# and such).
|
||||
self.workflow_building_mode = False
|
||||
|
||||
@property
|
||||
def sa_session( self ):
|
||||
"""
|
||||
Returns a SQLAlchemy session -- currently just gets the current
|
||||
session from the threadlocal session context, but this is provided
|
||||
to allow migration toward a more SQLAlchemy 0.4 style of use.
|
||||
"""
|
||||
return self.app.model.context.current
|
||||
|
||||
def log_event( self, message, tool_id=None, **kwargs ):
|
||||
"""
|
||||
|
||||
@@ -10,29 +10,40 @@ class Workflow( object ):
|
||||
self.node_order = None
|
||||
|
||||
@staticmethod
|
||||
def from_simple( data ):
|
||||
def from_simple( data, app, decode_inputs=True ):
|
||||
"""
|
||||
Create from simple (list/dict only) representation
|
||||
|
||||
FIXME: This understands data both as it is passed from the web
|
||||
interface and as it is encoded in the database, messy?
|
||||
"""
|
||||
workflow = Workflow()
|
||||
workflow.has_errors = False
|
||||
for id, step_data in data['steps'].iteritems():
|
||||
step = WorkflowStep.from_simple( step_data )
|
||||
step = WorkflowStep.from_simple( step_data, app, decode_inputs )
|
||||
if step.has_errors:
|
||||
workflow.has_errors = True
|
||||
workflow.steps[ int(id) ] = step
|
||||
# If reading from the database, we will also have these fields
|
||||
if 'has_errors' in data:
|
||||
assert workflow.has_errors == data['has_errors']
|
||||
if 'has_cycles' in data:
|
||||
workflow.has_cycles = data['has_cycles']
|
||||
if 'node_order' in data:
|
||||
workflow.node_order = data['node_order']
|
||||
return workflow
|
||||
|
||||
def to_simple( self ):
|
||||
def to_simple( self, app ):
|
||||
"""
|
||||
Convert to simple (list/dict only) representation
|
||||
"""
|
||||
steps = {}
|
||||
for id, step in self.steps.iteritems():
|
||||
steps[ id ] = step.to_simple()
|
||||
steps[ id ] = step.to_simple( app )
|
||||
return dict( steps=steps,
|
||||
has_cycles=self.has_cycles,
|
||||
has_errors=self.has_errors )
|
||||
has_errors=self.has_errors,
|
||||
node_order=self.node_order )
|
||||
|
||||
def edge_list( self ):
|
||||
edges = []
|
||||
@@ -53,8 +64,6 @@ class Workflow( object ):
|
||||
edges = self.edge_list()
|
||||
try:
|
||||
node_order = topsort( edges )
|
||||
#node_order_set = set( node_order )
|
||||
#node_order.extend( [ id for id in all_ids if id not in node_order ] )
|
||||
self.node_order = node_order
|
||||
return self.node_order
|
||||
except CycleError:
|
||||
@@ -81,12 +90,15 @@ class WorkflowStep( object ):
|
||||
self.position = None
|
||||
|
||||
@staticmethod
|
||||
def from_simple( data ):
|
||||
def from_simple( data, app, decode_inputs=True ):
|
||||
step = WorkflowStep()
|
||||
step.id = data['id']
|
||||
step.id = int( data['id'] )
|
||||
step.tool_id = data['tool_id']
|
||||
step.has_errors = data['has_errors']
|
||||
step.tool_inputs = data['tool_inputs']
|
||||
# Decode inputs *if* neccesary (HACK)
|
||||
if decode_inputs:
|
||||
step.tool_inputs = app.toolbox.tools_by_id[ step.tool_id ].params_from_strings( step.tool_inputs, app )
|
||||
# Position
|
||||
step.position = data.get( 'position', None )
|
||||
# Connections
|
||||
@@ -97,16 +109,18 @@ class WorkflowStep( object ):
|
||||
step.input_connections[ input_name ] = ( conn['node_id'], conn['output_name' ] )
|
||||
return step
|
||||
|
||||
def to_simple( self ):
|
||||
def to_simple( self, app ):
|
||||
input_connections = {}
|
||||
for name, conn in self.input_connections.iteritems():
|
||||
if conn is None:
|
||||
input_connections[ name ] = None
|
||||
else:
|
||||
input_connections[ name ] = dict( node_id = conn[0], output_name = conn[1] )
|
||||
# Convert input values to simple representation
|
||||
tool_inputs = app.toolbox.tools_by_id[ self.tool_id ].params_to_strings( self.tool_inputs, app )
|
||||
return dict( id = self.id,
|
||||
tool_id = self.tool_id,
|
||||
has_errors = self.has_errors,
|
||||
tool_inputs = self.tool_inputs,
|
||||
tool_inputs = tool_inputs,
|
||||
position = self.position,
|
||||
input_connections = input_connections )
|
||||
@@ -282,7 +282,7 @@ function Workflow() {
|
||||
}
|
||||
$.extend( Workflow.prototype, {
|
||||
add_node : function( node ) {
|
||||
node.id = String( this.id_counter );
|
||||
node.id = this.id_counter;
|
||||
this.id_counter++;
|
||||
this.nodes[ node.id ] = node;
|
||||
this.has_changes = true;
|
||||
@@ -336,7 +336,7 @@ $.extend( Workflow.prototype, {
|
||||
if ( step.position ) {
|
||||
node.element.css( { top: step.position.top, left: step.position.left } );
|
||||
}
|
||||
node.id = id;
|
||||
node.id = step.id;
|
||||
wf.nodes[ node.id ] = node;
|
||||
max_id = Math.max( max_id, parseInt( id ) )
|
||||
});
|
||||
|
||||
@@ -100,6 +100,26 @@ div#footer {
|
||||
</div>
|
||||
</div>
|
||||
#end for
|
||||
|
||||
## Link to workflow management. The location of this may change, but eventually
|
||||
## at least some workflows will appear here (the user should be able to
|
||||
## configure which of their stored workflows appear in the tools menu).
|
||||
|
||||
#if $app.config.enable_beta_features
|
||||
<div class="toolSectionPad"></div>
|
||||
<div class="toolSectionPad"></div>
|
||||
<div class="toolSectionTitle" id="title_XXinternalXXworkflow">
|
||||
<span>Workflow <i>(beta)</i></span>
|
||||
</div>
|
||||
<div id="XXinternalXXworkflow" class="toolSectionBody">
|
||||
<div class="toolSectionBg">
|
||||
<div class="toolTitle">
|
||||
<a href="${h.url_for( controller='workflow', action='index' )}" target="galaxy_main">Manage</a> workflows
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
#end if
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
@@ -91,7 +91,10 @@ $( function() {
|
||||
|
||||
$(document).ajaxError( function ( e, x ) {
|
||||
// console.error( "AJAX:", e, ", ", x );
|
||||
$("#error-display").html( x.responseText ).show();
|
||||
$("#error-display").empty()
|
||||
.append( $("<div/>").html( x.responseText ) )
|
||||
.append( $("<div><a>close</a></div>" ).click( function() { $("#error-display").hide(); } ) )
|
||||
.show();
|
||||
return false;
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
|
||||
<html>
|
||||
<head>
|
||||
<title>Galaxy Workflows</title>
|
||||
<link rel="stylesheet" type="text/css" href="${h.url_for('/static/style/base.css')}"></link>
|
||||
</head>
|
||||
<body>
|
||||
<div class="warningmessage">
|
||||
Workflow support is currently in <b><i>beta</i></b> testing.
|
||||
Workflows may not work with all tools, may fail unexpectedly, and may
|
||||
not be compatible with future updates to <b>Galaxy</b>.
|
||||
</div>
|
||||
|
||||
<h2>Workflow home</h2>
|
||||
|
||||
<p>
|
||||
<a href="${h.url_for( action='editor')}" target="_parent">Create new workflow</a> using
|
||||
the <b>Galaxy</b> workflow editor
|
||||
</p>
|
||||
|
||||
|
||||
<h2>Stored workflows</h2>
|
||||
|
||||
%if workflows:
|
||||
<table class="colored" border="0" cellspacing="0" cellpadding="0" width="100%">
|
||||
<tr class="header"><td>Name</td><td>Last Updated</td><td>Actions</td></tr>
|
||||
%for workflow in workflows:
|
||||
<tr>
|
||||
<td>${workflow.name}</td>
|
||||
<td>${str(workflow.update_time)[:19]}</td>
|
||||
<td>
|
||||
<a href="${h.url_for( action='run', workflow_name=workflow.name )}">run</a>
|
||||
| <a href="${h.url_for( action='editor', workflow_name=workflow.name )}" target="_parent">edit</a>
|
||||
| <a href="${h.url_for( action='delete', workflow_name=workflow.name )}">delete</a>
|
||||
</tr>
|
||||
%endfor
|
||||
</table>
|
||||
%else:
|
||||
|
||||
You have no stored workflows.
|
||||
|
||||
%endif
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -31,10 +31,10 @@ from galaxy.tools.parameters import DataToolParameter
|
||||
<div class="repeat-group-item">
|
||||
<div class="form-title-row"><b>${input.title} ${i + 1}</b></div>
|
||||
${do_inputs( input.inputs, repeat_values[ i ], rep_errors, prefix + input.name + "_" + str(i) + "|", step )}
|
||||
<div class="form-row"><input type="submit" name="${prefix}${input.name}_${i}_remove" value="Remove ${input.title} ${i+1}" /></div>
|
||||
<div class="form-row"><input type="submit" name="${step.id}|${prefix}${input.name}_${i}_remove" value="Remove ${input.title} ${i+1}" /></div>
|
||||
</div>
|
||||
%endfor
|
||||
<div class="form-row"><input type="submit" name="${prefix}${input.name}_add" value="Add new ${input.title}" /></div>
|
||||
<div class="form-row"><input type="submit" name="${step.id}|${prefix}${input.name}_add" value="Add new ${input.title}" /></div>
|
||||
</div>
|
||||
%elif input.type == "conditional":
|
||||
<% group_values = values[input.name] %>
|
||||
@@ -60,7 +60,7 @@ from galaxy.tools.parameters import DataToolParameter
|
||||
<div>
|
||||
%if isinstance( param, DataToolParameter ):
|
||||
%if step.input_connections[ prefix + param.name ] is None:
|
||||
${param.get_html_field( t, dict(), dict() ).get_html( prefix )}
|
||||
${param.get_html_field( t, dict(), dict() ).get_html( str(step.id) + "|" + prefix )}
|
||||
%else:
|
||||
<% id, name = step.input_connections[ prefix + param.name ] %>
|
||||
Output dataset '${name}' from step ${int(id)+1}
|
||||
@@ -79,14 +79,20 @@ from galaxy.tools.parameters import DataToolParameter
|
||||
</%def>
|
||||
|
||||
<body>
|
||||
%for step in steps:
|
||||
<h2>Running workflow "${workflow.name}"</h2>
|
||||
<form method="POST">
|
||||
## <input type="hidden" name="workflow_name" value="${workflow.name | h}" />
|
||||
<input type="submit" value="Run workflow" />
|
||||
%for i, step in enumerate( steps ):
|
||||
<% tool = app.toolbox.tools_by_id[step.tool_id] %>
|
||||
<input type="hidden" name="${step.id}|tool_state" value="${step.state.encode( tool, app )}">
|
||||
<div class="toolForm">
|
||||
<div class="toolFormTitle">${int(step.id) + 1}: ${tool.name}</div>
|
||||
<div class="toolFormTitle">Step ${i+1}: ${tool.name}</div>
|
||||
<div class="toolFormBody">
|
||||
${do_inputs( tool.inputs, step.state.inputs, dict(), "", step )}
|
||||
${do_inputs( tool.inputs, step.state.inputs, errors.get( step.id, dict() ), "", step )}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
%endfor
|
||||
</form>
|
||||
</body>
|
||||
@@ -0,0 +1,31 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
|
||||
|
||||
<head>
|
||||
<link href="${h.url_for('/static/style/base.css')}" rel="stylesheet" type="text/css" />
|
||||
<script>
|
||||
if ( parent.frames && parent.frames.galaxy_history ) {
|
||||
parent.frames.galaxy_history.location.href="${h.url_for( controller='root', action='history' ) }";
|
||||
}
|
||||
</script>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<div class="donemessage">
|
||||
<p>
|
||||
Sucesfully ran workflow "${workflow.name}", the following datasets have
|
||||
been added to the queue.
|
||||
</p>
|
||||
|
||||
<div style="padding-left: 10px;">
|
||||
%for step_outputs in outputs.itervalues():
|
||||
%for data in step_outputs.itervalues():
|
||||
<p><b>${data.hid}</b>: ${data.name}</p>
|
||||
%endfor
|
||||
%endfor
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
Reference in New Issue
Block a user