Workflow round five, workflows can now be constructed from histories. See

the history options page for "Construct workflow" link. It allows you to
select which steps from the current history you want to convert.

This is currently not very robust to changes in tool configuration, so old
histories may cause errors. This will be fixable to some extent, but we'll
probably need to go back and look at the past changes to tool configs and
define some migration rules.
This commit is contained in:
James Taylor
2007-10-31 03:57:06 +00:00
parent 25b0d31956
commit 5e9fe3e6b2
25 changed files with 495 additions and 50 deletions
+9
View File
@@ -67,6 +67,15 @@ class Job( object ):
# For historical reasons state propogates down to datasets
for da in self.output_datasets:
da.dataset.state = state
def get_param_values( self, app ):
"""
Read encoded parameter values from the database and turn back into a
dict of tool parameter values.
"""
param_dict = dict( [ ( p.name, p.value ) for p in self.parameters ] )
tool = app.toolbox.tools_by_id[self.tool_id]
param_dict = tool.params_from_strings( param_dict, app )
return param_dict
class JobParameter( object ):
def __init__( self, name, value ):
+16
View File
@@ -280,6 +280,9 @@ class Tool:
log.exception( "Failed to parse tool tests" )
else:
self.tests = None
# Determine if this tool can be used in workflows
self.is_workflow_compatible = self.check_workflow_compatible()
def parse_inputs( self, root ):
"""
@@ -459,6 +462,19 @@ class Tool:
for name in param.get_dependencies():
context[ name ].refresh_on_change = True
return param
def check_workflow_compatible( self ):
# This is probably the best bet for detecting external web tools
# right now
if self.action != "/tool_runner/index":
return False
# HACK: upload is (as always) a special case becuase file parameters
# can't be persisted.
if self.id == "upload1":
return False
# TODO: Anyway to capture tools that dynamically change their own
# outputs?
return True
def new_state( self, trans, all_pages=False ):
"""
+2 -1
View File
@@ -44,7 +44,8 @@ class DefaultToolAction( object ):
# FIXME: does this need to modify 'incoming' or should this be
# moved into 'build_param_dict'? Is this just about getting the
# metadata into the command line?
# metadata into the command line?
# NEED TO FIX THIS SOON.
input_names = []
input_ext = 'data'
input_dbkey = incoming.get( "dbkey", "?" )
+46 -1
View File
@@ -161,4 +161,49 @@ def topsort(pairlist):
for x in numpreds.keys():
assert numpreds[x] > 0
raise CycleError(answer, numpreds, successors)
return answer
return answer
def topsort_levels(pairlist):
numpreds = {} # elt -> # of predecessors
successors = {} # elt -> list of successors
for first, second in pairlist:
# make sure every elt is a key in numpreds
if not numpreds.has_key(first):
numpreds[first] = 0
if not numpreds.has_key(second):
numpreds[second] = 0
# if they're the same, there's no real dependence
if first == second:
continue
# since first < second, second gains a pred ...
numpreds[second] = numpreds[second] + 1
# ... and first gains a succ
if successors.has_key(first):
successors[first].append(second)
else:
successors[first] = [second]
answer = []
while 1:
# Suck up everything without a predecessor.
levparents = [x for x in numpreds.keys() if numpreds[x] == 0]
if not levparents:
break
answer.append( levparents )
for levparent in levparents:
del numpreds[levparent]
if successors.has_key(levparent):
for levparentsucc in successors[levparent]:
numpreds[levparentsucc] -= 1
del successors[levparent]
if numpreds:
# Everything in num_parents has at least one child ->
# there's a cycle.
raise CycleError( answer, numpreds, successors )
return answer
+158
View File
@@ -0,0 +1,158 @@
from galaxy.web.base.controller import *
from galaxy.tools.parameters import DataToolParameter
from galaxy.tools import DefaultToolState
from galaxy.tools.grouping import Repeat, Conditional
from galaxy.datatypes.data import Data
from galaxy.workflow import Workflow, WorkflowStep
from galaxy.util.odict import odict
import simplejson
class HistoryController( BaseController ):
def __get_job_dict( self, trans ):
"""
Return a dictionary of Job -> [ Dataset ] mappings, for all finished
active Datasets in the current history and the jobs that created them.
"""
history = trans.get_history()
# Get the jobs that created the datasets
warnings = set()
jobs = odict()
for dataset in history.active_datasets:
# FIXME: Create "Dataset.is_finished"
if dataset.state in ( 'new', 'running', 'queued' ):
warnings.add( "Some datasets still queued or running were ignored" )
continue
for assoc in dataset.creating_job_associations:
job = assoc.job
if job in jobs:
jobs[ job ].append( ( assoc.name, dataset ) )
else:
jobs[ job ] = [ ( assoc.name, dataset ) ]
return jobs, warnings
def __cleanup_param_values( self, inputs, values ):
"""
Remove 'Data' values from `param_values` but track the associations
"""
associations = []
names_to_clean = []
# dbkey is pushed in by the framework
if 'dbkey' in values:
del values['dbkey']
root_values = values
# Cleanup all data inputs
def cleanup( prefix, inputs, values ):
for key, input in inputs.items():
if isinstance( input, DataToolParameter ):
tmp = values[key]
values[key] = None
# HACK: Nested associations are not yet working, but we
# still need to clean them up so we can serialize
if not( prefix ):
associations.append( ( tmp.hid, prefix + key ) )
# Cleanup the other deprecated crap associated with datasets
# as well. Worse, for nested datasets all the metadata is
# being pushed into the root. FIXME: MUST REMOVE SOON
key = prefix + key + "_"
for k in root_values.keys():
if k.startswith( key ):
del root_values[k]
elif isinstance( input, Repeat ):
group_values = values[key]
for i in range( len( group_values ) ):
prefix = "%s_%d|" % ( key, i )
cleanup( prefix, input.inputs, group_values[i] )
elif isinstance( input, Conditional ):
group_values = values[input.name]
current_case = group_values['__current_case__']
prefix = "%s|" % ( key )
cleanup( prefix, input.cases[current_case].inputs, group_values )
cleanup( "", inputs, values )
return associations
@web.expose
def workflow_from_current_history( self, trans, job_ids=None, workflow_name=None ):
if trans.request.method == 'POST':
return self.workflow_from_current_history_post( trans, job_ids, workflow_name )
user = trans.get_user()
if not user:
return trans.show_error_message( "Must be logged in to create workflows" )
jobs, warnings = self.__get_job_dict( trans )
# Render
return trans.fill_template(
"history/workflow_from_current_history.mako",
jobs=jobs,
warnings=warnings )
def workflow_from_current_history_post( self, trans, job_ids, workflow_name ):
user = trans.get_user()
if not user:
return trans.show_error_message( "Must be logged in to create workflows" )
# Ensure a list
if type( job_ids ) == str:
job_ids = [ job_ids ]
job_ids = [ int( id ) for id in job_ids ]
# Find each job, for security we (implicately) check that they are
# associated witha job in the current history.
jobs, warnings = self.__get_job_dict( trans )
# Create a mapping from hid to ( job_id, output_name )
hid_to_output_pair = {}
for job, datasets in jobs.iteritems():
for assoc_name, data in datasets:
hid_to_output_pair[ data.hid ] = ( job.id, assoc_name )
# Mapping from job ids to workflow step ids (0, 1, 2, ...)
job_id_to_step_id = dict( ( job_id, i ) for ( i, job_id ) in enumerate( job_ids ) )
# Workflow to populate
workflow = Workflow()
# Back-translate each job
jobs_by_id = dict( ( job.id, job ) for job in jobs.keys() )
for step_id, job_id in enumerate( job_ids ):
assert job_id in jobs_by_id, "Attempt to create workflow with job not connected to current history"
job = jobs_by_id[ job_id ]
tool = trans.app.toolbox.tools_by_id[ job.tool_id ]
param_values = job.get_param_values( trans.app )
associations = self.__cleanup_param_values( tool.inputs, param_values )
step = WorkflowStep()
step.id = step_id
step.tool_id = job.tool_id
step.tool_inputs = param_values
for other_hid, input_name in associations:
other_job_id, other_name = hid_to_output_pair[ other_hid ]
# Only create association if the associated output dataset
# is being included in this workflow
if other_job_id in job_id_to_step_id:
step.input_connections[input_name] = ( job_id_to_step_id[ other_job_id ], other_name )
else:
step.input_connections[input_name] = None
workflow.steps[ step_id ] = step
# Try to order the nodes
workflow.order_nodes()
# And let's try to set up some reasonable locations
levorder = workflow.order_nodes_levels()
base_pos = 2510
for i, steps_at_level in enumerate( levorder ):
for j, step_id in enumerate( steps_at_level ):
step = workflow.steps[step_id]
step.position = dict( top = ( base_pos + 120 * j ),
left = ( base_pos + 220 * i ) )
# Store it
stored = model.StoredWorkflow.get_by( user = user, name = workflow_name )
if stored is None:
stored = model.StoredWorkflow()
stored.user = user
stored.name = workflow_name
stored.encoded_value = simplejson.dumps( workflow.to_simple() )
stored.flush()
#
return trans.show_ok_message( "Workflow '%s' created.<br/><a target='_top' href='%s'>Click to load in workflow editor</a>"
% ( workflow_name, web.url_for( controller='workflow_editor', action=None, workflow_name=workflow_name ) ) )
@@ -11,11 +11,12 @@ class WorkflowEditor( BaseController ):
beta = True
@web.expose
def index( self, trans ):
def index( self, trans, workflow_name=None ):
user = trans.get_user()
if not user:
return trans.show_error_message( "Must be logged in to create or modify workflows" )
return trans.fill_template( "workflow_editor/index.mako" )
return trans.fill_template( "workflow_editor/index.mako",
workflow_name=workflow_name )
@web.expose
def canvas( self, trans ):
@@ -67,6 +68,7 @@ class WorkflowEditor( BaseController ):
decode_state( node, trans.app )
# Create workflow from json data
workflow = Workflow.from_simple( data )
workflow.order_nodes()
# Store it
stored = model.StoredWorkflow.get_by( user = user, name = workflow_name )
if stored is None:
+24 -12
View File
@@ -1,5 +1,5 @@
from galaxy.tools import DefaultToolState
from galaxy.util.topsort import topsort, CycleError
from galaxy.util.topsort import topsort, topsort_levels, CycleError
class Workflow( object ):
@@ -21,7 +21,6 @@ class Workflow( object ):
if step.has_errors:
workflow.has_errors = True
workflow.steps[ id ] = step
workflow.node_order = workflow.order_nodes()
return workflow
def to_simple( self ):
@@ -35,27 +34,40 @@ class Workflow( object ):
has_cycles=self.has_cycles,
has_errors=self.has_errors )
def edge_list( self ):
edges = []
all_ids = set( self.steps.keys() )
for step in self.steps.values():
edges.append( ( step.id, step.id ) )
for name, conn in step.input_connections.iteritems():
if conn is not None:
other_node_id, _ = conn
edges.append( ( other_node_id, step.id ) )
return edges
def order_nodes( self ):
"""
Perform topological sort of the steps, return an ordered list of ids
"""
self.has_cycles = False
edges = []
all_ids = set( self.steps.keys() )
for step in self.steps.values():
for name, conn in step.input_connections.iteritems():
if conn is not None:
other_node_id, _ = conn
edges.append( ( other_node_id, step.id ) )
edges = self.edge_list()
try:
node_order = topsort( edges )
node_order_set = set( node_order )
node_order.extend( [ id for id in all_ids if id not in node_order ] )
#node_order_set = set( node_order )
#node_order.extend( [ id for id in all_ids if id not in node_order ] )
self.node_order = node_order
return self.node_order
except CycleError:
self.has_cycles = True
self.node_order = None
return self.node_order
return None
def order_nodes_levels( self ):
edges = self.edge_list()
try:
return topsort_levels( edges )
except CycleError:
return None
class WorkflowStep( object ):
+16
View File
@@ -76,6 +76,15 @@ div.toolFormBody div.toolFormTitle
margin-bottom: 5px;
}
div.toolFormDisabled div.toolFormTitle {
background: ${layout_bg};
border-color: ${layout_border};
}
div.toolFormDisabled {
border-color: ${layout_border};
}
div.toolHelp
{
}
@@ -86,6 +95,13 @@ div.toolHelpBody
overflow: auto;
}
div.titleRow {
font-weight: bold;
border-bottom: dotted gray 1px;
margin-bottom: 0.5em;
padding-bottom: 0.25em;
}
/* Forms */
div.form-row
+18 -9
View File
@@ -76,6 +76,15 @@ div.toolFormBody div.toolFormTitle
margin-bottom: 5px;
}
div.toolFormDisabled div.toolFormTitle {
background: #eee;
border-color: #999;
}
div.toolFormDisabled {
border-color: #999;
}
div.toolHelp
{
}
@@ -86,6 +95,13 @@ div.toolHelpBody
overflow: auto;
}
div.titleRow {
font-weight: bold;
border-bottom: dotted gray 1px;
margin-bottom: 0.5em;
padding-bottom: 0.25em;
}
/* Forms */
div.form-row
@@ -129,7 +145,7 @@ select, textarea, input[type="text"], input[type="file"]
/* Messages */
.errormessage, .warningmessage, .donemessage, .infomessage, .welcomeBlue, .welcomeRed
.errormessage, .warningmessage, .donemessage, .infomessage, .welcomemessage
{
padding: 10px;
padding-left: 52px;
@@ -162,20 +178,13 @@ select, textarea, input[type="text"], input[type="file"]
background-color: #CCCCFF;
}
.welcomeBlue {
.welcomemessage {
padding-left: 10px;
border-color: #6666AA;
background-color: #CCCCFF;
background-image: none;
}
.welcomeRed {
padding-left: 10px;
border-color: #AA6666;
background-color: #FFCCCC;
background-image: none;
}
.errormessagesmall, .warningmessagesmall, .donemessagesmall, .infomessagesmall
{
padding: 5px;
Binary file not shown.

Before

Width:  |  Height:  |  Size: 232 B

After

Width:  |  Height:  |  Size: 232 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 168 B

After

Width:  |  Height:  |  Size: 168 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 334 B

After

Width:  |  Height:  |  Size: 334 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 178 B

After

Width:  |  Height:  |  Size: 178 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 207 B

After

Width:  |  Height:  |  Size: 207 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 169 B

After

Width:  |  Height:  |  Size: 169 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 128 B

After

Width:  |  Height:  |  Size: 128 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 624 B

After

Width:  |  Height:  |  Size: 624 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 168 B

After

Width:  |  Height:  |  Size: 168 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 144 B

After

Width:  |  Height:  |  Size: 144 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 412 B

After

Width:  |  Height:  |  Size: 412 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 169 B

After

Width:  |  Height:  |  Size: 169 B

@@ -0,0 +1,130 @@
<!-- -->
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
<title>Galaxy</title>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
<link href="${h.url_for('/static/style/history.css')}" rel="stylesheet" type="text/css" />
<link href="${h.url_for('/static/style/base.css')}" rel="stylesheet" type="text/css" />
<style type="text/css">
div.toolForm{
margin-top: 10px;
margin-bottom: 10px;
}
div.historyItem {
margin-right: 0;
}
th {
border-bottom: solid black 1px;
}
</style>
<script type="text/javascript">
## Always try to hide the history pane since it is redundant on this page
if ( window.parent && window.parent.handle_minwidth_hint ) {
window.parent.handle_minwidth_hint( 10000 );
}
</script>
</head>
<body>
<%def name="history_item( data )">
%if data.state in [ "no state", "", None ]:
<% data_state = "queued" %>
%else:
<% data_state = data.state %>
%endif
<div class="historyItemWrapper historyItem historyItem-${data_state}" id="historyItem-$data.id">
<table cellpadding="0" cellspacing="0" border="0" width="100%">
<tr>
%if data_state != 'ok':
<td style="width: 20px;">
<div style='padding-right: 5px;'><img src="${h.url_for( '/static/style/data_%s.png' % data_state )}" border="0" align="middle"></div>
</td>
%endif
<td>
<div style="overflow: hidden;">
<span class="historyItemTitle"><b>${data.hid}: ${data.display_name()}</b></span>
</div>
</td>
</tr>
</table>
</div>
</%def>
<p>The following list contains each tool that was run to create the
datasets in your current history. Please select those that you wish
to include in the workflow.</p>
<p>Tools which cannot be run interactively and thus cannot be incorporated
into a workflow will be shown in gray.</p>
%for warning in warnings:
<div class="warningmark">${warning}</div>
%endfor
<form method="post" action="${h.url_for()}">
<div class='form-row'>
<label>Workflow name</label>
<input name="workflow_name" type="text" value="Unnamed workflow" />
</div>
<p><input type="submit" value="Create Workflow" /></p>
<table border="0" cellspacing="0">
<tr>
<th style="width: 47.5%">Tool</th>
<th style="width: 5%"></th>
<th style="width: 47.5%">History items created</th>
</tr>
%for job, datasets in jobs.iteritems():
<%
tool = app.toolbox.tools_by_id[ job.tool_id ]
cls = "toolForm"
if not( tool.is_workflow_compatible ):
cls += " toolFormDisabled"
disabled = True
else:
disabled = False
%>
<tr>
<td>
<div class="${cls}">
<div class="toolFormTitle">${tool.name}</div>
<div class="toolFormBody">
%if disabled:
<div style="font-style: italic; color: gray">This tool cannot be used in workflows</div>
%else:
<div><input type="checkbox" name="job_ids" value="${job.id}" checked="true" />Include "${tool.name}" in workflow</div>
%endif
</div>
</div>
</td>
<td style="text-align: center;">
&#x25B6;
</td>
<td>
%for _, data in datasets:
<div>${history_item( data )}</div>
%endfor
</td>
</tr>
%endfor
</table>
</form>
</body>
</html>
+5 -2
View File
@@ -33,9 +33,12 @@ q( "a[@confirm]" ).click( function() {
#if $len( $history.active_datasets ) > 0 and $history.user not in [ None , "" ]:
<li><a href="$h.url_for('/history_new')">Create</a> a new empty history</li>
#end if
#if $app.config.enable_beta_features
<li><a href="${h.url_for( controller='history', action='workflow_from_current_history' )}">Construct workflow</a> from the current history</li>
#end if
#end if
<li><a href="$h.url_for('/history_share')" target="galaxy_main">share</a> current history</div>
<li><a href="$h.url_for('/history_delete', id=$history.id )" confirm="Are you sure you want to delete the current history?">delete</a> current history</div>
<li><a href="$h.url_for('/history_share')" target="galaxy_main">Share</a> current history</div>
<li><a href="$h.url_for('/history_delete', id=$history.id )" confirm="Are you sure you want to delete the current history?">Delete</a> current history</div>
</ul>
</body>
+5
View File
@@ -19,6 +19,11 @@ if ( parent.frames && parent.frames.galaxy_history ) {
parent.frames.galaxy_history.location.href="$h.url_for('/history')";
}
#end if
if ( parent.handle_minwidth_hint )
{
parent.handle_minwidth_hint( -1 );
}
</script>
<body>
+62 -23
View File
@@ -11,6 +11,13 @@
#center { right: 309px; }
#right-border { right: 300px; }
#right { width: 300px; }
## /* Relative masthead size */
## #masthead { height: 2.5em; }
## #masthead div.title { font-size: 1.8em; }
## #left, #left-border, #center, #right-border, #right {
## top: 2.5em;
## margin-top: 7px;
## }
</style>
<script type='text/javascript' src="/static/scripts/jquery.js"> </script>
@@ -119,7 +126,25 @@ $( function() {
});
function notify() {
%if workflow_name:
$.ajax( {
url: "${h.url_for( action='load_workflow' )}",
data: { workflow_name: "${workflow_name}" },
dataType: 'json',
success: function( data ) {
window.frames.canvas.reset();
workflow.from_simple( data );
show_modal( "Workflow loaded", "Workflow loaded.", {
"Ok" : function () { hide_modal(); }
});
},
beforeSubmit: function( data ) {
show_modal( "Loading workflow", "progress" );
}
});
%else:
hide_modal();
%endif
};
function show_form_for_tool( text, node ) {
@@ -149,6 +174,10 @@ function show_form_for_tool( text, node ) {
var save_current_workflow = function () {
var body = $("#save-dialog-form").clone();
if ( workflow.name ) {
body.find( "input[name='workflow_name']" ).get(0).value = workflow.name;
}
body.find( "input[name='workflow_name']" ).get(0).focus();
var form = body.find( "form" ).ajaxForm( {
dataType: 'json',
success: function( data ) {
@@ -261,6 +290,16 @@ div.toolTitle {
display: list-item;
list-style: square outside;
}
div.toolTitleDisabled {
padding-top: 5px;
padding-bottom: 5px;
margin-left: 16px;
margin-right: 10px;
display: list-item;
list-style: square outside;
font-style: italic;
color: gray;
}
div.toolFormRow {
position: relative;
}
@@ -279,13 +318,6 @@ div.toolFormRow {
z-index: 50000;
}
div.titleRow {
font-weight: bold;
border-bottom: dotted gray 1px;
margin-bottom: 0.5em;
padding-bottom: 0.25em;
}
</style>
</head>
@@ -334,22 +366,29 @@ div.titleRow {
<div class="toolSectionBg">
%for tool in section.tools:
%if not tool.hidden:
<div class="toolTitle">
## #if $tool.input_required
## #set $link = $h.url_for( 'tool_runner', tool_id=$tool.id )
## #else
## #set $link = $h.url_for( $tool.action, ** $tool.get_static_param_values( $t ) )
## #end if
%if "[[" in tool.description and "]]" in tool.description:
${tool.description.replace( '[[', '<a href="javascript:add_node_for_tool( ${tool.id} )">' % tool.id ).replace( "]]", "</a>" )}
%elif tool.name:
<a id="link-${tool.id}" href="javascript:add_node_for_tool( '${tool.id}', '${tool.name}' )">${tool.name}</a> ${tool.description}
%else:
<a id="link-${tool.id}" href="javascript:add_node_for_tool( '${tool.id}', '${tool.name}' )">${tool.description}</a>
%endif
</div>
%if tool.is_workflow_compatible:
<div class="toolTitle ">
%if "[[" in tool.description and "]]" in tool.description:
${tool.description.replace( '[[', '<a id="link-${tool.id}" href="javascript:add_node_for_tool( ${tool.id} )">' % tool.id ).replace( "]]", "</a>" )}
%elif tool.name:
<a id="link-${tool.id}" href="javascript:add_node_for_tool( '${tool.id}', '${tool.name}' )">${tool.name}</a> ${tool.description}
%else:
<a id="link-${tool.id}" href="javascript:add_node_for_tool( '${tool.id}', '${tool.name}' )">${tool.description}</a>
%endif
</div>
%else:
<div class="toolTitleDisabled">
%if "[[" in tool.description and "]]" in tool.description:
${tool.description.replace( '[[', '' % tool.id ).replace( "]]", "" )}
%elif tool.name:
${tool.name} ${tool.description}
%else:
${tool.description}
%endif
</div>
%endif
%endif
%endfor
%endfor
</div>
</div>
%endfor
@@ -378,7 +417,7 @@ div.titleRow {
<div class="center-block-inner">Details</div>
</div>
</div>
<div class="unified-panel-body">
<div class="unified-panel-body" style="overflow: scroll;">
<div id="right-content"></div>
</div>
</div>