opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Workflow Output Tagging

Browse Source
This commit is contained in:
Dannon Baker
2010-08-19 23:18:56 -04:00
parent f6a3996d9b
commit 6803dea808
10 changed files with 375 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/model/__init__.py
+6 -1
View File
@@ -1328,7 +1328,7 @@ class CloudUserCredentials( object ):
self.accessKey = None
self.secretKey = None
self.credentials = []
class StoredWorkflow( object ):
def __init__( self ):
self.id = None
@@ -1364,6 +1364,11 @@ class WorkflowStepConnection( object ):
self.output_name = None
self.input_step_id = None
self.input_name = None
class WorkflowOutput(object):
def __init__( self, workflow_step, output_name):
self.workflow_step = workflow_step
self.output_name = output_name
class StoredWorkflowUserShareAssociation( object ):
def __init__( self ):
lib/galaxy/model/mapping.py
+13 -1
View File
@@ -463,6 +463,12 @@ WorkflowStepConnection.table = Table( "workflow_step_connection", metadata,
Column( "input_name", TEXT)
)
WorkflowOutput.table = Table( "workflow_output", metadata,
Column( "id", Integer, primary_key=True ),
Column( "workflow_step_id", Integer, ForeignKey("workflow_step.id"), index=True, nullable=False),
Column( "output_name", String(255), nullable=True)
)
WorkflowInvocation.table = Table( "workflow_invocation", metadata,
Column( "id", Integer, primary_key=True ),
Column( "create_time", DateTime, default=now ),
@@ -1238,7 +1244,10 @@ assign_mapper( context, Workflow, Workflow.table,
properties=dict( steps=relation( WorkflowStep, backref='workflow',
order_by=asc(WorkflowStep.table.c.order_index),
cascade="all, delete-orphan",
lazy=False )
lazy=False ),
# outputs = relation( WorkflowOutput, backref='workflow',
# primaryjoin=(Workflow.table.c.id == WorkflowStep.table.c.workflow_id),
# secondaryjoin=(WorkflowStep.table.c.id == WorkflowOutput.table.c.workflow_step_id))
) )
assign_mapper( context, WorkflowStep, WorkflowStep.table,
@@ -1247,6 +1256,9 @@ assign_mapper( context, WorkflowStep, WorkflowStep.table,
annotations=relation( WorkflowStepAnnotationAssociation, order_by=WorkflowStepAnnotationAssociation.table.c.id, backref="workflow_steps" ) )
)
assign_mapper( context, WorkflowOutput, WorkflowOutput.table,
properties=dict(workflow_step = relation( WorkflowStep, backref='workflow_outputs', primaryjoin=(WorkflowStep.table.c.id == WorkflowOutput.table.c.workflow_step_id))))
assign_mapper( context, WorkflowStepConnection, WorkflowStepConnection.table,
properties=dict( input_step=relation( WorkflowStep, backref="input_connections", cascade="all",
primaryjoin=( WorkflowStepConnection.table.c.input_step_id == WorkflowStep.table.c.id ) ),
lib/galaxy/model/migrate/versions/0056_workflow_outputs.py
+39
View File
@@ -0,0 +1,39 @@
"""
Migration script to create tables for adding explicit workflow outputs.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import logging
logging.basicConfig( level=logging.DEBUG )
log = logging.getLogger( __name__ )
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
metadata = MetaData( migrate_engine )
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
WorkflowOutput_table = Table( "workflow_output", metadata,
Column( "id", Integer, primary_key=True ),
Column( "workflow_step_id", Integer, ForeignKey("workflow_step.id"), index=True, nullable=False),
Column( "output_name", String(255), nullable=True))
tables = [WorkflowOutput_table]
def upgrade():
print __doc__
metadata.reflect()
for table in tables:
try:
table.create()
except:
log.warn( "Failed to create table '%s', ignoring (might result in wrong schema)" % table.name )
def downgrade():
metadata.reflect()
for table in tables:
table.drop()
lib/galaxy/web/base/controller.py
-1
View File
@@ -1407,7 +1407,6 @@ class Admin( object ):
@web.expose
@web.require_admin
def jobs( self, trans, stop = [], stop_msg = None, cutoff = 180, job_lock = None, **kwd ):
# DBTODO admin job lock.
deleted = []
msg = None
status = None
lib/galaxy/web/controllers/workflow.py
+90 -3
View File
@@ -358,7 +358,6 @@ class WorkflowController( BaseController, Sharable, UsesStoredWorkflow, UsesAnno
stored = self.get_stored_workflow( trans, id )
if not stored:
error( "You do not own this workflow or workflow ID is invalid." )
# Update workflow attributes if new values submitted.
if 'name' in kwargs:
# Rename workflow.
@@ -368,7 +367,6 @@ class WorkflowController( BaseController, Sharable, UsesStoredWorkflow, UsesAnno
annotation = sanitize_html( kwargs[ 'annotation' ], 'utf-8', 'text/html' )
self.add_item_annotation( trans, stored, annotation )
trans.sa_session.flush()
return trans.fill_template( 'workflow/edit_attributes.mako',
stored=stored,
annotation=self.get_item_annotation_str( trans, trans.user, stored )
@@ -773,7 +771,9 @@ class WorkflowController( BaseController, Sharable, UsesStoredWorkflow, UsesAnno
'data_inputs': module.get_data_inputs(),
'data_outputs': module.get_data_outputs(),
'form_html': module.get_config_form(),
'annotation' : annotation_str
'annotation' : annotation_str,
'post_job_actions' : {},
'workflow_outputs' : []
}
# Connections
input_connections = step.input_connections
@@ -794,6 +794,11 @@ class WorkflowController( BaseController, Sharable, UsesStoredWorkflow, UsesAnno
output_name = pja.output_name,
action_arguments = pja.action_arguments)
step_dict['post_job_actions'] = pja_dict
#workflow outputs
outputs = []
for output in step.workflow_outputs:
outputs.append(output.output_name)
step_dict['workflow_outputs'] = outputs
# Encode input connections as dictionary
input_conn_dict = {}
for conn in input_connections:
@@ -838,6 +843,10 @@ class WorkflowController( BaseController, Sharable, UsesStoredWorkflow, UsesAnno
step.position = step_dict['position']
module = module_factory.from_dict( trans, step_dict )
module.save_to_step( step )
if step_dict.has_key('workflow_outputs'):
for output_name in step_dict['workflow_outputs']:
m = model.WorkflowOutput(workflow_step = step, output_name = output_name)
trans.sa_session.add(m)
if step.tool_errors:
workflow.has_errors = True
# Stick this in the step temporarily
@@ -1278,6 +1287,84 @@ class WorkflowController( BaseController, Sharable, UsesStoredWorkflow, UsesAnno
errors=errors,
incoming=kwargs )
@web.expose
def tag_outputs( self, trans, id, check_user=True, **kwargs ):
stored = self.get_stored_workflow( trans, id, check_ownership=False )
if check_user:
user = trans.get_user()
if stored.user != user:
if trans.sa_session.query( model.StoredWorkflowUserShareAssociation ) \
.filter_by( user=user, stored_workflow=stored ).count() == 0:
error( "Workflow is not owned by or shared with current user" )
# Get the latest revision
workflow = stored.latest_workflow
# It is possible for a workflow to have 0 steps
if len( workflow.steps ) == 0:
error( "Workflow cannot be tagged for outputs because it does not have any steps" )
if workflow.has_cycles:
error( "Workflow cannot be tagged for outputs because it contains cycles" )
if workflow.has_errors:
error( "Workflow cannot be tagged for outputs because of validation errors in some steps" )
# Build the state for each step
errors = {}
has_upgrade_messages = False
has_errors = False
if kwargs:
# If kwargs were provided, the states for each step should have
# been POSTed
for step in workflow.steps:
if step.type == 'tool':
# Extract just the output flags for this step.
p = "%s|otag|" % step.id
l = len(p)
outputs = [k[l:] for ( k, v ) in kwargs.iteritems() if k.startswith( p )]
if step.workflow_outputs:
for existing_output in step.workflow_outputs:
if existing_output.output_name not in outputs:
# print "Deleting action %s on %s" % (step.id, existing_output.output_name)
trans.sa_session.delete(existing_output)
else:
outputs.remove(existing_output.output_name)
for outputname in outputs:
# print "Creating new workflow output at step %s on output name %s" %(step.id, outputname)
m = model.WorkflowOutput(workflow_step_id = int(step.id), output_name = outputname)
trans.sa_session.add(m)
# Prepare each step
trans.sa_session.flush()
for step in workflow.steps:
step.upgrade_messages = {}
# Contruct modules
if step.type == 'tool' or step.type is None:
# Restore the tool state for the step
step.module = module_factory.from_workflow_step( trans, step )
# Fix any missing parameters
step.upgrade_messages = step.module.check_and_update_state()
if step.upgrade_messages:
has_upgrade_messages = True
# Any connected input needs to have value DummyDataset (these
# are not persisted so we need to do it every time)
step.module.add_dummy_datasets( connections=step.input_connections )
# Store state with the step
step.state = step.module.state
# Error dict
if step.tool_errors:
has_errors = True
errors[step.id] = step.tool_errors
else:
## Non-tool specific stuff?
step.module = module_factory.from_workflow_step( trans, step )
step.state = step.module.get_runtime_state()
# Connections by input name
step.input_connections_by_name = dict( ( conn.input_name, conn ) for conn in step.input_connections )
# Render the form
return trans.fill_template(
"workflow/tag_outputs.mako",
steps=workflow.steps,
workflow=stored,
has_upgrade_messages=has_upgrade_messages,
errors=errors,
incoming=kwargs )
@web.expose
def configure_menu( self, trans, workflow_ids=None ):
user = trans.get_user()
lib/galaxy/workflow/modules.py
+5 -4
View File
@@ -165,6 +165,7 @@ class ToolModule( WorkflowModule ):
self.tool_id = tool_id
self.tool = trans.app.toolbox.tools_by_id[ tool_id ]
self.post_job_actions = {}
self.workflow_outputs = []
self.state = None
self.errors = None
@@ -181,10 +182,9 @@ class ToolModule( WorkflowModule ):
module.state = DefaultToolState()
module.state.decode( d["tool_state"], module.tool, module.trans.app, secure=secure )
module.errors = d.get( "tool_errors", None )
if 'post_job_actions' in d and d["post_job_actions"] != []:
module.post_job_actions = d["post_job_actions"]
else:
module.post_job_actions = {}
module.post_job_actions = d.get("post_job_actions", {})
module.workflow_outputs = d.get("workflow_outputs", [])
return module
@classmethod
@@ -195,6 +195,7 @@ class ToolModule( WorkflowModule ):
module.state.inputs = module.tool.params_from_strings( step.tool_inputs, trans.app, ignore_errors=True )
module.errors = step.tool_errors
# module.post_job_actions = step.post_job_actions
module.workflow_outputs = step.workflow_outputs
pjadict = {}
for pja in step.post_job_actions:
pjadict[pja.action_type] = pja
static/scripts/galaxy.workflow_editor.canvas.js
+7 -1
View File
@@ -275,6 +275,7 @@ $.extend( Node.prototype, {
this.tooltip = data.tooltip ? data.tooltip : ""
this.annotation = data.annotation;
this.post_job_actions = data.post_job_actions;
this.workflow_outputs = data.workflow_outputs
if ( this.tool_errors ) {
f.addClass( "tool-node-error" );
@@ -416,6 +417,10 @@ $.extend( Workflow.prototype, {
post_job_actions[ act.type + act.output_name ] = pja;
});
}
if (!node.workflow_outputs){
node.workflow_outputs = [];
// Just in case.
}
var node_data = {
id : node.id,
type : node.type,
@@ -425,7 +430,8 @@ $.extend( Workflow.prototype, {
input_connections : input_connections,
position : $(node.element).position(),
annotation: node.annotation,
post_job_actions: node.post_job_actions
post_job_actions: node.post_job_actions,
workflow_outputs: node.workflow_outputs
};
nodes[ node.id ] = node_data;
});
templates/workflow/editor.mako
+53 -6
View File
@@ -132,15 +132,51 @@
make_popupmenu( $("#workflow-options-button"), {
##"Create New" : create_new_workflow_dialog,
"Edit Attributes" : edit_workflow_attributes,
"Edit Workflow Outputs": edit_workflow_outputs,
"Layout": layout_editor,
"Save" : save_current_workflow,
##"Load a Workflow" : load_workflow,
"Close": close_editor,
"Hide Intermediate Steps": hide_intermediate_steps
});
function hide_intermediate_steps(){
function edit_workflow_outputs(){
workflow.clear_active_node();
$('.right-content').hide();
var new_content = "";
for (var node_key in workflow.nodes){
var node = workflow.nodes[node_key];
if(node.type == 'tool'){
new_content += "<div class='toolForm' style='margin-bottom:5px;'><div class='toolFormTitle'>Step " + node.id + " - " + node.name + "</div>";
for (var ot_key in node.output_terminals){
var output = node.output_terminals[ot_key];
// if (node.workflow_outputs[node.id + "|" + output.name]){
if ($.inArray(output.name, node.workflow_outputs) != -1){
new_content += "<p>"+output.name +"<input type='checkbox' name='"+ node.id + "|" + output.name +"' checked /></p>";
}
else{
new_content += "<p>"+output.name +"<input type='checkbox' name='"+ node.id + "|" + output.name +"' /></p>";
}
}
new_content += "</div>";
}
}
$("#output-fill-area").html(new_content);
$("#output-fill-area input").bind('click', function(){
var node_id = this.name.split('|')[0];
var output_name = this.name.split('|')[1];
if (this.checked){
if($.inArray(output_name, workflow.nodes[node_id].workflow_outputs) == -1){
workflow.nodes[node_id].workflow_outputs.push(output_name);
}//else it's already in the array. Shouldn't happen, but forget it.
}else{
while ($.inArray(output_name, workflow.nodes[node_id].workflow_outputs) != -1){
var ia = $.inArray(output_name, workflow.nodes[node_id].workflow_outputs);
workflow.nodes[node_id].workflow_outputs = workflow.nodes[node_id].workflow_outputs.slice(0,ia).concat( workflow.nodes[node_id].workflow_outputs.slice(ia+1) );
}
}
workflow.has_changes = true;
});
$('#workflow-output-area').show();
};
function layout_editor() {
@@ -152,8 +188,9 @@
function edit_workflow_attributes() {
workflow.clear_active_node();
$('.right-content').hide();
$('#edit-attributes').show();
$('#right-content').hide();
};
$.jStore.engineReady(function() {
@@ -315,7 +352,6 @@
from galaxy.jobs.actions.post import ActionBox
%>
// DBTODO Refactor to the post module.
// This function preloads how to display known pja's.
function display_pja(pja, node){
// DBTODO SANITIZE INPUTS.
@@ -891,7 +927,18 @@
</div>
</div>
</div>
## Div where tool details are loaded and modified.
## Div where tool details are loaded and modified.
<div id="right-content" class="right-content"></div>
## Workflow output tagging
<div style="display:none;" id="workflow-output-area" class="metadataForm right-content">
<div class="metadataFormTitle">Edit Workflow Outputs</div>
<div class="metadataFormBody"><div class="form-row">
<div class="toolParamHelp">Tag step outputs to indicate the final dataset(s) to be generated by running this workflow.</div>
<div id="output-fill-area"></div>
</div></div>
</div>
</div>
</%def>
templates/workflow/list.mako
+1
View File
@@ -58,6 +58,7 @@
<td>
<div popupmenu="wf-${i}-popup">
<a class="action-button" href="${h.url_for( action='editor', id=trans.security.encode_id(workflow.id) )}" target="_parent">Edit</a>
<a class="action-button" href="${h.url_for( action='tag_outputs', id=trans.security.encode_id(workflow.id) )}">Tag Outputs</a>
<a class="action-button" href="${h.url_for( controller='root', action='index', workflow_id=trans.security.encode_id(workflow.id) )}" target="_parent">Run</a>
<a class="action-button" href="${h.url_for( action='sharing', id=trans.security.encode_id(workflow.id) )}">Share or Publish</a>
<a class="action-button" href="${h.url_for( action='clone', id=trans.security.encode_id(workflow.id) )}">Clone</a>
templates/workflow/tag_outputs.mako
+161
View File
@@ -0,0 +1,161 @@
<%inherit file="/base.mako"/>
<%def name="javascripts()">
${parent.javascripts()}
${h.js( "jquery.autocomplete" )}
<script type="text/javascript">
$( function() {
$( "select[refresh_on_change='true']").change( function() {
$( "#tool_form" ).submit();
});
});
</script>
</%def>
<%def name="stylesheets()">
${parent.stylesheets()}
${h.css( "autocomplete_tagging" )}
<style type="text/css">
div.toolForm{
margin-top: 10px;
margin-bottom: 10px;
}
</style>
</%def>
<%
from galaxy.tools.parameters import DataToolParameter, RuntimeValue
from galaxy.jobs.actions.post import ActionBox
%>
<%def name="do_inputs( inputs, values, errors, prefix, step, other_values = None )">
%if other_values is None:
<% other_values = values %>
%endif
%for input_index, input in enumerate( inputs.itervalues() ):
%if input.type == "repeat":
<div class="repeat-group">
<div class="form-title-row"><b>${input.title_plural}</b></div>
<% repeat_values = values[input.name] %>
%for i in range( len( repeat_values ) ):
%if input.name in errors:
<% rep_errors = errors[input.name][i] %>
%else:
<% rep_errors = dict() %>
%endif
<div class="repeat-group-item">
<% index = repeat_values[i]['__index__'] %>
<div class="form-title-row"><b>${input.title} ${i + 1}</b></div>
${do_inputs( input.inputs, repeat_values[ i ], rep_errors, prefix + input.name + "_" + str(index) + "|", step, other_values )}
## <div class="form-row"><input type="submit" name="${step.id}|${prefix}${input.name}_${i}_remove" value="Remove ${input.title} ${i+1}" /></div>
</div>
%endfor
## <div class="form-row"><input type="submit" name="${step.id}|${prefix}${input.name}_add" value="Add new ${input.title}" /></div>
</div>
%elif input.type == "conditional":
<% group_values = values[input.name] %>
<% current_case = group_values['__current_case__'] %>
<% new_prefix = prefix + input.name + "|" %>
<% group_errors = errors.get( input.name, {} ) %>
${row_for_param( input.test_param, group_values[ input.test_param.name ], other_values, group_errors, prefix, step )}
${do_inputs( input.cases[ current_case ].inputs, group_values, group_errors, new_prefix, step, other_values )}
%else:
${row_for_param( input, values[ input.name ], other_values, errors, prefix, step )}
%endif
%endfor
</%def>
<%def name="row_for_param( param, value, other_values, error_dict, prefix, step )">
## -- ${param.name} -- ${step.state.inputs} --
%if error_dict.has_key( param.name ):
<% cls = "form-row form-row-error" %>
%else:
<% cls = "form-row" %>
%endif
<div class="${cls}">
<label>${param.get_label()}</label>
<div>
%if isinstance( param, DataToolParameter ):
%if ( prefix + param.name ) in step.input_connections_by_name:
<%
conn = step.input_connections_by_name[ prefix + param.name ]
%>
Output dataset '${conn.output_name}' from step ${int(conn.output_step.order_index)+1}
%else:
## FIXME: Initialize in the controller
<%
if value is None:
value = other_values[ param.name ] = param.get_initial_value( t, other_values )
%>
${param.get_html_field( t, value, other_values ).get_html( str(step.id) + "|" + prefix )}
<input type="hidden" name="${step.id}|__force_update__${prefix}${param.name}" value="true" />
%endif
%elif isinstance( value, RuntimeValue ) or ( str(step.id) + '|__runtime__' + prefix + param.name ) in incoming:
## On the first load we may see a RuntimeValue, so we write
## an input field using the initial value for the param.
## Subsequents posts will no longer have the runtime value
## (since an actualy value will be posted) so we add a hidden
## field so we know to continue drawing form for this param.
## FIXME: This logic shouldn't be in the template. The
## controller should go through the inputs on the first
## load, fill in initial values where needed, and mark
## all that are runtime modifiable in some way.
<% value = other_values[ param.name ] = param.get_initial_value( t, other_values ) %>
${param.get_html_field( t, value, other_values ).get_html( str(step.id) + "|" + prefix )}
<input type="hidden" name="${step.id}|__runtime__${prefix}${param.name}" value="true" />
%else:
${param.value_to_display_text( value, app )}
%endif
</div>
%if step.upgrade_messages and param.name in step.upgrade_messages:
<div class="warningmark">${step.upgrade_messages[param.name]}</div>
%endif
%if error_dict.has_key( param.name ):
<div style="color: red; font-weight: bold; padding-top: 1px; padding-bottom: 3px;">
<div style="width: 300px;"><img style="vertical-align: middle;" src="${h.url_for('/static/style/error_small.png')}">&nbsp;<span style="vertical-align: middle;">${error_dict[param.name]}</span></div>
</div>
%endif
<div style="clear: both"></div>
</div>
</%def>
<h2>Set outputs for workflow "${workflow.name}"</h2>
<p>Select the checkboxes for step outputs you would like to mark as whole-workflow outputs.</p>
%if has_upgrade_messages:
<div class="warningmessage">
Problems were encourered when loading this workflow, likely due to tool
version changes. Missing parameter values have been replaced with default.
Please review the parameter values below.
</div>
%endif
<form id="tool_form" name="tool_form" method="POST">
## <input type="hidden" name="workflow_name" value="${workflow.name | h}" />
%for i, step in enumerate( steps ):
%if step.type == 'tool':
<% tool = app.toolbox.tools_by_id[step.tool_id] %>
<input type="hidden" name="${step.id}|tool_state" value="${step.state.encode( tool, app )}">
<div class="toolForm">
<div class="toolFormTitle">Step ${int(step.order_index)+1}: ${tool.name}</div>
<div class="toolFormBody">
% for output in tool.outputs:
<div class='form-row'>
%if step.workflow_outputs and output in [wf_output.output_name for wf_output in step.workflow_outputs]:
<p>${output} <input type="checkbox" name="${step.id}|otag|${output}" checked /></p>
%else:
<p>${output} <input type="checkbox" name="${step.id}|otag|${output}"/></p>
%endif
</div>
% endfor
% if step.annotations:
<hr/>
<div class='form-row'>
<label>Annotation:</label> ${step.annotations[0].annotation}
</div>
% endif
</div>
</div>
%endif
%endfor
<input type="submit" name="save_outputs" value="Save output settings" />
</form>