From 6803dea808f4177e75a4165ebda282df00359e5c Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 19 Aug 2010 23:18:56 -0400 Subject: [PATCH] Workflow Output Tagging --- lib/galaxy/model/__init__.py | 7 +- lib/galaxy/model/mapping.py | 14 +- .../migrate/versions/0056_workflow_outputs.py | 39 +++++ lib/galaxy/web/base/controller.py | 1 - lib/galaxy/web/controllers/workflow.py | 93 +++++++++- lib/galaxy/workflow/modules.py | 9 +- .../scripts/galaxy.workflow_editor.canvas.js | 8 +- templates/workflow/editor.mako | 59 ++++++- templates/workflow/list.mako | 1 + templates/workflow/tag_outputs.mako | 161 ++++++++++++++++++ 10 files changed, 375 insertions(+), 17 deletions(-) create mode 100644 lib/galaxy/model/migrate/versions/0056_workflow_outputs.py create mode 100644 templates/workflow/tag_outputs.mako diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 53794e8b186..5c503787789 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1328,7 +1328,7 @@ class CloudUserCredentials( object ): self.accessKey = None self.secretKey = None self.credentials = [] - + class StoredWorkflow( object ): def __init__( self ): self.id = None @@ -1364,6 +1364,11 @@ class WorkflowStepConnection( object ): self.output_name = None self.input_step_id = None self.input_name = None + +class WorkflowOutput(object): + def __init__( self, workflow_step, output_name): + self.workflow_step = workflow_step + self.output_name = output_name class StoredWorkflowUserShareAssociation( object ): def __init__( self ): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index f5d973a513e..28c15f4c97f 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -463,6 +463,12 @@ WorkflowStepConnection.table = Table( "workflow_step_connection", metadata, Column( "input_name", TEXT) ) +WorkflowOutput.table = Table( "workflow_output", metadata, + Column( "id", Integer, primary_key=True ), + Column( "workflow_step_id", Integer, ForeignKey("workflow_step.id"), index=True, nullable=False), + Column( "output_name", String(255), nullable=True) + ) + WorkflowInvocation.table = Table( "workflow_invocation", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), @@ -1238,7 +1244,10 @@ assign_mapper( context, Workflow, Workflow.table, properties=dict( steps=relation( WorkflowStep, backref='workflow', order_by=asc(WorkflowStep.table.c.order_index), cascade="all, delete-orphan", - lazy=False ) + lazy=False ), + # outputs = relation( WorkflowOutput, backref='workflow', + # primaryjoin=(Workflow.table.c.id == WorkflowStep.table.c.workflow_id), + # secondaryjoin=(WorkflowStep.table.c.id == WorkflowOutput.table.c.workflow_step_id)) ) ) assign_mapper( context, WorkflowStep, WorkflowStep.table, @@ -1247,6 +1256,9 @@ assign_mapper( context, WorkflowStep, WorkflowStep.table, annotations=relation( WorkflowStepAnnotationAssociation, order_by=WorkflowStepAnnotationAssociation.table.c.id, backref="workflow_steps" ) ) ) +assign_mapper( context, WorkflowOutput, WorkflowOutput.table, + properties=dict(workflow_step = relation( WorkflowStep, backref='workflow_outputs', primaryjoin=(WorkflowStep.table.c.id == WorkflowOutput.table.c.workflow_step_id)))) + assign_mapper( context, WorkflowStepConnection, WorkflowStepConnection.table, properties=dict( input_step=relation( WorkflowStep, backref="input_connections", cascade="all", primaryjoin=( WorkflowStepConnection.table.c.input_step_id == WorkflowStep.table.c.id ) ), diff --git a/lib/galaxy/model/migrate/versions/0056_workflow_outputs.py b/lib/galaxy/model/migrate/versions/0056_workflow_outputs.py new file mode 100644 index 00000000000..1f32ff4b64a --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0056_workflow_outputs.py @@ -0,0 +1,39 @@ +""" +Migration script to create tables for adding explicit workflow outputs. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +import logging +logging.basicConfig( level=logging.DEBUG ) +log = logging.getLogger( __name__ ) + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +WorkflowOutput_table = Table( "workflow_output", metadata, + Column( "id", Integer, primary_key=True ), + Column( "workflow_step_id", Integer, ForeignKey("workflow_step.id"), index=True, nullable=False), + Column( "output_name", String(255), nullable=True)) + +tables = [WorkflowOutput_table] + +def upgrade(): + print __doc__ + metadata.reflect() + for table in tables: + try: + table.create() + except: + log.warn( "Failed to create table '%s', ignoring (might result in wrong schema)" % table.name ) + +def downgrade(): + metadata.reflect() + for table in tables: + table.drop() \ No newline at end of file diff --git a/lib/galaxy/web/base/controller.py b/lib/galaxy/web/base/controller.py index f7e3a221b1e..cc1ab1cce57 100644 --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -1407,7 +1407,6 @@ class Admin( object ): @web.expose @web.require_admin def jobs( self, trans, stop = [], stop_msg = None, cutoff = 180, job_lock = None, **kwd ): - # DBTODO admin job lock. deleted = [] msg = None status = None diff --git a/lib/galaxy/web/controllers/workflow.py b/lib/galaxy/web/controllers/workflow.py index 07ef6b21189..0e93ca783fe 100644 --- a/lib/galaxy/web/controllers/workflow.py +++ b/lib/galaxy/web/controllers/workflow.py @@ -358,7 +358,6 @@ class WorkflowController( BaseController, Sharable, UsesStoredWorkflow, UsesAnno stored = self.get_stored_workflow( trans, id ) if not stored: error( "You do not own this workflow or workflow ID is invalid." ) - # Update workflow attributes if new values submitted. if 'name' in kwargs: # Rename workflow. @@ -368,7 +367,6 @@ class WorkflowController( BaseController, Sharable, UsesStoredWorkflow, UsesAnno annotation = sanitize_html( kwargs[ 'annotation' ], 'utf-8', 'text/html' ) self.add_item_annotation( trans, stored, annotation ) trans.sa_session.flush() - return trans.fill_template( 'workflow/edit_attributes.mako', stored=stored, annotation=self.get_item_annotation_str( trans, trans.user, stored ) @@ -773,7 +771,9 @@ class WorkflowController( BaseController, Sharable, UsesStoredWorkflow, UsesAnno 'data_inputs': module.get_data_inputs(), 'data_outputs': module.get_data_outputs(), 'form_html': module.get_config_form(), - 'annotation' : annotation_str + 'annotation' : annotation_str, + 'post_job_actions' : {}, + 'workflow_outputs' : [] } # Connections input_connections = step.input_connections @@ -794,6 +794,11 @@ class WorkflowController( BaseController, Sharable, UsesStoredWorkflow, UsesAnno output_name = pja.output_name, action_arguments = pja.action_arguments) step_dict['post_job_actions'] = pja_dict + #workflow outputs + outputs = [] + for output in step.workflow_outputs: + outputs.append(output.output_name) + step_dict['workflow_outputs'] = outputs # Encode input connections as dictionary input_conn_dict = {} for conn in input_connections: @@ -838,6 +843,10 @@ class WorkflowController( BaseController, Sharable, UsesStoredWorkflow, UsesAnno step.position = step_dict['position'] module = module_factory.from_dict( trans, step_dict ) module.save_to_step( step ) + if step_dict.has_key('workflow_outputs'): + for output_name in step_dict['workflow_outputs']: + m = model.WorkflowOutput(workflow_step = step, output_name = output_name) + trans.sa_session.add(m) if step.tool_errors: workflow.has_errors = True # Stick this in the step temporarily @@ -1278,6 +1287,84 @@ class WorkflowController( BaseController, Sharable, UsesStoredWorkflow, UsesAnno errors=errors, incoming=kwargs ) + @web.expose + def tag_outputs( self, trans, id, check_user=True, **kwargs ): + stored = self.get_stored_workflow( trans, id, check_ownership=False ) + if check_user: + user = trans.get_user() + if stored.user != user: + if trans.sa_session.query( model.StoredWorkflowUserShareAssociation ) \ + .filter_by( user=user, stored_workflow=stored ).count() == 0: + error( "Workflow is not owned by or shared with current user" ) + # Get the latest revision + workflow = stored.latest_workflow + # It is possible for a workflow to have 0 steps + if len( workflow.steps ) == 0: + error( "Workflow cannot be tagged for outputs because it does not have any steps" ) + if workflow.has_cycles: + error( "Workflow cannot be tagged for outputs because it contains cycles" ) + if workflow.has_errors: + error( "Workflow cannot be tagged for outputs because of validation errors in some steps" ) + # Build the state for each step + errors = {} + has_upgrade_messages = False + has_errors = False + if kwargs: + # If kwargs were provided, the states for each step should have + # been POSTed + for step in workflow.steps: + if step.type == 'tool': + # Extract just the output flags for this step. + p = "%s|otag|" % step.id + l = len(p) + outputs = [k[l:] for ( k, v ) in kwargs.iteritems() if k.startswith( p )] + if step.workflow_outputs: + for existing_output in step.workflow_outputs: + if existing_output.output_name not in outputs: + # print "Deleting action %s on %s" % (step.id, existing_output.output_name) + trans.sa_session.delete(existing_output) + else: + outputs.remove(existing_output.output_name) + for outputname in outputs: + # print "Creating new workflow output at step %s on output name %s" %(step.id, outputname) + m = model.WorkflowOutput(workflow_step_id = int(step.id), output_name = outputname) + trans.sa_session.add(m) + # Prepare each step + trans.sa_session.flush() + for step in workflow.steps: + step.upgrade_messages = {} + # Contruct modules + if step.type == 'tool' or step.type is None: + # Restore the tool state for the step + step.module = module_factory.from_workflow_step( trans, step ) + # Fix any missing parameters + step.upgrade_messages = step.module.check_and_update_state() + if step.upgrade_messages: + has_upgrade_messages = True + # Any connected input needs to have value DummyDataset (these + # are not persisted so we need to do it every time) + step.module.add_dummy_datasets( connections=step.input_connections ) + # Store state with the step + step.state = step.module.state + # Error dict + if step.tool_errors: + has_errors = True + errors[step.id] = step.tool_errors + else: + ## Non-tool specific stuff? + step.module = module_factory.from_workflow_step( trans, step ) + step.state = step.module.get_runtime_state() + # Connections by input name + step.input_connections_by_name = dict( ( conn.input_name, conn ) for conn in step.input_connections ) + # Render the form + return trans.fill_template( + "workflow/tag_outputs.mako", + steps=workflow.steps, + workflow=stored, + has_upgrade_messages=has_upgrade_messages, + errors=errors, + incoming=kwargs ) + @web.expose def configure_menu( self, trans, workflow_ids=None ): user = trans.get_user() diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 40754422e0f..20d3d4d78d8 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -165,6 +165,7 @@ class ToolModule( WorkflowModule ): self.tool_id = tool_id self.tool = trans.app.toolbox.tools_by_id[ tool_id ] self.post_job_actions = {} + self.workflow_outputs = [] self.state = None self.errors = None @@ -181,10 +182,9 @@ class ToolModule( WorkflowModule ): module.state = DefaultToolState() module.state.decode( d["tool_state"], module.tool, module.trans.app, secure=secure ) module.errors = d.get( "tool_errors", None ) - if 'post_job_actions' in d and d["post_job_actions"] != []: - module.post_job_actions = d["post_job_actions"] - else: - module.post_job_actions = {} + module.post_job_actions = d.get("post_job_actions", {}) + module.workflow_outputs = d.get("workflow_outputs", []) + return module @classmethod @@ -195,6 +195,7 @@ class ToolModule( WorkflowModule ): module.state.inputs = module.tool.params_from_strings( step.tool_inputs, trans.app, ignore_errors=True ) module.errors = step.tool_errors # module.post_job_actions = step.post_job_actions + module.workflow_outputs = step.workflow_outputs pjadict = {} for pja in step.post_job_actions: pjadict[pja.action_type] = pja diff --git a/static/scripts/galaxy.workflow_editor.canvas.js b/static/scripts/galaxy.workflow_editor.canvas.js index 28021dda76d..ba30aa8276a 100644 --- a/static/scripts/galaxy.workflow_editor.canvas.js +++ b/static/scripts/galaxy.workflow_editor.canvas.js @@ -275,6 +275,7 @@ $.extend( Node.prototype, { this.tooltip = data.tooltip ? data.tooltip : "" this.annotation = data.annotation; this.post_job_actions = data.post_job_actions; + this.workflow_outputs = data.workflow_outputs if ( this.tool_errors ) { f.addClass( "tool-node-error" ); @@ -416,6 +417,10 @@ $.extend( Workflow.prototype, { post_job_actions[ act.type + act.output_name ] = pja; }); } + if (!node.workflow_outputs){ + node.workflow_outputs = []; + // Just in case. + } var node_data = { id : node.id, type : node.type, @@ -425,7 +430,8 @@ $.extend( Workflow.prototype, { input_connections : input_connections, position : $(node.element).position(), annotation: node.annotation, - post_job_actions: node.post_job_actions + post_job_actions: node.post_job_actions, + workflow_outputs: node.workflow_outputs }; nodes[ node.id ] = node_data; }); diff --git a/templates/workflow/editor.mako b/templates/workflow/editor.mako index f6dac6080d4..cf0f0bdd997 100644 --- a/templates/workflow/editor.mako +++ b/templates/workflow/editor.mako @@ -132,15 +132,51 @@ make_popupmenu( $("#workflow-options-button"), { ##"Create New" : create_new_workflow_dialog, "Edit Attributes" : edit_workflow_attributes, + "Edit Workflow Outputs": edit_workflow_outputs, "Layout": layout_editor, "Save" : save_current_workflow, ##"Load a Workflow" : load_workflow, "Close": close_editor, - "Hide Intermediate Steps": hide_intermediate_steps }); - function hide_intermediate_steps(){ - + function edit_workflow_outputs(){ + workflow.clear_active_node(); + $('.right-content').hide(); + var new_content = ""; + for (var node_key in workflow.nodes){ + var node = workflow.nodes[node_key]; + if(node.type == 'tool'){ + new_content += "
Step " + node.id + " - " + node.name + "
"; + for (var ot_key in node.output_terminals){ + var output = node.output_terminals[ot_key]; + // if (node.workflow_outputs[node.id + "|" + output.name]){ + if ($.inArray(output.name, node.workflow_outputs) != -1){ + new_content += "

"+output.name +"

"; + } + else{ + new_content += "

"+output.name +"

"; + } + } + new_content += "
"; + } + } + $("#output-fill-area").html(new_content); + $("#output-fill-area input").bind('click', function(){ + var node_id = this.name.split('|')[0]; + var output_name = this.name.split('|')[1]; + if (this.checked){ + if($.inArray(output_name, workflow.nodes[node_id].workflow_outputs) == -1){ + workflow.nodes[node_id].workflow_outputs.push(output_name); + }//else it's already in the array. Shouldn't happen, but forget it. + }else{ + while ($.inArray(output_name, workflow.nodes[node_id].workflow_outputs) != -1){ + var ia = $.inArray(output_name, workflow.nodes[node_id].workflow_outputs); + workflow.nodes[node_id].workflow_outputs = workflow.nodes[node_id].workflow_outputs.slice(0,ia).concat( workflow.nodes[node_id].workflow_outputs.slice(ia+1) ); + } + } + workflow.has_changes = true; + }); + $('#workflow-output-area').show(); }; function layout_editor() { @@ -152,8 +188,9 @@ function edit_workflow_attributes() { workflow.clear_active_node(); + $('.right-content').hide(); $('#edit-attributes').show(); - $('#right-content').hide(); + }; $.jStore.engineReady(function() { @@ -315,7 +352,6 @@ from galaxy.jobs.actions.post import ActionBox %> - // DBTODO Refactor to the post module. // This function preloads how to display known pja's. function display_pja(pja, node){ // DBTODO SANITIZE INPUTS. @@ -891,7 +927,18 @@ - ## Div where tool details are loaded and modified. + + ## Div where tool details are loaded and modified.
+ + ## Workflow output tagging + + diff --git a/templates/workflow/list.mako b/templates/workflow/list.mako index 04658093cb4..60f27657ac2 100644 --- a/templates/workflow/list.mako +++ b/templates/workflow/list.mako @@ -58,6 +58,7 @@
Edit + Tag Outputs Run Share or Publish Clone diff --git a/templates/workflow/tag_outputs.mako b/templates/workflow/tag_outputs.mako new file mode 100644 index 00000000000..b70713d02ed --- /dev/null +++ b/templates/workflow/tag_outputs.mako @@ -0,0 +1,161 @@ +<%inherit file="/base.mako"/> + +<%def name="javascripts()"> + ${parent.javascripts()} + ${h.js( "jquery.autocomplete" )} + + + +<%def name="stylesheets()"> + ${parent.stylesheets()} + ${h.css( "autocomplete_tagging" )} + + + +<% +from galaxy.tools.parameters import DataToolParameter, RuntimeValue +from galaxy.jobs.actions.post import ActionBox +%> + +<%def name="do_inputs( inputs, values, errors, prefix, step, other_values = None )"> + %if other_values is None: + <% other_values = values %> + %endif + %for input_index, input in enumerate( inputs.itervalues() ): + %if input.type == "repeat": +
+
${input.title_plural}
+ <% repeat_values = values[input.name] %> + %for i in range( len( repeat_values ) ): + %if input.name in errors: + <% rep_errors = errors[input.name][i] %> + %else: + <% rep_errors = dict() %> + %endif +
+ <% index = repeat_values[i]['__index__'] %> +
${input.title} ${i + 1}
+ ${do_inputs( input.inputs, repeat_values[ i ], rep_errors, prefix + input.name + "_" + str(index) + "|", step, other_values )} + ##
+
+ %endfor + ##
+
+ %elif input.type == "conditional": + <% group_values = values[input.name] %> + <% current_case = group_values['__current_case__'] %> + <% new_prefix = prefix + input.name + "|" %> + <% group_errors = errors.get( input.name, {} ) %> + ${row_for_param( input.test_param, group_values[ input.test_param.name ], other_values, group_errors, prefix, step )} + ${do_inputs( input.cases[ current_case ].inputs, group_values, group_errors, new_prefix, step, other_values )} + %else: + ${row_for_param( input, values[ input.name ], other_values, errors, prefix, step )} + %endif + %endfor + + +<%def name="row_for_param( param, value, other_values, error_dict, prefix, step )"> + ## -- ${param.name} -- ${step.state.inputs} -- + %if error_dict.has_key( param.name ): + <% cls = "form-row form-row-error" %> + %else: + <% cls = "form-row" %> + %endif +
+ +
+ %if isinstance( param, DataToolParameter ): + %if ( prefix + param.name ) in step.input_connections_by_name: + <% + conn = step.input_connections_by_name[ prefix + param.name ] + %> + Output dataset '${conn.output_name}' from step ${int(conn.output_step.order_index)+1} + %else: + ## FIXME: Initialize in the controller + <% + if value is None: + value = other_values[ param.name ] = param.get_initial_value( t, other_values ) + %> + ${param.get_html_field( t, value, other_values ).get_html( str(step.id) + "|" + prefix )} + + %endif + %elif isinstance( value, RuntimeValue ) or ( str(step.id) + '|__runtime__' + prefix + param.name ) in incoming: + ## On the first load we may see a RuntimeValue, so we write + ## an input field using the initial value for the param. + ## Subsequents posts will no longer have the runtime value + ## (since an actualy value will be posted) so we add a hidden + ## field so we know to continue drawing form for this param. + ## FIXME: This logic shouldn't be in the template. The + ## controller should go through the inputs on the first + ## load, fill in initial values where needed, and mark + ## all that are runtime modifiable in some way. + <% value = other_values[ param.name ] = param.get_initial_value( t, other_values ) %> + ${param.get_html_field( t, value, other_values ).get_html( str(step.id) + "|" + prefix )} + + %else: + ${param.value_to_display_text( value, app )} + %endif +
+ %if step.upgrade_messages and param.name in step.upgrade_messages: +
${step.upgrade_messages[param.name]}
+ %endif + %if error_dict.has_key( param.name ): +
+
 ${error_dict[param.name]}
+
+ %endif +
+
+ + +

Set outputs for workflow "${workflow.name}"

+

Select the checkboxes for step outputs you would like to mark as whole-workflow outputs.

+%if has_upgrade_messages: +
+ Problems were encourered when loading this workflow, likely due to tool + version changes. Missing parameter values have been replaced with default. + Please review the parameter values below. +
+%endif + +
+## +%for i, step in enumerate( steps ): + %if step.type == 'tool': + <% tool = app.toolbox.tools_by_id[step.tool_id] %> + +
+
Step ${int(step.order_index)+1}: ${tool.name}
+
+ % for output in tool.outputs: +
+ %if step.workflow_outputs and output in [wf_output.output_name for wf_output in step.workflow_outputs]: +

${output}

+ %else: +

${output}

+ %endif +
+ % endfor + % if step.annotations: +
+
+ ${step.annotations[0].annotation} +
+ % endif +
+
+ %endif +%endfor + +
\ No newline at end of file