From 24a3ce01c66ac75b378fedeb8099afc780bc741b Mon Sep 17 00:00:00 2001 From: James Taylor Date: Mon, 5 May 2008 02:08:32 +0000 Subject: [PATCH] Workflows can now have steps of other kinds (not just tools). The first one is "InputDataset" which allows a workflow to have an input dataset which many different tools can use (without needing to specify it repeatedly. Still need to extract out a common interface (tools will become another type of "module") which will clean this up a lot. --- lib/galaxy/model/__init__.py | 2 + lib/galaxy/model/mapping.py | 2 + lib/galaxy/web/controllers/workflow.py | 240 +++++++++++------- lib/galaxy/web/framework/__init__.py | 2 +- .../scripts/galaxy.workflow_editor.canvas.js | 16 +- .../workflow/build_from_current_history.mako | 8 +- templates/workflow/editor.mako | 45 +++- templates/workflow/editor_generic_form.mako | 43 ++++ 8 files changed, 261 insertions(+), 97 deletions(-) create mode 100644 templates/workflow/editor_generic_form.mako diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index d9d90316018..b45643e0b38 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -463,11 +463,13 @@ class Workflow( object ): class WorkflowStep( object ): def __init__( self ): self.id = None + self.type = None self.tool_id = None self.tool_inputs = None self.tool_errors = None self.position = None self.input_connections = None + self.config = None class WorkflowStepConnection( object ): def __init__( self ): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 4c42b8e6373..0e21c2b1c50 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -195,11 +195,13 @@ WorkflowStep.table = Table( "workflow_step", metadata, Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), Column( "workflow_id", Integer, ForeignKey( "workflow.id" ), index=True, nullable=False ), + Column( "type", String(64) ), Column( "tool_id", String ), Column( "tool_version", String ), # Reserved for future Column( "tool_inputs", JSONType ), Column( "tool_errors", JSONType ), Column( "position", JSONType ), + Column( "config", JSONType ), Column( "order_index", Integer ), ## Column( "input_connections", JSONType ) ) diff --git a/lib/galaxy/web/controllers/workflow.py b/lib/galaxy/web/controllers/workflow.py index 775935a5def..06567148db0 100644 --- a/lib/galaxy/web/controllers/workflow.py +++ b/lib/galaxy/web/controllers/workflow.py @@ -13,25 +13,22 @@ class WorkflowController( BaseController ): beta = True @web.expose + @web.require_login( "use Galaxy workflows" ) def index( self, trans ): """ Render workflow main page (management of existing workflows) """ user = trans.get_user() - if not user: - return error( "You must be logged in to use Galaxy workflows." ) workflows = trans.sa_session.query( model.StoredWorkflow ).filter_by( user=user, deleted=False ).all() - return trans.fill_template( "workflow/index.mako", - workflows = workflows ) + return trans.fill_template( "workflow/index.mako", workflows = workflows ) @web.expose + @web.require_login( "create workflows" ) def create( self, trans, workflow_name=None ): """ Create a new stored workflow with name `workflow_name`. """ user = trans.get_user() - if not user: - return error( "Must be logged in to create or modify workflows" ) if not workflow_name: return error( "Must provide a name for the new workflow" ) # Create the new stored workflow @@ -65,6 +62,7 @@ class WorkflowController( BaseController ): @web.expose + @web.require_login( "edit workflows" ) def editor( self, trans, id=None ): """ Render the main workflow editor interface. The canvas is embedded as @@ -74,9 +72,6 @@ class WorkflowController( BaseController ): if not id: return trans.show_error_message( "Invalid workflow id" ) id = trans.security.decode_id( id ) - user = trans.get_user() - if not user: - return trans.show_error_message( "Must be logged in to create or modify workflows" ) return trans.fill_template( "workflow/editor.mako", workflow_id=id ) @@ -128,6 +123,18 @@ class WorkflowController( BaseController ): rval['tool_state'] = state.encode( tool, trans.app ) rval['data_inputs'] = get_data_inputs( tool.inputs, state.inputs ) return rval + + @web.json + def get_module_info( self, trans, type ): + module = module_types[type]() + rval = {} + rval['name'] = module.name + rval['type'] = module.type + rval['tool_state'] = state = module.get_state() + rval['data_inputs'] = module.get_data_inputs( state ) + rval['data_outputs'] = module.get_data_outputs( state ) + rval['form_html'] = module.get_config_form_html( trans, state ) + return rval @web.json def load_workflow( self, trans, id ): @@ -151,34 +158,44 @@ class WorkflowController( BaseController ): for step in workflow.steps: step_dict = {} step_dict['id'] = step.order_index - step_dict['tool_id'] = tool_id = step.tool_id - # Load tool - tool = trans.app.toolbox.tools_by_id[tool_id] - # Build a state from the tool_inputs dict - state = DefaultToolState() - state.inputs = tool.params_from_strings( step.tool_inputs, trans.app, ignore_errors=True ) - step_dict['tool_state'] = state.encode( tool, trans.app ) - # Error messages for the tool - step_dict['tool_errors'] = ( step.tool_errors or None ) + step_dict['type'] = step_type = ( step.type or "tool" ) + if step_type == 'tool': + step_dict['tool_id'] = tool_id = step.tool_id + # Load tool + tool = trans.app.toolbox.tools_by_id[tool_id] + # Build a state from the tool_inputs dict + state = DefaultToolState() + state.inputs = tool.params_from_strings( step.tool_inputs, trans.app, ignore_errors=True ) + step_dict['tool_state'] = state.encode( tool, trans.app ) + # Error messages for the tool + step_dict['tool_errors'] = ( step.tool_errors or None ) + # Input and output specs + step_dict['data_inputs'] = get_data_inputs( tool.inputs, state.inputs ) + data_outputs = [] + for name, ( format, metadata_source, parent ) in tool.outputs.iteritems(): + data_outputs.append( dict( name=name, extension=format ) ) + step_dict['data_outputs'] = data_outputs + # Build the tool form html + errors = step.tool_errors + step_dict['form_html'] = trans.fill_template( "workflow/editor_tool_form.mako", + tool=tool, as_html=as_html, values=state.inputs, errors=( step.tool_errors or {} ) ) + step_dict['name'] = tool.name + else: + module = module_types[step.type].from_workflow_step( step ) + step_dict['name'] = module.name + step_dict['tool_state'] = state = module.get_state() + step_dict['data_inputs'] = module.get_data_inputs( state ) + step_dict['data_outputs'] = module.get_data_outputs( state ) + step_dict['form_html'] = module.get_config_form_html( trans, state ) # Connections input_conn_dict = {} for conn in step.input_connections: - input_conn_dict[ conn.input_name ] = dict( id=conn.output_step.order_index, - output_name=conn.output_name ) + input_conn_dict[ conn.input_name ] = \ + dict( id=conn.output_step.order_index, output_name=conn.output_name ) step_dict['input_connections'] = input_conn_dict # Position step_dict['position'] = step.position - # Input and output specs - step_dict['data_inputs'] = get_data_inputs( tool.inputs, state.inputs ) - data_outputs = [] - for name, ( format, metadata_source, parent ) in tool.outputs.iteritems(): - data_outputs.append( dict( name=name, extension=format ) ) - step_dict['data_outputs'] = data_outputs - # Build the tool form html - errors = step.tool_errors - step_dict['form_html'] = trans.fill_template( "workflow/editor_tool_form.mako", - tool=tool, as_html=as_html, values=state.inputs, errors=( step.tool_errors or {} ) ) - step_dict['name'] = tool.name + # Add to return value data['steps'][step.order_index] = step_dict return data @@ -206,22 +223,27 @@ class WorkflowController( BaseController ): steps_by_external_id = {} # First pass to build step objects and populate basic values for key, step_dict in data['steps'].iteritems(): - # Decode the tool state from the step dict - tool = trans.app.toolbox.tools_by_id[ step_dict['tool_id'] ] - state = DefaultToolState() - state.decode( step_dict['tool_state'], tool, trans.app ) - # Convert back to strings for database - tool_inputs = tool.params_to_strings( state.inputs, trans.app ) # Create the model class for the step step = model.WorkflowStep() + step.type = step_type = step_dict['type'] steps.append( step ) steps_by_external_id[ step_dict['id' ] ] = step - step.tool_id = step_dict['tool_id'] - step.tool_inputs = tool_inputs - step.tool_errors = step_dict['tool_errors'] - if step.tool_errors: - workflow.has_errors = True step.position = step_dict['position'] + if step_type == 'tool': + step.tool_id = step_dict['tool_id'] + # Decode the tool state from the step dict + tool = trans.app.toolbox.tools_by_id[ step_dict['tool_id'] ] + state = DefaultToolState() + state.decode( step_dict['tool_state'], tool, trans.app ) + # Convert back to strings for database + tool_inputs = tool.params_to_strings( state.inputs, trans.app ) + step.tool_inputs = tool_inputs + step.tool_errors = step_dict['tool_errors'] + if step.tool_errors: + workflow.has_errors = True + else: + module = module_types[step_type].from_state( step_dict['tool_state'] ) + module.save_to_step( step ) # Stick this in the step temporarily step.temp_input_connections = step_dict['input_connections'] # Second pass to deal with connections between steps @@ -279,7 +301,7 @@ class WorkflowController( BaseController ): return dict( ext_to_class_name=ext_to_class_name, class_to_classes=class_to_classes ) @web.expose - def build_from_current_history( self, trans, job_ids=None, workflow_name=None ): + def build_from_current_history( self, trans, job_ids=None, dataset_ids=None, workflow_name=None ): user = trans.get_user() history = trans.get_history() if not user: @@ -297,19 +319,24 @@ class WorkflowController( BaseController ): if type( job_ids ) == str: job_ids = [ job_ids ] job_ids = [ int( id ) for id in job_ids ] + if type( dataset_ids ) == str: + dataset_ids = [ job_ids ] + dataset_ids = [ int( id ) for id in dataset_ids ] # Find each job, for security we (implicately) check that they are # associated witha job in the current history. jobs, warnings = get_job_dict( trans ) - # Create a mapping from hid to ( job_id, output_name ) - hid_to_output_pair = {} - for job, datasets in jobs.iteritems(): - for assoc_name, data in datasets: - hid_to_output_pair[ data.hid ] = ( job.id, assoc_name ) - # Mapping from job ids to workflow step ids (0, 1, 2, ...) - job_id_to_step_index = dict( ( job_id, i ) for ( i, job_id ) in enumerate( job_ids ) ) - # Back-translate each job jobs_by_id = dict( ( job.id, job ) for job in jobs.keys() ) steps = [] + steps_by_job_id= {} + hid_to_output_pair = {} + # Input dataset steps + for hid in dataset_ids: + step = model.WorkflowStep() + step.type = 'data_input' + hid_to_output_pair[ hid ] = ( step, 'output' ) + steps.append( step ) + print hid_to_output_pair + # Tool steps for job_id in job_ids: assert job_id in jobs_by_id, "Attempt to create workflow with job not connected to current history" job = jobs_by_id[ job_id ] @@ -324,17 +351,19 @@ class WorkflowController( BaseController ): # job. for other_hid, input_name in associations: if other_hid in hid_to_output_pair: - other_job_id, other_name = hid_to_output_pair[ other_hid ] - # Only create association if the associated output dataset - # is being included in this workflow - if other_job_id in job_id_to_step_index: - conn = model.WorkflowStepConnection() - conn.input_step = step - conn.input_name = input_name - # Should always be connected to an earlier step - conn.output_step = steps[ job_id_to_step_index[ other_job_id ] ] - conn.output_name = other_name + other_step, other_name = hid_to_output_pair[ other_hid ] + conn = model.WorkflowStepConnection() + conn.input_step = step + conn.input_name = input_name + # Should always be connected to an earlier step + conn.output_step = other_step + conn.output_name = other_name steps.append( step ) + steps_by_job_id[ job_id ] = step + # Store created dataset hids + for assoc in job.output_datasets: + hid_to_output_pair[ assoc.dataset.hid ] = ( step, assoc.name ) + print hid_to_output_pair # Workflow to populate workflow = model.Workflow() workflow.name = workflow_name @@ -343,7 +372,7 @@ class WorkflowController( BaseController ): # And let's try to set up some reasonable locations on the canvas # (these are pretty arbitrary values) levorder = order_workflow_steps_with_levels( steps ) - base_pos = 2510 + base_pos = 10 for i, steps_at_level in enumerate( levorder ): for j, index in enumerate( steps_at_level ): step = steps[ index ] @@ -389,21 +418,24 @@ class WorkflowController( BaseController ): p = "%s|" % step.id l = len(p) step_args = dict( ( k[l:], v ) for ( k, v ) in kwargs.iteritems() if k.startswith( p ) ) - # Get the tool - tool = trans.app.toolbox.tools_by_id[ step.tool_id ] - # Get the state - state = DefaultToolState() - state.decode( step_args.pop("tool_state"), tool, trans.app ) - step.state = state - # Connections by input name - step.input_connections_by_name = dict( ( conn.input_name, conn ) for conn in step.input_connections ) - # Get old errors - old_errors = state.inputs.pop( "__errors__", {} ) - # Update the state - step_errors = tool.update_state( trans, tool.inputs, step.state.inputs, step_args, - update_only=True, old_errors=old_errors ) - if step_errors: - errors[step.id] = state.inputs["__errors__"] = step_errors + if step.type == 'tool': + # Get the tool + tool = trans.app.toolbox.tools_by_id[ step.tool_id ] + # Get the state + state = DefaultToolState() + state.decode( step_args.pop("tool_state"), tool, trans.app ) + step.state = state + # Connections by input name + step.input_connections_by_name = dict( ( conn.input_name, conn ) for conn in step.input_connections ) + # Get old errors + old_errors = state.inputs.pop( "__errors__", {} ) + # Update the state + step_errors = tool.update_state( trans, tool.inputs, step.state.inputs, step_args, + update_only=True, old_errors=old_errors ) + if step_errors: + errors[step.id] = state.inputs["__errors__"] = step_errors + else: + return error( "Modules not yet supported for running" ) if not errors: # Run each step, connecting outputs to inputs outputs = {} @@ -419,18 +451,21 @@ class WorkflowController( BaseController ): outputs=outputs ) else: for step in workflow.steps: - # Build a new tool state for the step - tool = trans.app.toolbox.tools_by_id[ step.tool_id ] - state = DefaultToolState() - state.inputs = tool.params_from_strings( step.tool_inputs, trans.app ) - # Store state with the step - step.state = state - # Connections by input name - step.input_connections_by_name = dict( ( conn.input_name, conn ) for conn in step.input_connections ) - # This should never actually happen since we don't allow - # running workflows with errors (yet?) - if step.tool_errors: - errors[step.id] = step.tool_errors + if step.type == 'tool': + # Build a new tool state for the step + tool = trans.app.toolbox.tools_by_id[ step.tool_id ] + state = DefaultToolState() + state.inputs = tool.params_from_strings( step.tool_inputs, trans.app ) + # Store state with the step + step.state = state + # Connections by input name + step.input_connections_by_name = dict( ( conn.input_name, conn ) for conn in step.input_connections ) + # This should never actually happen since we don't allow + # running workflows with errors (yet?) + if step.tool_errors: + errors[step.id] = step.tool_errors + else: + return error( "Modules not yet supported for running" ) # Render the form return trans.fill_template( "workflow/run.mako", @@ -438,6 +473,33 @@ class WorkflowController( BaseController ): workflow=stored, errors=errors ) +## ---- Workflow modules (to be factored out) --------------------------------- + +## TODO: 'Tool' should be a module rather than a special case + +class InputDataModule( object ): + type = "data_input" + name = "Input dataset" + @classmethod + def from_state( cls, state ): + return cls() + @classmethod + def from_workflow_step( cls, state ): + return cls() + def get_state( self ): + return None + def get_data_inputs( self, state ): + return [] + def get_data_outputs( self, state ): + return [ dict( name='output', extension='input' ) ] + def get_config_form_html( self, trans, state ): + form = web.FormBuilder( title=self.name ) + return trans.fill_template( "workflow/editor_generic_form.mako", form=form ) + def save_to_step( self, step ): + pass + +module_types = dict( data_input=InputDataModule ) + ## ---- Utility methods ------------------------------------------------------- def get_stored_workflow( trans, id ): diff --git a/lib/galaxy/web/framework/__init__.py b/lib/galaxy/web/framework/__init__.py index f6e56265189..6e27916293e 100644 --- a/lib/galaxy/web/framework/__init__.py +++ b/lib/galaxy/web/framework/__init__.py @@ -384,7 +384,7 @@ class FormBuilder( object ): """ Simple class describing an HTML form """ - def __init__( self, action, title, name="form", submit_text="submit" ): + def __init__( self, action="", title="", name="form", submit_text="submit" ): self.title = title self.name = name self.action = action diff --git a/static/scripts/galaxy.workflow_editor.canvas.js b/static/scripts/galaxy.workflow_editor.canvas.js index b1d13ff76e7..02f454ba421 100644 --- a/static/scripts/galaxy.workflow_editor.canvas.js +++ b/static/scripts/galaxy.workflow_editor.canvas.js @@ -257,6 +257,7 @@ $.extend( Node.prototype, { }, init_field_data : function ( data ) { var f = this.element; + this.type = data.type this.form_html = data.form_html; this.tool_state = data.tool_state; this.tool_errors = data.tool_errors; @@ -273,7 +274,7 @@ $.extend( Node.prototype, { t = $("
") node.enable_input_terminal( t, input.name, input.extensions ); ibox.append( $("
" + input.label + "
" ).prepend( t ) ); - }); + }); if ( ( data.data_inputs.length > 0 ) && ( data.data_outputs.length > 0 ) ) { b.append( $( "
" ) ); } @@ -380,6 +381,7 @@ $.extend( Workflow.prototype, { }); var node_data = { id : node.id, + type : node.type, tool_id : node.tool_id, tool_state : node.tool_state, tool_errors : node.tool_errors, @@ -396,7 +398,7 @@ $.extend( Workflow.prototype, { wf.name = data.name; // First pass, nodes $.each( data.steps, function( id, step ) { - var node = prebuild_node_for_tool( step.tool_id, step.name ); + var node = prebuild_node( "tool", step.name, step.tool_id ); node.init_field_data( step ); if ( step.position ) { node.element.css( { top: step.position.top, left: step.position.left } ); @@ -441,10 +443,13 @@ $.extend( Workflow.prototype, { } }); -function prebuild_node_for_tool( id, title_text ) { +function prebuild_node( type, title_text, tool_id ) { var f = $("
"); var node = new Node( f ); - node.tool_id = id; + node.type = type + if ( type == 'tool' ) { + node.tool_id = tool_id; + } var title = $("
" + title_text + "
" ) f.append( title ); f.css( "left", $(window).scrollLeft() + 20 ); f.css( "top", $(window).scrollTop() + 20 ); @@ -464,6 +469,9 @@ function prebuild_node_for_tool( id, title_text ) { function() { $(this).attr( 'src', "../images/delete_icon.png" ) } ) ); f.appendTo( "#canvas-container" ); + // Position in container + var o = $("#canvas-container").position(); + f.css( { left: ( - o.left ) + 10, top: ( - o.top ) + 10 } ); var width = f.width(); buttons.prependTo( title ); width += ( buttons.width() + 10 ); diff --git a/templates/workflow/build_from_current_history.mako b/templates/workflow/build_from_current_history.mako index eea2037f8a2..fa89f00d8ee 100644 --- a/templates/workflow/build_from_current_history.mako +++ b/templates/workflow/build_from_current_history.mako @@ -33,7 +33,7 @@ if ( window.parent && window.parent.handle_minwidth_hint ) { -<%def name="history_item( data )"> +<%def name="history_item( data, creator_disabled=False )"> %if data.state in [ "no state", "", None ]: <% data_state = "queued" %> %else: @@ -54,6 +54,10 @@ if ( window.parent && window.parent.handle_minwidth_hint ) { + %if disabled: +
+
Treat as input dataset
+ %endif @@ -114,7 +118,7 @@ into a workflow will be shown in gray.

%for _, data in datasets: -
${history_item( data )}
+
${history_item( data, disabled )}
%endfor diff --git a/templates/workflow/editor.mako b/templates/workflow/editor.mako index 579dfd78aaf..12bb6391028 100644 --- a/templates/workflow/editor.mako +++ b/templates/workflow/editor.mako @@ -233,7 +233,7 @@ // Add a new step to the workflow by tool id function add_node_for_tool( id, title ) { - node = prebuild_node_for_tool( id, title ); + node = prebuild_node( 'tool', title, id ); workflow.add_node( node ); workflow.activate_node( node ); $.ajax( { @@ -253,6 +253,27 @@ }); }; + function add_node_for_module( type, title ) { + node = prebuild_node( type, title ); + workflow.add_node( node ); + workflow.activate_node( node ); + $.ajax( { + url: "${h.url_for( action='get_module_info' )}", + data: { type: type, "_": "true" }, + dataType: "json", + success: function( data ) { + node.init_field_data( data ); + }, + error: function( x, e ) { + var m = "error loading field data" + if ( x.status == 0 ) { + m += ", server unavailable" + } + node.error( m ); + } + }); + }; + function show_form_for_tool( text, node ) { // $("#overlay, #modalwrapper" ).show(); //$("#modal iframe").attr( 'src', "${h.url_for( action='tool_form' )}?tool_id=" + tool_id ).load( function () { @@ -418,6 +439,16 @@ padding-bottom: 0px; font-weight: bold; } + div.toolMenuGroupHeader { + font-weight: bold; + padding-top: 0.5em; + padding-bottom: 0.5em; + color: #333; + font-style: italic; + border-bottom: dotted #333 1px; + margin-bottom: 0.5em; + } + div.toolTitle { padding-top: 5px; padding-bottom: 5px; @@ -611,6 +642,18 @@ %endfor +
 
+
Workflow control
+
+ Inputs +
+
+
+
+ Input dataset +
+
+
diff --git a/templates/workflow/editor_generic_form.mako b/templates/workflow/editor_generic_form.mako new file mode 100644 index 00000000000..71b917b85c6 --- /dev/null +++ b/templates/workflow/editor_generic_form.mako @@ -0,0 +1,43 @@ +
+
${form.title}
+
+
+ + %if form.inputs: + %for input in form.inputs: + <% + cls = "form-row" + if input.error: + cls += " form-row-error" + %> +
+ +
+ +
+ %if input.error: +
+
 ${input.error}
+
+ %endif + + %if input.help: +
+ ${input.help} +
+ %endif + +
+ +
+ %endfor + + %endif +
+ %else: +
No options
+
+
+
\ No newline at end of file