From 5e9fe3e6b25fe6bd08e89fa16e904bd231703a95 Mon Sep 17 00:00:00 2001
From: James Taylor
Date: Wed, 31 Oct 2007 03:57:06 +0000
Subject: [PATCH] Workflow round five, workflows can now be constructed from
histories. See the history options page for "Construct workflow" link. It
allows you to select which steps from the current history you want to
convert.
This is currently not very robust to changes in tool configuration, so old
histories may cause errors. This will be fixable to some extent, but we'll
probably need to go back and look at the past changes to tool configs and
define some migration rules.
---
lib/galaxy/model/__init__.py | 9 +
lib/galaxy/tools/__init__.py | 16 ++
lib/galaxy/tools/actions/__init__.py | 3 +-
lib/galaxy/util/topsort.py | 47 +++++-
lib/galaxy/web/controllers/history.py | 158 ++++++++++++++++++
lib/galaxy/web/controllers/workflow_editor.py | 6 +-
lib/galaxy/workflow/__init__.py | 36 ++--
static/june_2007_style/base.css.tmpl | 16 ++
static/june_2007_style/blue/base.css | 27 ++-
static/june_2007_style/blue/base_bg.png | Bin 232 -> 232 bytes
static/june_2007_style/blue/error_bg.png | Bin 168 -> 168 bytes
.../june_2007_style/blue/footer_title_bg.png | Bin 334 -> 334 bytes
static/june_2007_style/blue/form_body_bg.png | Bin 178 -> 178 bytes
static/june_2007_style/blue/form_title_bg.png | Bin 207 -> 207 bytes
static/june_2007_style/blue/gray_bg.png | Bin 169 -> 169 bytes
static/june_2007_style/blue/masthead_bg.png | Bin 128 -> 128 bytes
static/june_2007_style/blue/menu_bg.png | Bin 624 -> 624 bytes
static/june_2007_style/blue/ok_bg.png | Bin 168 -> 168 bytes
.../june_2007_style/blue/panel_header_bg.png | Bin 144 -> 144 bytes
.../blue/popupmenu_callout_top.png | Bin 412 -> 412 bytes
static/june_2007_style/blue/warn_bg.png | Bin 169 -> 169 bytes
.../workflow_from_current_history.mako | 130 ++++++++++++++
templates/history_options.tmpl | 7 +-
templates/message.tmpl | 5 +
templates/workflow_editor/index.mako | 85 +++++++---
25 files changed, 495 insertions(+), 50 deletions(-)
create mode 100644 lib/galaxy/web/controllers/history.py
create mode 100644 templates/history/workflow_from_current_history.mako
diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index 4a9ee3e490b..cdea79f4323 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -67,6 +67,15 @@ class Job( object ):
# For historical reasons state propogates down to datasets
for da in self.output_datasets:
da.dataset.state = state
+ def get_param_values( self, app ):
+ """
+ Read encoded parameter values from the database and turn back into a
+ dict of tool parameter values.
+ """
+ param_dict = dict( [ ( p.name, p.value ) for p in self.parameters ] )
+ tool = app.toolbox.tools_by_id[self.tool_id]
+ param_dict = tool.params_from_strings( param_dict, app )
+ return param_dict
class JobParameter( object ):
def __init__( self, name, value ):
diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index 1cb3c5cd174..68e674d66bf 100644
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -280,6 +280,9 @@ class Tool:
log.exception( "Failed to parse tool tests" )
else:
self.tests = None
+ # Determine if this tool can be used in workflows
+ self.is_workflow_compatible = self.check_workflow_compatible()
+
def parse_inputs( self, root ):
"""
@@ -459,6 +462,19 @@ class Tool:
for name in param.get_dependencies():
context[ name ].refresh_on_change = True
return param
+
+ def check_workflow_compatible( self ):
+ # This is probably the best bet for detecting external web tools
+ # right now
+ if self.action != "/tool_runner/index":
+ return False
+ # HACK: upload is (as always) a special case becuase file parameters
+ # can't be persisted.
+ if self.id == "upload1":
+ return False
+ # TODO: Anyway to capture tools that dynamically change their own
+ # outputs?
+ return True
def new_state( self, trans, all_pages=False ):
"""
diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py
index 3169afe09a4..f77a5379d92 100644
--- a/lib/galaxy/tools/actions/__init__.py
+++ b/lib/galaxy/tools/actions/__init__.py
@@ -44,7 +44,8 @@ class DefaultToolAction( object ):
# FIXME: does this need to modify 'incoming' or should this be
# moved into 'build_param_dict'? Is this just about getting the
- # metadata into the command line?
+ # metadata into the command line?
+ # NEED TO FIX THIS SOON.
input_names = []
input_ext = 'data'
input_dbkey = incoming.get( "dbkey", "?" )
diff --git a/lib/galaxy/util/topsort.py b/lib/galaxy/util/topsort.py
index f187155d6b9..15fc988727e 100644
--- a/lib/galaxy/util/topsort.py
+++ b/lib/galaxy/util/topsort.py
@@ -161,4 +161,49 @@ def topsort(pairlist):
for x in numpreds.keys():
assert numpreds[x] > 0
raise CycleError(answer, numpreds, successors)
- return answer
\ No newline at end of file
+ return answer
+
+def topsort_levels(pairlist):
+ numpreds = {} # elt -> # of predecessors
+ successors = {} # elt -> list of successors
+ for first, second in pairlist:
+ # make sure every elt is a key in numpreds
+ if not numpreds.has_key(first):
+ numpreds[first] = 0
+ if not numpreds.has_key(second):
+ numpreds[second] = 0
+
+ # if they're the same, there's no real dependence
+ if first == second:
+ continue
+
+ # since first < second, second gains a pred ...
+ numpreds[second] = numpreds[second] + 1
+
+ # ... and first gains a succ
+ if successors.has_key(first):
+ successors[first].append(second)
+ else:
+ successors[first] = [second]
+
+ answer = []
+
+ while 1:
+ # Suck up everything without a predecessor.
+ levparents = [x for x in numpreds.keys() if numpreds[x] == 0]
+ if not levparents:
+ break
+ answer.append( levparents )
+ for levparent in levparents:
+ del numpreds[levparent]
+ if successors.has_key(levparent):
+ for levparentsucc in successors[levparent]:
+ numpreds[levparentsucc] -= 1
+ del successors[levparent]
+
+ if numpreds:
+ # Everything in num_parents has at least one child ->
+ # there's a cycle.
+ raise CycleError( answer, numpreds, successors )
+
+ return answer
diff --git a/lib/galaxy/web/controllers/history.py b/lib/galaxy/web/controllers/history.py
new file mode 100644
index 00000000000..7c3f03a530b
--- /dev/null
+++ b/lib/galaxy/web/controllers/history.py
@@ -0,0 +1,158 @@
+from galaxy.web.base.controller import *
+
+from galaxy.tools.parameters import DataToolParameter
+from galaxy.tools import DefaultToolState
+from galaxy.tools.grouping import Repeat, Conditional
+from galaxy.datatypes.data import Data
+from galaxy.workflow import Workflow, WorkflowStep
+
+from galaxy.util.odict import odict
+
+import simplejson
+
+class HistoryController( BaseController ):
+
+ def __get_job_dict( self, trans ):
+ """
+ Return a dictionary of Job -> [ Dataset ] mappings, for all finished
+ active Datasets in the current history and the jobs that created them.
+ """
+ history = trans.get_history()
+ # Get the jobs that created the datasets
+ warnings = set()
+ jobs = odict()
+ for dataset in history.active_datasets:
+ # FIXME: Create "Dataset.is_finished"
+ if dataset.state in ( 'new', 'running', 'queued' ):
+ warnings.add( "Some datasets still queued or running were ignored" )
+ continue
+ for assoc in dataset.creating_job_associations:
+ job = assoc.job
+ if job in jobs:
+ jobs[ job ].append( ( assoc.name, dataset ) )
+ else:
+ jobs[ job ] = [ ( assoc.name, dataset ) ]
+ return jobs, warnings
+
+ def __cleanup_param_values( self, inputs, values ):
+ """
+ Remove 'Data' values from `param_values` but track the associations
+ """
+ associations = []
+ names_to_clean = []
+ # dbkey is pushed in by the framework
+ if 'dbkey' in values:
+ del values['dbkey']
+ root_values = values
+ # Cleanup all data inputs
+ def cleanup( prefix, inputs, values ):
+ for key, input in inputs.items():
+ if isinstance( input, DataToolParameter ):
+ tmp = values[key]
+ values[key] = None
+ # HACK: Nested associations are not yet working, but we
+ # still need to clean them up so we can serialize
+ if not( prefix ):
+ associations.append( ( tmp.hid, prefix + key ) )
+ # Cleanup the other deprecated crap associated with datasets
+ # as well. Worse, for nested datasets all the metadata is
+ # being pushed into the root. FIXME: MUST REMOVE SOON
+ key = prefix + key + "_"
+ for k in root_values.keys():
+ if k.startswith( key ):
+ del root_values[k]
+ elif isinstance( input, Repeat ):
+ group_values = values[key]
+ for i in range( len( group_values ) ):
+ prefix = "%s_%d|" % ( key, i )
+ cleanup( prefix, input.inputs, group_values[i] )
+ elif isinstance( input, Conditional ):
+ group_values = values[input.name]
+ current_case = group_values['__current_case__']
+ prefix = "%s|" % ( key )
+ cleanup( prefix, input.cases[current_case].inputs, group_values )
+ cleanup( "", inputs, values )
+ return associations
+
+ @web.expose
+ def workflow_from_current_history( self, trans, job_ids=None, workflow_name=None ):
+ if trans.request.method == 'POST':
+ return self.workflow_from_current_history_post( trans, job_ids, workflow_name )
+ user = trans.get_user()
+ if not user:
+ return trans.show_error_message( "Must be logged in to create workflows" )
+ jobs, warnings = self.__get_job_dict( trans )
+ # Render
+ return trans.fill_template(
+ "history/workflow_from_current_history.mako",
+ jobs=jobs,
+ warnings=warnings )
+
+ def workflow_from_current_history_post( self, trans, job_ids, workflow_name ):
+ user = trans.get_user()
+ if not user:
+ return trans.show_error_message( "Must be logged in to create workflows" )
+ # Ensure a list
+ if type( job_ids ) == str:
+ job_ids = [ job_ids ]
+ job_ids = [ int( id ) for id in job_ids ]
+ # Find each job, for security we (implicately) check that they are
+ # associated witha job in the current history.
+ jobs, warnings = self.__get_job_dict( trans )
+ # Create a mapping from hid to ( job_id, output_name )
+ hid_to_output_pair = {}
+ for job, datasets in jobs.iteritems():
+ for assoc_name, data in datasets:
+ hid_to_output_pair[ data.hid ] = ( job.id, assoc_name )
+ # Mapping from job ids to workflow step ids (0, 1, 2, ...)
+ job_id_to_step_id = dict( ( job_id, i ) for ( i, job_id ) in enumerate( job_ids ) )
+ # Workflow to populate
+ workflow = Workflow()
+ # Back-translate each job
+ jobs_by_id = dict( ( job.id, job ) for job in jobs.keys() )
+ for step_id, job_id in enumerate( job_ids ):
+ assert job_id in jobs_by_id, "Attempt to create workflow with job not connected to current history"
+ job = jobs_by_id[ job_id ]
+ tool = trans.app.toolbox.tools_by_id[ job.tool_id ]
+ param_values = job.get_param_values( trans.app )
+ associations = self.__cleanup_param_values( tool.inputs, param_values )
+ step = WorkflowStep()
+ step.id = step_id
+ step.tool_id = job.tool_id
+ step.tool_inputs = param_values
+ for other_hid, input_name in associations:
+ other_job_id, other_name = hid_to_output_pair[ other_hid ]
+ # Only create association if the associated output dataset
+ # is being included in this workflow
+ if other_job_id in job_id_to_step_id:
+ step.input_connections[input_name] = ( job_id_to_step_id[ other_job_id ], other_name )
+ else:
+ step.input_connections[input_name] = None
+ workflow.steps[ step_id ] = step
+ # Try to order the nodes
+ workflow.order_nodes()
+ # And let's try to set up some reasonable locations
+ levorder = workflow.order_nodes_levels()
+ base_pos = 2510
+ for i, steps_at_level in enumerate( levorder ):
+ for j, step_id in enumerate( steps_at_level ):
+ step = workflow.steps[step_id]
+ step.position = dict( top = ( base_pos + 120 * j ),
+ left = ( base_pos + 220 * i ) )
+ # Store it
+ stored = model.StoredWorkflow.get_by( user = user, name = workflow_name )
+ if stored is None:
+ stored = model.StoredWorkflow()
+ stored.user = user
+ stored.name = workflow_name
+ stored.encoded_value = simplejson.dumps( workflow.to_simple() )
+ stored.flush()
+ #
+ return trans.show_ok_message( "Workflow '%s' created. Click to load in workflow editor"
+ % ( workflow_name, web.url_for( controller='workflow_editor', action=None, workflow_name=workflow_name ) ) )
+
+
+
+
+
+
\ No newline at end of file
diff --git a/lib/galaxy/web/controllers/workflow_editor.py b/lib/galaxy/web/controllers/workflow_editor.py
index 5b0f7dc2d3d..2f33730cbcb 100644
--- a/lib/galaxy/web/controllers/workflow_editor.py
+++ b/lib/galaxy/web/controllers/workflow_editor.py
@@ -11,11 +11,12 @@ class WorkflowEditor( BaseController ):
beta = True
@web.expose
- def index( self, trans ):
+ def index( self, trans, workflow_name=None ):
user = trans.get_user()
if not user:
return trans.show_error_message( "Must be logged in to create or modify workflows" )
- return trans.fill_template( "workflow_editor/index.mako" )
+ return trans.fill_template( "workflow_editor/index.mako",
+ workflow_name=workflow_name )
@web.expose
def canvas( self, trans ):
@@ -67,6 +68,7 @@ class WorkflowEditor( BaseController ):
decode_state( node, trans.app )
# Create workflow from json data
workflow = Workflow.from_simple( data )
+ workflow.order_nodes()
# Store it
stored = model.StoredWorkflow.get_by( user = user, name = workflow_name )
if stored is None:
diff --git a/lib/galaxy/workflow/__init__.py b/lib/galaxy/workflow/__init__.py
index 11f9236d0c6..a42f3e11d7d 100644
--- a/lib/galaxy/workflow/__init__.py
+++ b/lib/galaxy/workflow/__init__.py
@@ -1,5 +1,5 @@
from galaxy.tools import DefaultToolState
-from galaxy.util.topsort import topsort, CycleError
+from galaxy.util.topsort import topsort, topsort_levels, CycleError
class Workflow( object ):
@@ -21,7 +21,6 @@ class Workflow( object ):
if step.has_errors:
workflow.has_errors = True
workflow.steps[ id ] = step
- workflow.node_order = workflow.order_nodes()
return workflow
def to_simple( self ):
@@ -35,27 +34,40 @@ class Workflow( object ):
has_cycles=self.has_cycles,
has_errors=self.has_errors )
+ def edge_list( self ):
+ edges = []
+ all_ids = set( self.steps.keys() )
+ for step in self.steps.values():
+ edges.append( ( step.id, step.id ) )
+ for name, conn in step.input_connections.iteritems():
+ if conn is not None:
+ other_node_id, _ = conn
+ edges.append( ( other_node_id, step.id ) )
+ return edges
+
def order_nodes( self ):
"""
Perform topological sort of the steps, return an ordered list of ids
"""
self.has_cycles = False
- edges = []
- all_ids = set( self.steps.keys() )
- for step in self.steps.values():
- for name, conn in step.input_connections.iteritems():
- if conn is not None:
- other_node_id, _ = conn
- edges.append( ( other_node_id, step.id ) )
+ edges = self.edge_list()
try:
node_order = topsort( edges )
- node_order_set = set( node_order )
- node_order.extend( [ id for id in all_ids if id not in node_order ] )
+ #node_order_set = set( node_order )
+ #node_order.extend( [ id for id in all_ids if id not in node_order ] )
self.node_order = node_order
+ return self.node_order
except CycleError:
self.has_cycles = True
self.node_order = None
- return self.node_order
+ return None
+
+ def order_nodes_levels( self ):
+ edges = self.edge_list()
+ try:
+ return topsort_levels( edges )
+ except CycleError:
+ return None
class WorkflowStep( object ):
diff --git a/static/june_2007_style/base.css.tmpl b/static/june_2007_style/base.css.tmpl
index dccd05a10e5..0c7a250edcc 100644
--- a/static/june_2007_style/base.css.tmpl
+++ b/static/june_2007_style/base.css.tmpl
@@ -76,6 +76,15 @@ div.toolFormBody div.toolFormTitle
margin-bottom: 5px;
}
+div.toolFormDisabled div.toolFormTitle {
+ background: ${layout_bg};
+ border-color: ${layout_border};
+}
+
+div.toolFormDisabled {
+ border-color: ${layout_border};
+}
+
div.toolHelp
{
}
@@ -86,6 +95,13 @@ div.toolHelpBody
overflow: auto;
}
+div.titleRow {
+ font-weight: bold;
+ border-bottom: dotted gray 1px;
+ margin-bottom: 0.5em;
+ padding-bottom: 0.25em;
+}
+
/* Forms */
div.form-row
diff --git a/static/june_2007_style/blue/base.css b/static/june_2007_style/blue/base.css
index 53c51eccb63..ad87a6337cc 100644
--- a/static/june_2007_style/blue/base.css
+++ b/static/june_2007_style/blue/base.css
@@ -76,6 +76,15 @@ div.toolFormBody div.toolFormTitle
margin-bottom: 5px;
}
+div.toolFormDisabled div.toolFormTitle {
+ background: #eee;
+ border-color: #999;
+}
+
+div.toolFormDisabled {
+ border-color: #999;
+}
+
div.toolHelp
{
}
@@ -86,6 +95,13 @@ div.toolHelpBody
overflow: auto;
}
+div.titleRow {
+ font-weight: bold;
+ border-bottom: dotted gray 1px;
+ margin-bottom: 0.5em;
+ padding-bottom: 0.25em;
+}
+
/* Forms */
div.form-row
@@ -129,7 +145,7 @@ select, textarea, input[type="text"], input[type="file"]
/* Messages */
-.errormessage, .warningmessage, .donemessage, .infomessage, .welcomeBlue, .welcomeRed
+.errormessage, .warningmessage, .donemessage, .infomessage, .welcomemessage
{
padding: 10px;
padding-left: 52px;
@@ -162,20 +178,13 @@ select, textarea, input[type="text"], input[type="file"]
background-color: #CCCCFF;
}
-.welcomeBlue {
+.welcomemessage {
padding-left: 10px;
border-color: #6666AA;
background-color: #CCCCFF;
background-image: none;
}
-.welcomeRed {
- padding-left: 10px;
- border-color: #AA6666;
- background-color: #FFCCCC;
- background-image: none;
-}
-
.errormessagesmall, .warningmessagesmall, .donemessagesmall, .infomessagesmall
{
padding: 5px;
diff --git a/static/june_2007_style/blue/base_bg.png b/static/june_2007_style/blue/base_bg.png
index 2b1fa2e7f8480846e5fba27a9a133c98a1a13815..882f8d3597194c409b1100ad44b2f8848f820a36 100644
GIT binary patch
delta 17
ZcmaFC_=0hQJ-ZmU!p{{meou5;1OPsR2bTZ<
delta 17
YcmaFC_=0hQJv*zIgu$}FfQfF405oC+%m4rY
diff --git a/static/june_2007_style/blue/error_bg.png b/static/june_2007_style/blue/error_bg.png
index 355651aec820e39a2a5d4a55cabd5ff27049c95f..f5c4edddef8e7d884fb8e0f3f66d8d71463756c3 100644
GIT binary patch
delta 17
YcmZ3%xPozlJ-ZmUV&D1|xf9)z05PBi_5c6?
delta 17
YcmZ3%xPozlJv*zIr2E3eV-wwy04~)9r2qf`
diff --git a/static/june_2007_style/blue/footer_title_bg.png b/static/june_2007_style/blue/footer_title_bg.png
index fe2cf633d92115533cb1243d5f60be1abe171704..5b5bea63ca81ccf68ba7273abcadc02e92216047 100644
GIT binary patch
delta 18
ZcmX@dbdG6)J-ZmUV&D1|xf|U$7y&$w1`7ZH
delta 18
ZcmX@dbdG6)Jv*zIr2E3eV;kK#7y&tR1-bwL
diff --git a/static/june_2007_style/blue/form_body_bg.png b/static/june_2007_style/blue/form_body_bg.png
index 93799a55cb6ea88b6984c1eb9d038f1a9786ed64..584f9a1813214343b58d97b39f502a6af2a2919e 100644
GIT binary patch
delta 17
ZcmdnQxQTIsJ-ZmU!p{{meou7E0RT6R2K4{{
delta 17
YcmdnQxQTIsJv*zIgu$}FfQfE704;0nC-xJ-e05KZ|UjP6A
delta 17
YcmZo*Y+#&V&(10)VX!PPV4|B904DkblmGw#
diff --git a/static/june_2007_style/blue/menu_bg.png b/static/june_2007_style/blue/menu_bg.png
index dc2ef23c4dfaf152a69a06ef7e0edb0c42b3c0c7..16c4d5f170f8b577bd5f8b0d17b492f16b4fdd5d 100644
GIT binary patch
delta 18
acmeys@_}W7J-ZmU!p{{mes6TsVgdk190ykb
delta 18
Zcmeys@_}W7Jv*zIgu$}FfQ@ciOaMDD1&jay
diff --git a/static/june_2007_style/blue/ok_bg.png b/static/june_2007_style/blue/ok_bg.png
index da05f23a96ede33e906e6a394f2ee30f1ad92d6d..3ba07fa901c2a92cb5b6704bb99e4b4334e168a6 100644
GIT binary patch
delta 17
YcmZ3%xPozlJ-ZmUV&D1|xf9)z05PBi_5c6?
delta 17
YcmZ3%xPozlJv*zIr2E3eV-wwy04~)9r2qf`
diff --git a/static/june_2007_style/blue/panel_header_bg.png b/static/june_2007_style/blue/panel_header_bg.png
index 8bad7f6c81527ef2aa8c40e874766892174f6330..39f9520cb1f22a411e50445c7bf06c53af491250 100644
GIT binary patch
delta 17
ZcmbQhIDv73J-ZmU!p{{meou7s0su2B29E#$
delta 17
YcmbQhIDv73Jv*zIgu$}FfQfEi04WLt!~g&Q
diff --git a/static/june_2007_style/blue/popupmenu_callout_top.png b/static/june_2007_style/blue/popupmenu_callout_top.png
index ec09a444ada924f6142703235f3255f9c6d1126a..4318e8eea517163b7b6e7afd38f12691542179d5 100644
GIT binary patch
delta 18
ZcmbQkJcoILJ-ZmUV&D1|xf|WW7y&k?1@`~|
delta 18
ZcmbQkJcoILJv*zI)V&o3H5=W+7y&iq1
+
+
+
+
+
+Galaxy
+
+
+
+
+
+
+
+
+
+
+
+
+<%def name="history_item( data )">
+ %if data.state in [ "no state", "", None ]:
+ <% data_state = "queued" %>
+ %else:
+ <% data_state = data.state %>
+ %endif
+
+
+
+ %if data_state != 'ok':
+
+
+
+ %endif
+
+
+ ${data.hid}: ${data.display_name()}
+
+
+
+
+
+%def>
+
+
The following list contains each tool that was run to create the
+datasets in your current history. Please select those that you wish
+to include in the workflow.
+
+
Tools which cannot be run interactively and thus cannot be incorporated
+into a workflow will be shown in gray.
+
+%for warning in warnings:
+
${warning}
+%endfor
+
+
+
+
+
+
\ No newline at end of file
diff --git a/templates/history_options.tmpl b/templates/history_options.tmpl
index 78b22f0a5b7..c236a35d141 100644
--- a/templates/history_options.tmpl
+++ b/templates/history_options.tmpl
@@ -33,9 +33,12 @@ q( "a[@confirm]" ).click( function() {
#if $len( $history.active_datasets ) > 0 and $history.user not in [ None , "" ]: