Extend workflow editor to allow output collections in tools.

Mostly test cases - because I guess I could have sworn it was going to be harder than this.

client/galaxy/scripts/galaxy.workflow_editor.canvas.js

@@ -377,14 +377,15 @@ var BaseInputTerminal = Terminal.extend( {
         return false;
     },
     _otherCollectionType: function( other ) {
+        // Effective collection type for other - base collection type
+        // with map over appended.
         var otherCollectionType = NULL_COLLECTION_TYPE_DESCRIPTION;
         if( other.isCollection ) {
             otherCollectionType = other.collectionType;
-        } else {
-            var otherMapOver = other.mapOver();
-            if( otherMapOver.isCollection ) {
-                otherCollectionType = otherMapOver;
-            }
+        }
+        var otherMapOver = other.mapOver();
+        if( otherMapOver.isCollection ) {
+            otherCollectionType = otherMapOver.append(otherCollectionType);
         }
         return otherCollectionType;
     },
@@ -513,7 +514,7 @@ var InputCollectionTerminal = BaseInputTerminal.extend( {
     }
 });
 
-var OutputCollectionTerminal = Terminal.extend( {
+var OutputCollectionTerminal = OutputTerminal.extend( {
     initialize: function( attr ) {
         Terminal.prototype.initialize.call( this, attr );
         this.datatypes = attr.datatypes;
@@ -1413,7 +1414,7 @@ var NodeView = Backbone.View.extend( {
     },
 
     addDataOutput: function( output ) {
-        var terminalViewClass = ( output.collection_type ) ? OutputCollectionTerminalView : OutputTerminalView;
+        var terminalViewClass = ( output.collection ) ? OutputCollectionTerminalView : OutputTerminalView;
         var terminalView = new terminalViewClass( {
             node: this.node,
             output: output

lib/galaxy/tools/__init__.py

@@ -496,8 +496,16 @@ class Tool( object, Dictifiable ):
         return None
 
     @property
-    def produces_collections( self ):
-        return any( o.collection for o in self.outputs.values() )
+    def produces_collections_of_unknown_type( self ):
+
+        def output_is_dynamic_collection(output):
+            if not output.collection:
+                return False
+            if output.structure.collection_type:
+                return False
+            return True
+
+        return any( map( output_is_dynamic_collection,  self.outputs.values() ) )
 
     def __get_job_tool_configuration(self, job_params=None):
         """Generalized method for getting this tool's job configuration.
@@ -1118,8 +1126,8 @@ class Tool( object, Dictifiable ):
         if self.tool_type.startswith( 'data_source' ):
             return False
 
-        if self.produces_collections:
-            # Someday we will get there!
+        if self.produces_collections_of_unknown_type:
+            # Getting there...
             return False
 
         if hasattr( tool_source, "root"):

lib/galaxy/workflow/modules.py

@@ -403,7 +403,14 @@ class InputDataCollectionModule( InputModule ):
         return form
 
     def get_data_outputs( self ):
-        return [ dict( name='output', extensions=['input_collection'], collection_type=self.state[ 'collection_type' ] ) ]
+        return [
+            dict(
+                name='output',
+                extensions=['input_collection'],
+                collection=True,
+                collection_type=self.state[ 'collection_type' ]
+            )
+        ]
 
 
 class PauseModule( SimpleWorkflowModule ):
@@ -655,9 +662,12 @@ class ToolModule( WorkflowModule ):
         data_outputs = []
         data_inputs = None
         for name, tool_output in self.tool.outputs.iteritems():
+            extra_kwds = {}
             if tool_output.collection:
-                formats = [ 'input' ]
-            elif tool_output.format_source != None:
+                extra_kwds["collection"] = True
+                extra_kwds["collection_type"] = tool_output.structure.collection_type
+                formats = [ 'input' ]  # TODO: fix
+            elif tool_output.format_source is not None:
                 formats = [ 'input' ]  # default to special name "input" which remove restrictions on connections
                 if data_inputs == None:
                     data_inputs = self.get_data_inputs()
@@ -674,7 +684,13 @@ class ToolModule( WorkflowModule ):
                     format = when_elem.get( 'format', None )
                     if format and format not in formats:
                         formats.append( format )
-            data_outputs.append( dict( name=name, extensions=formats ) )
+            data_outputs.append(
+                dict(
+                    name=name,
+                    extensions=formats,
+                    **extra_kwds
+                )
+            )
         return data_outputs
 
     def get_runtime_input_dicts( self, step_annotation ):

test/qunit/tests/workflow_editor_tests.js

@@ -301,6 +301,37 @@ define([
         } );
     } );
 
+    module( "Input collection terminal model test", {
+        setup: function( ) {
+            this.node = new Node( {  } );
+            this.input = { extensions: [ "txt" ], collection_type: "list" };
+            this.input_terminal = new InputCollectionTerminal( { input: this.input } );
+            this.input_terminal.node = this.node;
+        }
+    } );
+
+    test( "Collection output can connect to same collection input type", function() {
+        var self = this;
+        var inputTerminal = self.input_terminal;
+        var outputTerminal = new OutputCollectionTerminal( {
+            datatypes: 'txt',
+            collection_type: 'list'
+        } );
+        outputTerminal.node = {};
+        ok( this.input_terminal.canAccept( outputTerminal ) );
+    } );
+
+    test( "Collection output cannot connect to different collection input type", function() {
+        var self = this;
+        var inputTerminal = self.input_terminal;
+        var outputTerminal = new OutputCollectionTerminal( {
+            datatypes: 'txt',
+            collection_type: 'paired'
+        } );
+        outputTerminal.node = {};
+        ok( ! this.input_terminal.canAccept( outputTerminal ) );
+    } );
+
     module( "Node unit test", {
         setup: function() {
             this.input_terminal = { destroy: sinon.spy(), redraw: sinon.spy() };
@@ -797,27 +828,54 @@ define([
             }
             return outputTerminal;
         },
+        newOutputCollectionTerminal: function( collectionType, output, node, mapOver ) {
+            collectionType = collectionType || "list";
+            output = output || {};
+            node = node || this.newNode();
+            if( ! ( 'extensions' in output ) ) {
+                output[ 'extensions'] = [ 'data' ];
+            }
+            var outputEl = $("<div>")[ 0 ];
+            var outputTerminal = new OutputCollectionTerminal( { element: outputEl, datatypes: output.extensions, collection_type: collectionType } );
+            var outputTerminalMapping = new OutputCollectionTerminalMapping( { terminal: outputTerminal } );
+            outputTerminal.node = node;
+            if( mapOver ) {
+                outputTerminal.setMapOver( new CollectionTypeDescription( mapOver ) );
+            }
+            return outputTerminal;
+        },
         newNode: function( ) {
             var nodeEl = $("<div>")[ 0 ];
             var node = new Node( { element: nodeEl } );
             return node;
         },
-        addOutput: function( terminal, connected ) {
+        _addExistingOutput: function( terminal, output, connected ) {
             var self = this;
-            var connectedOutput = this.newOutputTerminal();
             var node = terminal.node;
             if( connected ) {
                 with_workflow_global( function() {
                     var inputTerminal = self.newInputTerminal();
-                    new Connector( inputTerminal, connectedOutput );
+                    new Connector( inputTerminal, output );
                 } );
             }
-            this._addTerminalTo( connectedOutput, node.output_terminals );
-            return connectedOutput;
+            this._addTerminalTo( output, node.output_terminals );
+            return output;
+        },
+        addOutput: function( terminal, connected ) {
+            var connectedOutput = this.newOutputTerminal();
+            return this._addExistingOutput( terminal, connectedOutput, connected );
+        },
+        addCollectionOutput: function( terminal, connected ) {
+            var collectionOutput = this.newOutputCollectionTerminal();
+            return this._addExistingOutput( terminal, collectionOutput, connected );
         },
         addConnectedOutput: function( terminal ) {
             return this.addOutput( terminal, true );
         },
+        addConnectedCollectionOutput: function( terminal ) {
+            var connectedOutput = this.newOutputCollectionTerminal();
+            return this._addExistingOutput( terminal, connectedOutput, true );
+        },
         addConnectedInput: function( terminal ) {
             var self = this;
             var connectedInput = this.newInputTerminal();
@@ -892,6 +950,54 @@ define([
         this.verifyNotAttachable( inputTerminal1, "list" );
     } );
 
+    test( "unmapped input can be attached to by output collection if matching connected input terminals map type", function() {
+        var inputTerminal1 = this.newInputTerminal();
+        var connectedInput1 = this.addConnectedInput( inputTerminal1 );
+        var connectedInput2 = this.addConnectedInput( inputTerminal1 );
+        connectedInput2.setMapOver( new CollectionTypeDescription( "list") );
+        var outputTerminal = this.newOutputCollectionTerminal( "list" );
+        this.verifyAttachable( inputTerminal1, outputTerminal );
+    } );
+
+    test( "unmapped input cannot be attached to by output collection if matching connected input terminals don't match map type", function() {
+        var inputTerminal1 = this.newInputTerminal();
+        var connectedInput1 = this.addConnectedInput( inputTerminal1 );
+        var connectedInput2 = this.addConnectedInput( inputTerminal1 );
+        connectedInput2.setMapOver( new CollectionTypeDescription( "list") );
+        var outputTerminal = this.newOutputCollectionTerminal( "paired" );
+        this.verifyNotAttachable( inputTerminal1, outputTerminal );
+    } );
+
+    test( "unmapped input can be attached to by output collection if effective output type (output+mapover) is same as mapped over input", function() {
+        var inputTerminal1 = this.newInputTerminal();
+        var connectedInput1 = this.addConnectedInput( inputTerminal1 );
+        var connectedInput2 = this.addConnectedInput( inputTerminal1 );
+        connectedInput2.setMapOver( new CollectionTypeDescription( "list:paired") );
+        var outputTerminal = this.newOutputCollectionTerminal( "paired" );
+        outputTerminal.setMapOver( new CollectionTypeDescription( "list" ) );
+        this.verifyAttachable( inputTerminal1, outputTerminal );
+    } );
+
+    test( "unmapped input cannot be attached to by output collection if effective output type (output+mapover) is not same as mapped over input (1)", function() {
+        var inputTerminal1 = this.newInputTerminal();
+        var connectedInput1 = this.addConnectedInput( inputTerminal1 );
+        var connectedInput2 = this.addConnectedInput( inputTerminal1 );
+        connectedInput2.setMapOver( new CollectionTypeDescription( "list:paired") );
+        var outputTerminal = this.newOutputCollectionTerminal( "list" );
+        outputTerminal.setMapOver( new CollectionTypeDescription( "list" ) );
+        this.verifyNotAttachable( inputTerminal1, outputTerminal );
+    } );
+
+    test( "unmapped input cannot be attached to by output collection if effective output type (output+mapover) is not same as mapped over input (2)", function() {
+        var inputTerminal1 = this.newInputTerminal();
+        var connectedInput1 = this.addConnectedInput( inputTerminal1 );
+        var connectedInput2 = this.addConnectedInput( inputTerminal1 );
+        connectedInput2.setMapOver( new CollectionTypeDescription( "list:paired") );
+        var outputTerminal = this.newOutputCollectionTerminal( "list" );
+        outputTerminal.setMapOver( new CollectionTypeDescription( "paired" ) );
+        this.verifyNotAttachable( inputTerminal1, outputTerminal );
+    } );
+
     test( "unmapped input with unmapped, connected outputs cannot be mapped over", function() {
         // It would invalidate the connections - someday maybe we could try to
         // recursively map over everything down the DAG - it would be expensive
@@ -1010,6 +1116,14 @@ define([
         this.verifyNotMappedOver( output );
     } );
 
+    test( "resetMappingIfNeeded an input resets node collection outputs if they not connected to anything", function() {
+        var inputTerminal1 = this.newInputTerminal( "list" );
+        var output = this.addCollectionOutput( inputTerminal1 );
+        output.setMapOver( new CollectionTypeDescription( "list" ) );
+        inputTerminal1.resetMappingIfNeeded();
+        this.verifyNotMappedOver( output );
+    } );
+
     test( "resetMappingIfNeeded resets if not last mapped over input", function() {
         // Idea here is that other nodes are forcing output to still be mapped
         // over so don't need to disconnect output nodes.
@@ -1028,4 +1142,19 @@ define([
         this.verifyMappedOver( connectedOutput );
     } );
 
+    test( "simple mapping over collection outputs works correctly", function() {
+        var inputTerminal1 = this.newInputTerminal();
+        var connectedOutput = this.addConnectedCollectionOutput( inputTerminal1 );
+        inputTerminal1.setMapOver( new CollectionTypeDescription( "list" ) );
+
+        // Can attach list output of collection type list that is being mapped
+        // over another list to a list:list (because this is what it is) but not
+        // to a list:list:list.
+        var testTerminal2 = this.newInputTerminal( "list:list" );
+        this.verifyAttachable( testTerminal2, connectedOutput );
+
+        var testTerminal1 = this.newInputTerminal( "list:list:list" );
+        this.verifyNotAttachable( testTerminal1, connectedOutput );
+    } );
+
 });