Merge pull request #1353 from jmchilton/output_refactor

Refactor tool output classes out of galaxy.tools and into parser module.

lib/galaxy/tools/__init__.py

@@ -37,6 +37,7 @@ from galaxy.tools.parameters.validation import LateValidationError
 from galaxy.tools.test import parse_tests
 from galaxy.tools.parser import get_tool_source
 from galaxy.tools.parser.xml import XmlPageSource
+from galaxy.tools.parser import ToolOutputCollectionPart
 from galaxy.tools.toolbox import AbstractToolBox
 from galaxy.util import rst_to_html, string_as_bool
 from galaxy.util import ExecutionTimer
@@ -226,222 +227,6 @@ class DefaultToolState( object ):
         return new_state
 
 
-class ToolOutputBase( object, Dictifiable ):
-
-    def __init__( self, name, label=None, filters=None, hidden=False ):
-        super( ToolOutputBase, self ).__init__()
-        self.name = name
-        self.label = label
-        self.filters = filters or []
-        self.hidden = hidden
-        self.collection = False
-
-
-class ToolOutput( ToolOutputBase ):
-    """
-    Represents an output datasets produced by a tool. For backward
-    compatibility this behaves as if it were the tuple::
-
-      (format, metadata_source, parent)
-    """
-
-    dict_collection_visible_keys = ( 'name', 'format', 'label', 'hidden' )
-
-    def __init__( self, name, format=None, format_source=None, metadata_source=None,
-                  parent=None, label=None, filters=None, actions=None, hidden=False,
-                  implicit=False ):
-        super( ToolOutput, self ).__init__( name, label=label, filters=filters, hidden=hidden )
-        self.format = format
-        self.format_source = format_source
-        self.metadata_source = metadata_source
-        self.parent = parent
-        self.actions = actions
-
-        # Initialize default values
-        self.change_format = []
-        self.implicit = implicit
-        self.from_work_dir = None
-
-    # Tuple emulation
-
-    def __len__( self ):
-        return 3
-
-    def __getitem__( self, index ):
-        if index == 0:
-            return self.format
-        elif index == 1:
-            return self.metadata_source
-        elif index == 2:
-            return self.parent
-        else:
-            raise IndexError( index )
-
-    def __iter__( self ):
-        return iter( ( self.format, self.metadata_source, self.parent ) )
-
-    def to_dict( self, view='collection', value_mapper=None, app=None ):
-        as_dict = super( ToolOutput, self ).to_dict( view=view, value_mapper=value_mapper )
-        format = self.format
-        if format and format != "input" and app:
-            edam_format = app.datatypes_registry.edam_formats.get(self.format)
-            as_dict["edam_format"] = edam_format
-        return as_dict
-
-
-class ToolOutputCollection( ToolOutputBase ):
-    """
-    Represents a HistoryDatasetCollectionAssociation of output datasets produced
-    by a tool.
-
-    <outputs>
-      <collection type="list" label="${tool.name} on ${on_string} fasta">
-        <discover_datasets pattern="__name__" ext="fasta" visible="True" directory="outputFiles" />
-      </collection>
-      <collection type="paired" label="${tool.name} on ${on_string} paired reads">
-        <data name="forward" format="fastqsanger" />
-        <data name="reverse" format="fastqsanger"/>
-      </collection>
-    <outputs>
-    """
-
-    def __init__(
-        self,
-        name,
-        structure,
-        label=None,
-        filters=None,
-        hidden=False,
-        default_format="data",
-        default_format_source=None,
-        default_metadata_source=None,
-        inherit_format=False,
-        inherit_metadata=False
-    ):
-        super( ToolOutputCollection, self ).__init__( name, label=label, filters=filters, hidden=hidden )
-        self.collection = True
-        self.default_format = default_format
-        self.structure = structure
-        self.outputs = odict()
-
-        self.inherit_format = inherit_format
-        self.inherit_metadata = inherit_metadata
-
-        self.metadata_source = default_metadata_source
-        self.format_source = default_format_source
-        self.change_format = []  # TODO
-
-    def known_outputs( self, inputs, type_registry ):
-        if self.dynamic_structure:
-            return []
-
-        # This line is probably not right - should verify structured_like
-        # or have outputs and all outputs have name.
-        if len( self.outputs ) > 1:
-            output_parts = [ToolOutputCollectionPart(self, k, v) for k, v in self.outputs.iteritems()]
-        else:
-            # either must have specified structured_like or something worse
-            if self.structure.structured_like:
-                collection_prototype = inputs[ self.structure.structured_like ].collection
-            else:
-                collection_prototype = type_registry.prototype( self.structure.collection_type )
-
-            def prototype_dataset_element_to_output( element, parent_ids=[] ):
-                name = element.element_identifier
-                format = self.default_format
-                if self.inherit_format:
-                    format = element.dataset_instance.ext
-                output = ToolOutput(
-                    name,
-                    format=format,
-                    format_source=self.format_source,
-                    metadata_source=self.metadata_source,
-                    implicit=True,
-                )
-                if self.inherit_metadata:
-                    output.metadata_source = element.dataset_instance
-                return ToolOutputCollectionPart(
-                    self,
-                    element.element_identifier,
-                    output,
-                    parent_ids=parent_ids,
-                )
-
-            def prototype_collection_to_output( collection_prototype, parent_ids=[] ):
-                output_parts = []
-                for element in collection_prototype.elements:
-                    element_parts = []
-                    if not element.is_collection:
-                        element_parts.append(prototype_dataset_element_to_output( element, parent_ids ))
-                    else:
-                        new_parent_ids = parent_ids[:] + [element.element_identifier]
-                        element_parts.extend(prototype_collection_to_output(element.element_object, new_parent_ids))
-                    output_parts.extend(element_parts)
-
-                return output_parts
-
-            output_parts = prototype_collection_to_output( collection_prototype )
-
-        return output_parts
-
-    @property
-    def dynamic_structure(self):
-        return self.structure.dynamic
-
-    @property
-    def dataset_collector_descriptions(self):
-        if not self.dynamic_structure:
-            raise Exception("dataset_collector_descriptions called for output collection with static structure")
-        return self.structure.dataset_collector_descriptions
-
-
-class ToolOutputCollectionStructure( object ):
-
-    def __init__(
-        self,
-        collection_type,
-        collection_type_source,
-        structured_like,
-        dataset_collector_descriptions,
-    ):
-        self.collection_type = collection_type
-        self.collection_type_source = collection_type_source
-        self.structured_like = structured_like
-        self.dataset_collector_descriptions = dataset_collector_descriptions
-        if collection_type and collection_type_source:
-            raise ValueError("Cannot set both type and type_source on collection output.")
-        if collection_type is None and structured_like is None and dataset_collector_descriptions is None and collection_type_source is None:
-            raise ValueError( "Output collection types must be specify type of structured_like" )
-        if dataset_collector_descriptions and structured_like:
-            raise ValueError( "Cannot specify dynamic structure (discovered_datasets) and structured_like attribute." )
-        self.dynamic = dataset_collector_descriptions is not None
-
-
-class ToolOutputCollectionPart( object ):
-
-    def __init__( self, output_collection_def, element_identifier, output_def, parent_ids=[] ):
-        self.output_collection_def = output_collection_def
-        self.element_identifier = element_identifier
-        self.output_def = output_def
-        self.parent_ids = parent_ids
-
-    @property
-    def effective_output_name( self ):
-        name = self.output_collection_def.name
-        part_name = self.element_identifier
-        effective_output_name = "%s|__part__|%s" % ( name, part_name )
-        return effective_output_name
-
-    @staticmethod
-    def is_named_collection_part_name( name ):
-        return "|__part__|" in name
-
-    @staticmethod
-    def split_output_name( name ):
-        assert ToolOutputCollectionPart.is_named_collection_part_name( name )
-        return name.split("|__part__|")
-
-
 class Tool( object, Dictifiable ):
     """
     Represents a computational tool that can be executed through Galaxy.

lib/galaxy/tools/actions/__init__.py

@@ -660,7 +660,7 @@ def filter_output(output, incoming):
 
 def determine_output_format(output, parameter_context, input_datasets, input_dataset_collections, random_input_ext):
     """ Determines the output format for a dataset based on an abstract
-    description of the output (galaxy.tools.ToolOutput), the parameter
+    description of the output (galaxy.tools.parser.ToolOutput), the parameter
     wrappers, a map of the input datasets (name => HDA), and the last input
     extensions in the tool form.
 

lib/galaxy/tools/execute.py

@@ -4,7 +4,7 @@ from various states, tracking results, and building implicit dataset
 collections from matched collections.
 """
 import collections
-import galaxy.tools
+from galaxy.tools.parser import ToolOutputCollectionPart
 from galaxy.util import ExecutionTimer
 from galaxy.tools.actions import on_text_for_names, ToolExecutionCache
 from threading import Thread
@@ -104,7 +104,7 @@ class ToolExecutionTracker( object ):
         self.successful_jobs.append( job )
         self.output_datasets.extend( outputs )
         for output_name, output_dataset in outputs:
-            if galaxy.tools.ToolOutputCollectionPart.is_named_collection_part_name( output_name ):
+            if ToolOutputCollectionPart.is_named_collection_part_name( output_name ):
                 # Skip known collection outputs, these will be covered by
                 # output collections.
                 continue

lib/galaxy/tools/parser/__init__.py

@@ -3,5 +3,8 @@
 from .interface import ToolSource
 from .factory import get_tool_source
 from .factory import get_input_source
+from .output_objects import (
+    ToolOutputCollectionPart,
+)
 
-__all__ = ["ToolSource", "get_tool_source", "get_input_source"]
+__all__ = ["ToolSource", "get_tool_source", "get_input_source", "ToolOutputCollectionPart"]

lib/galaxy/tools/parser/output_actions.py

@@ -205,7 +205,7 @@ class FromFileToolOutputActionOption( ToolOutputActionOption ):
         try:
             if options:
                 return str( options[ self.offset ][ self.column ] )
-        except Exception, e:
+        except Exception as e:
             log.debug( "Error in FromFileToolOutputActionOption get_value: %s" % e )
         return None
 
@@ -253,7 +253,7 @@ class FromParamToolOutputActionOption( ToolOutputActionOption ):
         try:
             if options:
                 return str( options[ self.offset ][ self.column ] )
-        except Exception, e:
+        except Exception as e:
             log.debug( "Error in FromParamToolOutputActionOption get_value: %s" % e )
         return None
 
@@ -288,7 +288,7 @@ class FromDataTableOutputActionOption( ToolOutputActionOption ):
         try:
             if options:
                 return str( options[ self.offset ][ self.column ] )
-        except Exception, e:
+        except Exception as e:
             log.debug( "Error in FromDataTableOutputActionOption get_value: %s" % e )
         return None
 
@@ -383,7 +383,7 @@ class ParamValueToolOutputActionOptionFilter( ToolOutputActionOptionFilter ):
             try:
                 if self.keep == ( self.compare( self.cast( fields[self.column] ), value ) ):
                     rval.append( fields )
-            except Exception, e:
+            except Exception as e:
                 log.debug(e)
                 continue  # likely a bad cast or column out of range
         return rval

lib/galaxy/tools/parser/output_objects.py

@@ -0,0 +1,218 @@
+from galaxy.util.dictifiable import Dictifiable
+from galaxy.util.odict import odict
+
+
+class ToolOutputBase( object, Dictifiable ):
+
+    def __init__( self, name, label=None, filters=None, hidden=False ):
+        super( ToolOutputBase, self ).__init__()
+        self.name = name
+        self.label = label
+        self.filters = filters or []
+        self.hidden = hidden
+        self.collection = False
+
+
+class ToolOutput( ToolOutputBase ):
+    """
+    Represents an output datasets produced by a tool. For backward
+    compatibility this behaves as if it were the tuple::
+
+      (format, metadata_source, parent)
+    """
+
+    dict_collection_visible_keys = ( 'name', 'format', 'label', 'hidden' )
+
+    def __init__( self, name, format=None, format_source=None, metadata_source=None,
+                  parent=None, label=None, filters=None, actions=None, hidden=False,
+                  implicit=False ):
+        super( ToolOutput, self ).__init__( name, label=label, filters=filters, hidden=hidden )
+        self.format = format
+        self.format_source = format_source
+        self.metadata_source = metadata_source
+        self.parent = parent
+        self.actions = actions
+
+        # Initialize default values
+        self.change_format = []
+        self.implicit = implicit
+        self.from_work_dir = None
+
+    # Tuple emulation
+
+    def __len__( self ):
+        return 3
+
+    def __getitem__( self, index ):
+        if index == 0:
+            return self.format
+        elif index == 1:
+            return self.metadata_source
+        elif index == 2:
+            return self.parent
+        else:
+            raise IndexError( index )
+
+    def __iter__( self ):
+        return iter( ( self.format, self.metadata_source, self.parent ) )
+
+    def to_dict( self, view='collection', value_mapper=None, app=None ):
+        as_dict = super( ToolOutput, self ).to_dict( view=view, value_mapper=value_mapper )
+        format = self.format
+        if format and format != "input" and app:
+            edam_format = app.datatypes_registry.edam_formats.get(self.format)
+            as_dict["edam_format"] = edam_format
+        return as_dict
+
+
+class ToolOutputCollection( ToolOutputBase ):
+    """
+    Represents a HistoryDatasetCollectionAssociation of output datasets produced
+    by a tool.
+
+    <outputs>
+      <collection type="list" label="${tool.name} on ${on_string} fasta">
+        <discover_datasets pattern="__name__" ext="fasta" visible="True" directory="outputFiles" />
+      </collection>
+      <collection type="paired" label="${tool.name} on ${on_string} paired reads">
+        <data name="forward" format="fastqsanger" />
+        <data name="reverse" format="fastqsanger"/>
+      </collection>
+    <outputs>
+    """
+
+    def __init__(
+        self,
+        name,
+        structure,
+        label=None,
+        filters=None,
+        hidden=False,
+        default_format="data",
+        default_format_source=None,
+        default_metadata_source=None,
+        inherit_format=False,
+        inherit_metadata=False
+    ):
+        super( ToolOutputCollection, self ).__init__( name, label=label, filters=filters, hidden=hidden )
+        self.collection = True
+        self.default_format = default_format
+        self.structure = structure
+        self.outputs = odict()
+
+        self.inherit_format = inherit_format
+        self.inherit_metadata = inherit_metadata
+
+        self.metadata_source = default_metadata_source
+        self.format_source = default_format_source
+        self.change_format = []  # TODO
+
+    def known_outputs( self, inputs, type_registry ):
+        if self.dynamic_structure:
+            return []
+
+        # This line is probably not right - should verify structured_like
+        # or have outputs and all outputs have name.
+        if len( self.outputs ) > 1:
+            output_parts = [ToolOutputCollectionPart(self, k, v) for k, v in self.outputs.iteritems()]
+        else:
+            # either must have specified structured_like or something worse
+            if self.structure.structured_like:
+                collection_prototype = inputs[ self.structure.structured_like ].collection
+            else:
+                collection_prototype = type_registry.prototype( self.structure.collection_type )
+
+            def prototype_dataset_element_to_output( element, parent_ids=[] ):
+                name = element.element_identifier
+                format = self.default_format
+                if self.inherit_format:
+                    format = element.dataset_instance.ext
+                output = ToolOutput(
+                    name,
+                    format=format,
+                    format_source=self.format_source,
+                    metadata_source=self.metadata_source,
+                    implicit=True,
+                )
+                if self.inherit_metadata:
+                    output.metadata_source = element.dataset_instance
+                return ToolOutputCollectionPart(
+                    self,
+                    element.element_identifier,
+                    output,
+                    parent_ids=parent_ids,
+                )
+
+            def prototype_collection_to_output( collection_prototype, parent_ids=[] ):
+                output_parts = []
+                for element in collection_prototype.elements:
+                    element_parts = []
+                    if not element.is_collection:
+                        element_parts.append(prototype_dataset_element_to_output( element, parent_ids ))
+                    else:
+                        new_parent_ids = parent_ids[:] + [element.element_identifier]
+                        element_parts.extend(prototype_collection_to_output(element.element_object, new_parent_ids))
+                    output_parts.extend(element_parts)
+
+                return output_parts
+
+            output_parts = prototype_collection_to_output( collection_prototype )
+
+        return output_parts
+
+    @property
+    def dynamic_structure(self):
+        return self.structure.dynamic
+
+    @property
+    def dataset_collector_descriptions(self):
+        if not self.dynamic_structure:
+            raise Exception("dataset_collector_descriptions called for output collection with static structure")
+        return self.structure.dataset_collector_descriptions
+
+
+class ToolOutputCollectionStructure( object ):
+
+    def __init__(
+        self,
+        collection_type,
+        collection_type_source,
+        structured_like,
+        dataset_collector_descriptions,
+    ):
+        self.collection_type = collection_type
+        self.collection_type_source = collection_type_source
+        self.structured_like = structured_like
+        self.dataset_collector_descriptions = dataset_collector_descriptions
+        if collection_type and collection_type_source:
+            raise ValueError("Cannot set both type and type_source on collection output.")
+        if collection_type is None and structured_like is None and dataset_collector_descriptions is None and collection_type_source is None:
+            raise ValueError( "Output collection types must be specify type of structured_like" )
+        if dataset_collector_descriptions and structured_like:
+            raise ValueError( "Cannot specify dynamic structure (discovered_datasets) and structured_like attribute." )
+        self.dynamic = dataset_collector_descriptions is not None
+
+
+class ToolOutputCollectionPart( object ):
+
+    def __init__( self, output_collection_def, element_identifier, output_def, parent_ids=[] ):
+        self.output_collection_def = output_collection_def
+        self.element_identifier = element_identifier
+        self.output_def = output_def
+        self.parent_ids = parent_ids
+
+    @property
+    def effective_output_name( self ):
+        name = self.output_collection_def.name
+        part_name = self.element_identifier
+        effective_output_name = "%s|__part__|%s" % ( name, part_name )
+        return effective_output_name
+
+    @staticmethod
+    def is_named_collection_part_name( name ):
+        return "|__part__|" in name
+
+    @staticmethod
+    def split_output_name( name ):
+        assert ToolOutputCollectionPart.is_named_collection_part_name( name )
+        return name.split("|__part__|")

lib/galaxy/tools/parser/xml.py

@@ -24,7 +24,12 @@ from .output_actions import ToolOutputActionGroup
 from galaxy.util import string_as_bool, xml_text, xml_to_string
 from galaxy.util.odict import odict
 from galaxy.tools.deps import requirements
-import galaxy.tools
+from .output_objects import (
+    ToolOutput,
+    ToolOutputCollection,
+    ToolOutputCollectionStructure
+)
+
 
 log = logging.getLogger( __name__ )
 
@@ -191,13 +196,13 @@ class XmlToolSource(ToolSource):
             dataset_collector_descriptions = None
             if collection_elem.find( "discover_datasets" ) is not None:
                 dataset_collector_descriptions = dataset_collector_descriptions_from_elem( collection_elem )
-            structure = galaxy.tools.ToolOutputCollectionStructure(
+            structure = ToolOutputCollectionStructure(
                 collection_type=collection_type,
                 collection_type_source=collection_type_source,
                 structured_like=structured_like,
                 dataset_collector_descriptions=dataset_collector_descriptions,
             )
-            output_collection = galaxy.tools.ToolOutputCollection(
+            output_collection = ToolOutputCollection(
                 name,
                 structure,
                 label=label,
@@ -238,7 +243,7 @@ class XmlToolSource(ToolSource):
         default_format_source=None,
         default_metadata_source="",
     ):
-        output = galaxy.tools.ToolOutput( data_elem.get("name") )
+        output = ToolOutput( data_elem.get("name") )
         output_format = data_elem.get("format", default_format)
         auto_format = string_as_bool( data_elem.get( "auto_format", "false" ) )
         if auto_format and output_format != "data":

lib/galaxy/tools/parser/yaml.py

@@ -8,7 +8,9 @@ from galaxy.tools.deps import requirements
 from .output_collection_def import dataset_collector_descriptions_from_list
 from .output_actions import ToolOutputActionGroup
 from galaxy.util.odict import odict
-import galaxy.tools
+from .output_objects import (
+    ToolOutput
+)
 
 
 class YamlToolSource(ToolSource):
@@ -79,7 +81,7 @@ class YamlToolSource(ToolSource):
 
     def _parse_output(self, tool, name, output_dict):
         # TODO: handle filters, actions, change_format
-        output = galaxy.tools.ToolOutput( name )
+        output = ToolOutput( name )
         output.format = output_dict.get("format", "data")
         output.change_format = []
         output.format_source = output_dict.get("format_source", None)

lib/galaxy/workflow/extract.py

@@ -11,7 +11,7 @@ from galaxy.tools.parameters.basic import (
     SelectToolParameter,
     UnvalidatedValue
 )
-from galaxy.tools import ToolOutputCollectionPart
+from galaxy.tools.parser import ToolOutputCollectionPart
 from galaxy.tools.parameters.grouping import (
     Conditional,
     Repeat

test/unit/tools/test_actions.py

@@ -2,7 +2,7 @@ import string
 import unittest
 
 from galaxy import model
-from galaxy.tools import ToolOutput
+from galaxy.tools.parser.output_objects import ToolOutput
 from galaxy.tools.actions import DefaultToolAction
 from galaxy.tools.actions import on_text_for_names
 from galaxy.tools.actions import determine_output_format

test/unit/tools/test_evaluation.py

@@ -15,7 +15,7 @@ from galaxy.util.bunch import Bunch
 
 # For MockTool
 from galaxy.tools.parameters import params_from_strings
-from galaxy.tools import ToolOutput
+from galaxy.tools.parser.output_objects import ToolOutput
 from galaxy.tools.parameters.grouping import Repeat
 from galaxy.tools.parameters.grouping import Conditional
 from galaxy.tools.parameters.grouping import ConditionalWhen