From 61f64e828530c54bae6a090580a6e4278b32a699 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 16 Dec 2015 20:22:29 +0000 Subject: [PATCH] Refactor tool output classes out of galaxy.tools and into parser module. This makes the galaxy.tool.parser much more portable, it no longer has a dependency on anything in galaxy.tools or submodules except galaxy.tools.loader (which in turn has no dependencies on galaxy.tools or submodules). --- lib/galaxy/tools/__init__.py | 217 +-------------------- lib/galaxy/tools/actions/__init__.py | 2 +- lib/galaxy/tools/execute.py | 4 +- lib/galaxy/tools/parser/__init__.py | 5 +- lib/galaxy/tools/parser/output_actions.py | 8 +- lib/galaxy/tools/parser/output_objects.py | 218 ++++++++++++++++++++++ lib/galaxy/tools/parser/xml.py | 13 +- lib/galaxy/tools/parser/yaml.py | 6 +- lib/galaxy/workflow/extract.py | 2 +- test/unit/tools/test_actions.py | 2 +- test/unit/tools/test_evaluation.py | 2 +- 11 files changed, 246 insertions(+), 233 deletions(-) create mode 100644 lib/galaxy/tools/parser/output_objects.py diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 462faf7337d..3ed4324d0b8 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -37,6 +37,7 @@ from galaxy.tools.parameters.validation import LateValidationError from galaxy.tools.test import parse_tests from galaxy.tools.parser import get_tool_source from galaxy.tools.parser.xml import XmlPageSource +from galaxy.tools.parser import ToolOutputCollectionPart from galaxy.tools.toolbox import AbstractToolBox from galaxy.util import rst_to_html, string_as_bool from galaxy.util import ExecutionTimer @@ -226,222 +227,6 @@ class DefaultToolState( object ): return new_state -class ToolOutputBase( object, Dictifiable ): - - def __init__( self, name, label=None, filters=None, hidden=False ): - super( ToolOutputBase, self ).__init__() - self.name = name - self.label = label - self.filters = filters or [] - self.hidden = hidden - self.collection = False - - -class ToolOutput( ToolOutputBase ): - """ - Represents an output datasets produced by a tool. For backward - compatibility this behaves as if it were the tuple:: - - (format, metadata_source, parent) - """ - - dict_collection_visible_keys = ( 'name', 'format', 'label', 'hidden' ) - - def __init__( self, name, format=None, format_source=None, metadata_source=None, - parent=None, label=None, filters=None, actions=None, hidden=False, - implicit=False ): - super( ToolOutput, self ).__init__( name, label=label, filters=filters, hidden=hidden ) - self.format = format - self.format_source = format_source - self.metadata_source = metadata_source - self.parent = parent - self.actions = actions - - # Initialize default values - self.change_format = [] - self.implicit = implicit - self.from_work_dir = None - - # Tuple emulation - - def __len__( self ): - return 3 - - def __getitem__( self, index ): - if index == 0: - return self.format - elif index == 1: - return self.metadata_source - elif index == 2: - return self.parent - else: - raise IndexError( index ) - - def __iter__( self ): - return iter( ( self.format, self.metadata_source, self.parent ) ) - - def to_dict( self, view='collection', value_mapper=None, app=None ): - as_dict = super( ToolOutput, self ).to_dict( view=view, value_mapper=value_mapper ) - format = self.format - if format and format != "input" and app: - edam_format = app.datatypes_registry.edam_formats.get(self.format) - as_dict["edam_format"] = edam_format - return as_dict - - -class ToolOutputCollection( ToolOutputBase ): - """ - Represents a HistoryDatasetCollectionAssociation of output datasets produced - by a tool. - - - - - - - - - - - """ - - def __init__( - self, - name, - structure, - label=None, - filters=None, - hidden=False, - default_format="data", - default_format_source=None, - default_metadata_source=None, - inherit_format=False, - inherit_metadata=False - ): - super( ToolOutputCollection, self ).__init__( name, label=label, filters=filters, hidden=hidden ) - self.collection = True - self.default_format = default_format - self.structure = structure - self.outputs = odict() - - self.inherit_format = inherit_format - self.inherit_metadata = inherit_metadata - - self.metadata_source = default_metadata_source - self.format_source = default_format_source - self.change_format = [] # TODO - - def known_outputs( self, inputs, type_registry ): - if self.dynamic_structure: - return [] - - # This line is probably not right - should verify structured_like - # or have outputs and all outputs have name. - if len( self.outputs ) > 1: - output_parts = [ToolOutputCollectionPart(self, k, v) for k, v in self.outputs.iteritems()] - else: - # either must have specified structured_like or something worse - if self.structure.structured_like: - collection_prototype = inputs[ self.structure.structured_like ].collection - else: - collection_prototype = type_registry.prototype( self.structure.collection_type ) - - def prototype_dataset_element_to_output( element, parent_ids=[] ): - name = element.element_identifier - format = self.default_format - if self.inherit_format: - format = element.dataset_instance.ext - output = ToolOutput( - name, - format=format, - format_source=self.format_source, - metadata_source=self.metadata_source, - implicit=True, - ) - if self.inherit_metadata: - output.metadata_source = element.dataset_instance - return ToolOutputCollectionPart( - self, - element.element_identifier, - output, - parent_ids=parent_ids, - ) - - def prototype_collection_to_output( collection_prototype, parent_ids=[] ): - output_parts = [] - for element in collection_prototype.elements: - element_parts = [] - if not element.is_collection: - element_parts.append(prototype_dataset_element_to_output( element, parent_ids )) - else: - new_parent_ids = parent_ids[:] + [element.element_identifier] - element_parts.extend(prototype_collection_to_output(element.element_object, new_parent_ids)) - output_parts.extend(element_parts) - - return output_parts - - output_parts = prototype_collection_to_output( collection_prototype ) - - return output_parts - - @property - def dynamic_structure(self): - return self.structure.dynamic - - @property - def dataset_collector_descriptions(self): - if not self.dynamic_structure: - raise Exception("dataset_collector_descriptions called for output collection with static structure") - return self.structure.dataset_collector_descriptions - - -class ToolOutputCollectionStructure( object ): - - def __init__( - self, - collection_type, - collection_type_source, - structured_like, - dataset_collector_descriptions, - ): - self.collection_type = collection_type - self.collection_type_source = collection_type_source - self.structured_like = structured_like - self.dataset_collector_descriptions = dataset_collector_descriptions - if collection_type and collection_type_source: - raise ValueError("Cannot set both type and type_source on collection output.") - if collection_type is None and structured_like is None and dataset_collector_descriptions is None and collection_type_source is None: - raise ValueError( "Output collection types must be specify type of structured_like" ) - if dataset_collector_descriptions and structured_like: - raise ValueError( "Cannot specify dynamic structure (discovered_datasets) and structured_like attribute." ) - self.dynamic = dataset_collector_descriptions is not None - - -class ToolOutputCollectionPart( object ): - - def __init__( self, output_collection_def, element_identifier, output_def, parent_ids=[] ): - self.output_collection_def = output_collection_def - self.element_identifier = element_identifier - self.output_def = output_def - self.parent_ids = parent_ids - - @property - def effective_output_name( self ): - name = self.output_collection_def.name - part_name = self.element_identifier - effective_output_name = "%s|__part__|%s" % ( name, part_name ) - return effective_output_name - - @staticmethod - def is_named_collection_part_name( name ): - return "|__part__|" in name - - @staticmethod - def split_output_name( name ): - assert ToolOutputCollectionPart.is_named_collection_part_name( name ) - return name.split("|__part__|") - - class Tool( object, Dictifiable ): """ Represents a computational tool that can be executed through Galaxy. diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index 281e11fc7cd..bc50efb7254 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -637,7 +637,7 @@ def filter_output(output, incoming): def determine_output_format(output, parameter_context, input_datasets, input_dataset_collections, random_input_ext): """ Determines the output format for a dataset based on an abstract - description of the output (galaxy.tools.ToolOutput), the parameter + description of the output (galaxy.tools.parser.ToolOutput), the parameter wrappers, a map of the input datasets (name => HDA), and the last input extensions in the tool form. diff --git a/lib/galaxy/tools/execute.py b/lib/galaxy/tools/execute.py index a0feae0ee99..e1763f34615 100644 --- a/lib/galaxy/tools/execute.py +++ b/lib/galaxy/tools/execute.py @@ -4,7 +4,7 @@ from various states, tracking results, and building implicit dataset collections from matched collections. """ import collections -import galaxy.tools +from galaxy.tools.parser import ToolOutputCollectionPart from galaxy.util import ExecutionTimer from galaxy.tools.actions import on_text_for_names, ToolExecutionCache from threading import Thread @@ -104,7 +104,7 @@ class ToolExecutionTracker( object ): self.successful_jobs.append( job ) self.output_datasets.extend( outputs ) for output_name, output_dataset in outputs: - if galaxy.tools.ToolOutputCollectionPart.is_named_collection_part_name( output_name ): + if ToolOutputCollectionPart.is_named_collection_part_name( output_name ): # Skip known collection outputs, these will be covered by # output collections. continue diff --git a/lib/galaxy/tools/parser/__init__.py b/lib/galaxy/tools/parser/__init__.py index f2c6431be23..8cbd373d3ff 100644 --- a/lib/galaxy/tools/parser/__init__.py +++ b/lib/galaxy/tools/parser/__init__.py @@ -3,5 +3,8 @@ from .interface import ToolSource from .factory import get_tool_source from .factory import get_input_source +from .output_objects import ( + ToolOutputCollectionPart, +) -__all__ = ["ToolSource", "get_tool_source", "get_input_source"] +__all__ = ["ToolSource", "get_tool_source", "get_input_source", "ToolOutputCollectionPart"] diff --git a/lib/galaxy/tools/parser/output_actions.py b/lib/galaxy/tools/parser/output_actions.py index a0f8da261dc..c92b07faca3 100644 --- a/lib/galaxy/tools/parser/output_actions.py +++ b/lib/galaxy/tools/parser/output_actions.py @@ -205,7 +205,7 @@ class FromFileToolOutputActionOption( ToolOutputActionOption ): try: if options: return str( options[ self.offset ][ self.column ] ) - except Exception, e: + except Exception as e: log.debug( "Error in FromFileToolOutputActionOption get_value: %s" % e ) return None @@ -253,7 +253,7 @@ class FromParamToolOutputActionOption( ToolOutputActionOption ): try: if options: return str( options[ self.offset ][ self.column ] ) - except Exception, e: + except Exception as e: log.debug( "Error in FromParamToolOutputActionOption get_value: %s" % e ) return None @@ -288,7 +288,7 @@ class FromDataTableOutputActionOption( ToolOutputActionOption ): try: if options: return str( options[ self.offset ][ self.column ] ) - except Exception, e: + except Exception as e: log.debug( "Error in FromDataTableOutputActionOption get_value: %s" % e ) return None @@ -383,7 +383,7 @@ class ParamValueToolOutputActionOptionFilter( ToolOutputActionOptionFilter ): try: if self.keep == ( self.compare( self.cast( fields[self.column] ), value ) ): rval.append( fields ) - except Exception, e: + except Exception as e: log.debug(e) continue # likely a bad cast or column out of range return rval diff --git a/lib/galaxy/tools/parser/output_objects.py b/lib/galaxy/tools/parser/output_objects.py new file mode 100644 index 00000000000..5ded72bbcff --- /dev/null +++ b/lib/galaxy/tools/parser/output_objects.py @@ -0,0 +1,218 @@ +from galaxy.util.dictifiable import Dictifiable +from galaxy.util.odict import odict + + +class ToolOutputBase( object, Dictifiable ): + + def __init__( self, name, label=None, filters=None, hidden=False ): + super( ToolOutputBase, self ).__init__() + self.name = name + self.label = label + self.filters = filters or [] + self.hidden = hidden + self.collection = False + + +class ToolOutput( ToolOutputBase ): + """ + Represents an output datasets produced by a tool. For backward + compatibility this behaves as if it were the tuple:: + + (format, metadata_source, parent) + """ + + dict_collection_visible_keys = ( 'name', 'format', 'label', 'hidden' ) + + def __init__( self, name, format=None, format_source=None, metadata_source=None, + parent=None, label=None, filters=None, actions=None, hidden=False, + implicit=False ): + super( ToolOutput, self ).__init__( name, label=label, filters=filters, hidden=hidden ) + self.format = format + self.format_source = format_source + self.metadata_source = metadata_source + self.parent = parent + self.actions = actions + + # Initialize default values + self.change_format = [] + self.implicit = implicit + self.from_work_dir = None + + # Tuple emulation + + def __len__( self ): + return 3 + + def __getitem__( self, index ): + if index == 0: + return self.format + elif index == 1: + return self.metadata_source + elif index == 2: + return self.parent + else: + raise IndexError( index ) + + def __iter__( self ): + return iter( ( self.format, self.metadata_source, self.parent ) ) + + def to_dict( self, view='collection', value_mapper=None, app=None ): + as_dict = super( ToolOutput, self ).to_dict( view=view, value_mapper=value_mapper ) + format = self.format + if format and format != "input" and app: + edam_format = app.datatypes_registry.edam_formats.get(self.format) + as_dict["edam_format"] = edam_format + return as_dict + + +class ToolOutputCollection( ToolOutputBase ): + """ + Represents a HistoryDatasetCollectionAssociation of output datasets produced + by a tool. + + + + + + + + + + + """ + + def __init__( + self, + name, + structure, + label=None, + filters=None, + hidden=False, + default_format="data", + default_format_source=None, + default_metadata_source=None, + inherit_format=False, + inherit_metadata=False + ): + super( ToolOutputCollection, self ).__init__( name, label=label, filters=filters, hidden=hidden ) + self.collection = True + self.default_format = default_format + self.structure = structure + self.outputs = odict() + + self.inherit_format = inherit_format + self.inherit_metadata = inherit_metadata + + self.metadata_source = default_metadata_source + self.format_source = default_format_source + self.change_format = [] # TODO + + def known_outputs( self, inputs, type_registry ): + if self.dynamic_structure: + return [] + + # This line is probably not right - should verify structured_like + # or have outputs and all outputs have name. + if len( self.outputs ) > 1: + output_parts = [ToolOutputCollectionPart(self, k, v) for k, v in self.outputs.iteritems()] + else: + # either must have specified structured_like or something worse + if self.structure.structured_like: + collection_prototype = inputs[ self.structure.structured_like ].collection + else: + collection_prototype = type_registry.prototype( self.structure.collection_type ) + + def prototype_dataset_element_to_output( element, parent_ids=[] ): + name = element.element_identifier + format = self.default_format + if self.inherit_format: + format = element.dataset_instance.ext + output = ToolOutput( + name, + format=format, + format_source=self.format_source, + metadata_source=self.metadata_source, + implicit=True, + ) + if self.inherit_metadata: + output.metadata_source = element.dataset_instance + return ToolOutputCollectionPart( + self, + element.element_identifier, + output, + parent_ids=parent_ids, + ) + + def prototype_collection_to_output( collection_prototype, parent_ids=[] ): + output_parts = [] + for element in collection_prototype.elements: + element_parts = [] + if not element.is_collection: + element_parts.append(prototype_dataset_element_to_output( element, parent_ids )) + else: + new_parent_ids = parent_ids[:] + [element.element_identifier] + element_parts.extend(prototype_collection_to_output(element.element_object, new_parent_ids)) + output_parts.extend(element_parts) + + return output_parts + + output_parts = prototype_collection_to_output( collection_prototype ) + + return output_parts + + @property + def dynamic_structure(self): + return self.structure.dynamic + + @property + def dataset_collector_descriptions(self): + if not self.dynamic_structure: + raise Exception("dataset_collector_descriptions called for output collection with static structure") + return self.structure.dataset_collector_descriptions + + +class ToolOutputCollectionStructure( object ): + + def __init__( + self, + collection_type, + collection_type_source, + structured_like, + dataset_collector_descriptions, + ): + self.collection_type = collection_type + self.collection_type_source = collection_type_source + self.structured_like = structured_like + self.dataset_collector_descriptions = dataset_collector_descriptions + if collection_type and collection_type_source: + raise ValueError("Cannot set both type and type_source on collection output.") + if collection_type is None and structured_like is None and dataset_collector_descriptions is None and collection_type_source is None: + raise ValueError( "Output collection types must be specify type of structured_like" ) + if dataset_collector_descriptions and structured_like: + raise ValueError( "Cannot specify dynamic structure (discovered_datasets) and structured_like attribute." ) + self.dynamic = dataset_collector_descriptions is not None + + +class ToolOutputCollectionPart( object ): + + def __init__( self, output_collection_def, element_identifier, output_def, parent_ids=[] ): + self.output_collection_def = output_collection_def + self.element_identifier = element_identifier + self.output_def = output_def + self.parent_ids = parent_ids + + @property + def effective_output_name( self ): + name = self.output_collection_def.name + part_name = self.element_identifier + effective_output_name = "%s|__part__|%s" % ( name, part_name ) + return effective_output_name + + @staticmethod + def is_named_collection_part_name( name ): + return "|__part__|" in name + + @staticmethod + def split_output_name( name ): + assert ToolOutputCollectionPart.is_named_collection_part_name( name ) + return name.split("|__part__|") diff --git a/lib/galaxy/tools/parser/xml.py b/lib/galaxy/tools/parser/xml.py index 743f8953f95..e0e1f973a62 100644 --- a/lib/galaxy/tools/parser/xml.py +++ b/lib/galaxy/tools/parser/xml.py @@ -24,7 +24,12 @@ from .output_actions import ToolOutputActionGroup from galaxy.util import string_as_bool, xml_text, xml_to_string from galaxy.util.odict import odict from galaxy.tools.deps import requirements -import galaxy.tools +from .output_objects import ( + ToolOutput, + ToolOutputCollection, + ToolOutputCollectionStructure +) + log = logging.getLogger( __name__ ) @@ -191,13 +196,13 @@ class XmlToolSource(ToolSource): dataset_collector_descriptions = None if collection_elem.find( "discover_datasets" ) is not None: dataset_collector_descriptions = dataset_collector_descriptions_from_elem( collection_elem ) - structure = galaxy.tools.ToolOutputCollectionStructure( + structure = ToolOutputCollectionStructure( collection_type=collection_type, collection_type_source=collection_type_source, structured_like=structured_like, dataset_collector_descriptions=dataset_collector_descriptions, ) - output_collection = galaxy.tools.ToolOutputCollection( + output_collection = ToolOutputCollection( name, structure, label=label, @@ -238,7 +243,7 @@ class XmlToolSource(ToolSource): default_format_source=None, default_metadata_source="", ): - output = galaxy.tools.ToolOutput( data_elem.get("name") ) + output = ToolOutput( data_elem.get("name") ) output_format = data_elem.get("format", default_format) auto_format = string_as_bool( data_elem.get( "auto_format", "false" ) ) if auto_format and output_format != "data": diff --git a/lib/galaxy/tools/parser/yaml.py b/lib/galaxy/tools/parser/yaml.py index 406b2b55ff3..89b7dd0a8c5 100644 --- a/lib/galaxy/tools/parser/yaml.py +++ b/lib/galaxy/tools/parser/yaml.py @@ -8,7 +8,9 @@ from galaxy.tools.deps import requirements from .output_collection_def import dataset_collector_descriptions_from_list from .output_actions import ToolOutputActionGroup from galaxy.util.odict import odict -import galaxy.tools +from .output_objects import ( + ToolOutput +) class YamlToolSource(ToolSource): @@ -79,7 +81,7 @@ class YamlToolSource(ToolSource): def _parse_output(self, tool, name, output_dict): # TODO: handle filters, actions, change_format - output = galaxy.tools.ToolOutput( name ) + output = ToolOutput( name ) output.format = output_dict.get("format", "data") output.change_format = [] output.format_source = output_dict.get("format_source", None) diff --git a/lib/galaxy/workflow/extract.py b/lib/galaxy/workflow/extract.py index d2ff8f163fe..ec672db9c9b 100644 --- a/lib/galaxy/workflow/extract.py +++ b/lib/galaxy/workflow/extract.py @@ -11,7 +11,7 @@ from galaxy.tools.parameters.basic import ( SelectToolParameter, UnvalidatedValue ) -from galaxy.tools import ToolOutputCollectionPart +from galaxy.tools.parser import ToolOutputCollectionPart from galaxy.tools.parameters.grouping import ( Conditional, Repeat diff --git a/test/unit/tools/test_actions.py b/test/unit/tools/test_actions.py index 0688a615a00..e09c11bc84b 100644 --- a/test/unit/tools/test_actions.py +++ b/test/unit/tools/test_actions.py @@ -2,7 +2,7 @@ import string import unittest from galaxy import model -from galaxy.tools import ToolOutput +from galaxy.tools.parser.output_objects import ToolOutput from galaxy.tools.actions import DefaultToolAction from galaxy.tools.actions import on_text_for_names from galaxy.tools.actions import determine_output_format diff --git a/test/unit/tools/test_evaluation.py b/test/unit/tools/test_evaluation.py index 428e2819923..127a6073c24 100644 --- a/test/unit/tools/test_evaluation.py +++ b/test/unit/tools/test_evaluation.py @@ -15,7 +15,7 @@ from galaxy.util.bunch import Bunch # For MockTool from galaxy.tools.parameters import params_from_strings -from galaxy.tools import ToolOutput +from galaxy.tools.parser.output_objects import ToolOutput from galaxy.tools.parameters.grouping import Repeat from galaxy.tools.parameters.grouping import Conditional from galaxy.tools.parameters.grouping import ConditionalWhen