From 4de71a0c823f1a40c769dc25459a5bba494d3581 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 17 Apr 2015 17:11:14 -0400 Subject: [PATCH] Allow setting ``auto_format="True"`` on tool ``output``s. The metadata system will then "sniff" the outputs and set the type appropriately. Tool used to demonstrate and test this functionality is included, run the test with: ./run_tests.sh -framework -id output_auto_format If auto_format="true" than explicitly setting ``format="another_type"`` will prevent the tool from loading so that we can allow this to specify a default type later without breaking backward compatiblity. Update 1: - Switched from ``format="auto"`` to ``auto_format="true"`` based on Dan's comments (https://github.com/galaxyproject/galaxy/pull/130). Update 2: - Updated this to work when setting metadata internally. - Switch underlying representation of type from ``_auto_`` to ``_sniff_`` based on seeing a new block in ``galaxy.jobs``. ``auto`` indicates the tool should specify the datatype at runtime. This is in some ways the inverse of that (the tool telling Galaxy to detect the type instead of Galaxy telling the tool to) so I decided it would be best to not have ``auto`` and ``_auto_`` doing completely different things. --- lib/galaxy/datatypes/metadata.py | 5 ++++ lib/galaxy/jobs/__init__.py | 6 +++++ lib/galaxy/tools/parser/xml.py | 8 ++++++- lib/galaxy_ext/metadata/set_metadata.py | 24 ++++++++++++++++---- test/functional/tools/output_auto_format.xml | 16 +++++++++++++ test/functional/tools/samples_tool_conf.xml | 1 + 6 files changed, 55 insertions(+), 5 deletions(-) create mode 100644 test/functional/tools/output_auto_format.xml diff --git a/lib/galaxy/datatypes/metadata.py b/lib/galaxy/datatypes/metadata.py index a90a50a5bca..754643a526c 100644 --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -166,6 +166,9 @@ class MetadataCollection( object ): # if the metadata value is not found in our externally set metadata but it has a value in the 'old' # metadata associated with our dataset, we'll delete it from our dataset's metadata dict del dataset._metadata[ name ] + if '__extension__' in JSONified_dict: + dataset.extension = JSONified_dict['__extension__'] + def to_JSON_dict( self, filename=None ): # galaxy.model.customtypes.json_encoder.encode() @@ -174,6 +177,8 @@ class MetadataCollection( object ): for name, spec in self.spec.items(): if name in dataset_meta_dict: meta_dict[ name ] = spec.param.to_external_value( dataset_meta_dict[ name ] ) + if '__extension__' in dataset_meta_dict: + meta_dict[ '__extension__' ] = dataset_meta_dict['__extension__'] if filename is None: return json.dumps( meta_dict ) json.dump( meta_dict, open( filename, 'wb+' ) ) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 592f75850d4..a016221728a 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -26,6 +26,7 @@ from galaxy.util.bunch import Bunch from galaxy.util.expressions import ExpressionContext from galaxy.util.json import loads from galaxy.util import unicodify +from galaxy.datatypes import sniff from .output_checker import check_output from .datasets import TaskPathRewriter @@ -1153,6 +1154,11 @@ class JobWrapper( object ): # but somewhat trickier (need to recurse up the copied_from tree), for now we'll call set_meta() if ( not self.external_output_metadata.external_metadata_set_successfully( dataset, self.sa_session ) and self.app.config.retry_metadata_internally ): + # If Galaxy was expected to sniff type and didn't - do so. + if dataset.ext == "_sniff_": + extension = sniff.handle_uploaded_dataset_file( dataset.dataset.file_name, self.app.datatypes_registry ) + dataset.extension = extension + # call datatype.set_meta directly for the initial set_meta call during dataset creation dataset.datatype.set_meta( dataset, overwrite=False ) elif ( not self.external_output_metadata.external_metadata_set_successfully( dataset, self.sa_session ) diff --git a/lib/galaxy/tools/parser/xml.py b/lib/galaxy/tools/parser/xml.py index c7e8b3595d7..180b76db90b 100644 --- a/lib/galaxy/tools/parser/xml.py +++ b/lib/galaxy/tools/parser/xml.py @@ -217,7 +217,13 @@ class XmlToolSource(ToolSource): default_metadata_source="", ): output = galaxy.tools.ToolOutput( data_elem.get("name") ) - output.format = data_elem.get("format", default_format) + output_format = data_elem.get("format", default_format) + auto_format = string_as_bool( data_elem.get( "auto_format", "false" ) ) + if auto_format and output_format != "data": + raise ValueError("Setting format and auto_format is not supported at this time.") + elif auto_format: + output_format = "_sniff_" + output.format = output_format output.change_format = data_elem.findall("change_format") output.format_source = data_elem.get("format_source", default_format_source) output.metadata_source = data_elem.get("metadata_source", default_metadata_source) diff --git a/lib/galaxy_ext/metadata/set_metadata.py b/lib/galaxy_ext/metadata/set_metadata.py index 6f7c022ed2d..df85133d735 100644 --- a/lib/galaxy_ext/metadata/set_metadata.py +++ b/lib/galaxy_ext/metadata/set_metadata.py @@ -35,12 +35,28 @@ from galaxy.util import stringify_dictionary_keys from sqlalchemy.orm import clear_mappers -def set_meta_with_tool_provided( dataset_instance, file_dict, set_meta_kwds ): +def set_meta_with_tool_provided( dataset_instance, file_dict, set_meta_kwds, datatypes_registry ): # This method is somewhat odd, in that we set the metadata attributes from tool, # then call set_meta, then set metadata attributes from tool again. # This is intentional due to interplay of overwrite kwd, the fact that some metadata # parameters may rely on the values of others, and that we are accepting the - # values provided by the tool as Truth. + # values provided by the tool as Truth. + extension = dataset_instance.extension + if extension == "_sniff_": + try: + from galaxy.datatypes import sniff + extension = sniff.handle_uploaded_dataset_file( dataset_instance.dataset.external_filename, datatypes_registry ) + # We need to both set the extension so it is available to set_meta + # and record it in the metadata so it can be reloaded on the server + # side and the model updated (see MetadataCollection.{from,to}_JSON_dict) + dataset_instance.extension = extension + # Set special metadata property that will reload this on server side. + setattr( dataset_instance.metadata, "__extension__", extension ) + except Exception: + # TODO: log this when metadata can log stuff... + # https://trello.com/c/Nrwodu9d + pass + for metadata_name, metadata_value in file_dict.get( 'metadata', {} ).iteritems(): setattr( dataset_instance.metadata, metadata_name, metadata_value ) dataset_instance.datatype.set_meta( dataset_instance, **set_meta_kwds ) @@ -102,7 +118,7 @@ def set_metadata(): metadata_file_override = galaxy.datatypes.metadata.MetadataTempFile.from_JSON( metadata_file_override ) setattr( dataset.metadata, metadata_name, metadata_file_override ) file_dict = existing_job_metadata_dict.get( dataset.dataset.id, {} ) - set_meta_with_tool_provided( dataset, file_dict, set_meta_kwds ) + set_meta_with_tool_provided( dataset, file_dict, set_meta_kwds, datatypes_registry ) dataset.metadata.to_JSON_dict( filename_out ) # write out results of set_meta json.dump( ( True, 'Metadata has been set successfully' ), open( filename_results_code, 'wb+' ) ) # setting metadata has succeeded except Exception, e: @@ -115,7 +131,7 @@ def set_metadata(): new_dataset._extra_files_path = os.path.join( tool_job_working_directory, extra_files ) new_dataset.state = new_dataset.states.OK new_dataset_instance = galaxy.model.HistoryDatasetAssociation( id=-i, dataset=new_dataset, extension=file_dict.get( 'ext', 'data' ) ) - set_meta_with_tool_provided( new_dataset_instance, file_dict, set_meta_kwds ) + set_meta_with_tool_provided( new_dataset_instance, file_dict, set_meta_kwds, datatypes_registry ) file_dict[ 'metadata' ] = json.loads( new_dataset_instance.metadata.to_JSON_dict() ) #storing metadata in external form, need to turn back into dict, then later jsonify if existing_job_metadata_dict or new_job_metadata_dict: with open( job_metadata, 'wb' ) as job_metadata_fh: diff --git a/test/functional/tools/output_auto_format.xml b/test/functional/tools/output_auto_format.xml new file mode 100644 index 00000000000..6a2c8f05123 --- /dev/null +++ b/test/functional/tools/output_auto_format.xml @@ -0,0 +1,16 @@ + + cp $input 'out' + + + + + + + + + + + + + + diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml index bd437eb7100..c4c86f5272f 100644 --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -23,6 +23,7 @@ +