Allow setting `auto_format="True" on tool output`s.

The metadata system will then "sniff" the outputs and set the type appropriately.

Tool used to demonstrate and test this functionality is included, run the test with:

    ./run_tests.sh -framework -id output_auto_format

If auto_format="true" than explicitly setting ``format="another_type"`` will prevent the tool from loading so that we can allow this to specify a default type later without breaking backward compatiblity.

Update 1:
 - Switched from ``format="auto"`` to ``auto_format="true"`` based on Dan's comments (https://github.com/galaxyproject/galaxy/pull/130).

Update 2:
 - Updated this to work when setting metadata internally.
 - Switch underlying representation of type from ``_auto_`` to ``_sniff_`` based on seeing a new block in ``galaxy.jobs``. ``auto`` indicates the tool should specify the datatype at runtime. This is in some ways the inverse of that (the tool telling Galaxy to detect the type instead of Galaxy telling the tool to) so I decided it would be best to not have ``auto`` and ``_auto_`` doing completely different things.

lib/galaxy/datatypes/metadata.py

@@ -166,6 +166,9 @@ class MetadataCollection( object ):
                 # if the metadata value is not found in our externally set metadata but it has a value in the 'old'
                 # metadata associated with our dataset, we'll delete it from our dataset's metadata dict
                 del dataset._metadata[ name ]
+        if '__extension__' in JSONified_dict:
+            dataset.extension = JSONified_dict['__extension__']
+
 
     def to_JSON_dict( self, filename=None ):
         # galaxy.model.customtypes.json_encoder.encode()
@@ -174,6 +177,8 @@ class MetadataCollection( object ):
         for name, spec in self.spec.items():
             if name in dataset_meta_dict:
                 meta_dict[ name ] = spec.param.to_external_value( dataset_meta_dict[ name ] )
+        if '__extension__' in dataset_meta_dict:
+            meta_dict[ '__extension__' ] = dataset_meta_dict['__extension__']
         if filename is None:
             return json.dumps( meta_dict )
         json.dump( meta_dict, open( filename, 'wb+' ) )

lib/galaxy/jobs/__init__.py

@@ -26,6 +26,7 @@ from galaxy.util.bunch import Bunch
 from galaxy.util.expressions import ExpressionContext
 from galaxy.util.json import loads
 from galaxy.util import unicodify
+from galaxy.datatypes import sniff
 
 from .output_checker import check_output
 from .datasets import TaskPathRewriter
@@ -1153,6 +1154,11 @@ class JobWrapper( object ):
                     # but somewhat trickier (need to recurse up the copied_from tree), for now we'll call set_meta()
                     if ( not self.external_output_metadata.external_metadata_set_successfully( dataset, self.sa_session )
                          and self.app.config.retry_metadata_internally ):
+                        # If Galaxy was expected to sniff type and didn't - do so.
+                        if dataset.ext == "_sniff_":
+                            extension = sniff.handle_uploaded_dataset_file( dataset.dataset.file_name, self.app.datatypes_registry )
+                            dataset.extension = extension
+
                         # call datatype.set_meta directly for the initial set_meta call during dataset creation
                         dataset.datatype.set_meta( dataset, overwrite=False )
                     elif ( not self.external_output_metadata.external_metadata_set_successfully( dataset, self.sa_session )

lib/galaxy/tools/parser/xml.py

@@ -217,7 +217,13 @@ class XmlToolSource(ToolSource):
         default_metadata_source="",
     ):
         output = galaxy.tools.ToolOutput( data_elem.get("name") )
-        output.format = data_elem.get("format", default_format)
+        output_format = data_elem.get("format", default_format)
+        auto_format = string_as_bool( data_elem.get( "auto_format", "false" ) )
+        if auto_format and output_format != "data":
+            raise ValueError("Setting format and auto_format is not supported at this time.")
+        elif auto_format:
+            output_format = "_sniff_"
+        output.format = output_format
         output.change_format = data_elem.findall("change_format")
         output.format_source = data_elem.get("format_source", default_format_source)
         output.metadata_source = data_elem.get("metadata_source", default_metadata_source)

lib/galaxy_ext/metadata/set_metadata.py

@@ -35,12 +35,28 @@ from galaxy.util import stringify_dictionary_keys
 from sqlalchemy.orm import clear_mappers
 
 
-def set_meta_with_tool_provided( dataset_instance, file_dict, set_meta_kwds ):
+def set_meta_with_tool_provided( dataset_instance, file_dict, set_meta_kwds, datatypes_registry ):
     # This method is somewhat odd, in that we set the metadata attributes from tool,
     # then call set_meta, then set metadata attributes from tool again.
     # This is intentional due to interplay of overwrite kwd, the fact that some metadata
     # parameters may rely on the values of others, and that we are accepting the
-    # values provided by the tool as Truth. 
+    # values provided by the tool as Truth.
+    extension = dataset_instance.extension
+    if extension == "_sniff_":
+        try:
+            from galaxy.datatypes import sniff
+            extension = sniff.handle_uploaded_dataset_file( dataset_instance.dataset.external_filename, datatypes_registry )
+            # We need to both set the extension so it is available to set_meta
+            # and record it in the metadata so it can be reloaded on the server
+            # side and the model updated (see MetadataCollection.{from,to}_JSON_dict)
+            dataset_instance.extension = extension
+            # Set special metadata property that will reload this on server side.
+            setattr( dataset_instance.metadata, "__extension__", extension )
+        except Exception:
+            # TODO: log this when metadata can log stuff...
+            # https://trello.com/c/Nrwodu9d
+            pass
+
     for metadata_name, metadata_value in file_dict.get( 'metadata', {} ).iteritems():
         setattr( dataset_instance.metadata, metadata_name, metadata_value )
     dataset_instance.datatype.set_meta( dataset_instance, **set_meta_kwds )
@@ -102,7 +118,7 @@ def set_metadata():
                         metadata_file_override = galaxy.datatypes.metadata.MetadataTempFile.from_JSON( metadata_file_override )
                     setattr( dataset.metadata, metadata_name, metadata_file_override )
             file_dict = existing_job_metadata_dict.get( dataset.dataset.id, {} )
-            set_meta_with_tool_provided( dataset, file_dict, set_meta_kwds )
+            set_meta_with_tool_provided( dataset, file_dict, set_meta_kwds, datatypes_registry )
             dataset.metadata.to_JSON_dict( filename_out )  # write out results of set_meta
             json.dump( ( True, 'Metadata has been set successfully' ), open( filename_results_code, 'wb+' ) )  # setting metadata has succeeded
         except Exception, e:
@@ -115,7 +131,7 @@ def set_metadata():
             new_dataset._extra_files_path = os.path.join( tool_job_working_directory, extra_files )
         new_dataset.state = new_dataset.states.OK
         new_dataset_instance = galaxy.model.HistoryDatasetAssociation( id=-i, dataset=new_dataset, extension=file_dict.get( 'ext', 'data' ) )
-        set_meta_with_tool_provided( new_dataset_instance, file_dict, set_meta_kwds )
+        set_meta_with_tool_provided( new_dataset_instance, file_dict, set_meta_kwds, datatypes_registry )
         file_dict[ 'metadata' ] = json.loads( new_dataset_instance.metadata.to_JSON_dict() ) #storing metadata in external form, need to turn back into dict, then later jsonify
     if existing_job_metadata_dict or new_job_metadata_dict:
         with open( job_metadata, 'wb' ) as job_metadata_fh:

test/functional/tools/output_auto_format.xml

@@ -0,0 +1,16 @@
+<tool id="output_auto_format" name="output_auto_format" version="1.0.0">
+  <command>cp $input 'out'</command>
+  <inputs>
+    <param name="input" type="data" format="data" label="An input dataset" help=""/>
+  </inputs>
+  <outputs>
+    <data auto_format="true" name="output" label="Auto Output" from_work_dir="out">
+    </data>
+  </outputs>
+  <tests>
+    <test>
+      <param name="input" value="simple_line.txt" ftype="txt" />
+      <output name="output" file="simple_line.txt" ftype="txt" />
+    </test>
+  </tests>
+</tool>

test/functional/tools/samples_tool_conf.xml

@@ -23,6 +23,7 @@
   <tool file="output_order.xml" />
   <tool file="output_format.xml" />
   <tool file="output_filter.xml" />
+  <tool file="output_auto_format.xml" />
   <tool file="disambiguate_repeats.xml" />
   <tool file="min_repeat.xml" />
   <tool file="parallelism.xml" />