Determine dynamically if an extension corresponds to an unsniffable binary datatype

Remove the need to call `Binary.register_unsniffable_binary_ext()` for
each unsniffable binary datatype.

Fix https://github.com/galaxyproject/galaxy/issues/3441 , where the upload
of files of a datatype defined in datatypes_conf.xml as subclass of an
unsniffable binary datatype ended up with "The uploaded binary file
contains inappropriate content" because it was not possible to register the
subclassed datatype as unsniffable.

Also remove unused `stop_err()` function in upload.py .

lib/galaxy/datatypes/binary.py

@@ -33,7 +33,6 @@ log = logging.getLogger(__name__)
 class Binary(data.Data):
     """Binary data"""
     edam_format = "format_2333"
-    unsniffable_binary_formats = []
 
     @staticmethod
     def register_sniffable_binary_format(data_type, ext, type_class):
@@ -42,11 +41,8 @@ class Binary(data.Data):
 
     @staticmethod
     def register_unsniffable_binary_ext(ext):
-        Binary.unsniffable_binary_formats.append(ext.lower())
-
-    @staticmethod
-    def is_ext_unsniffable(ext):
-        return ext in Binary.unsniffable_binary_formats
+        """Deprecated method."""
+        pass
 
     def set_peek(self, dataset, is_multi_byte=False):
         """Set the peek and blurb text"""
@@ -93,9 +89,6 @@ class Ab1(Binary):
             return "Binary ab1 sequence file (%s)" % (nice_size(dataset.get_size()))
 
 
-Binary.register_unsniffable_binary_ext("ab1")
-
-
 class Idat(Binary):
     """Binary data in idat format"""
     file_ext = "idat"
@@ -164,9 +157,6 @@ class CompressedArchive(Binary):
             return "Compressed binary file (%s)" % (nice_size(dataset.get_size()))
 
 
-Binary.register_unsniffable_binary_ext("compressed_archive")
-
-
 class CompressedZipArchive(CompressedArchive):
     """
         Class describing an compressed binary file
@@ -189,9 +179,6 @@ class CompressedZipArchive(CompressedArchive):
             return "Compressed zip file (%s)" % (nice_size(dataset.get_size()))
 
 
-Binary.register_unsniffable_binary_ext("zip")
-
-
 class GenericAsn1Binary(Binary):
     """Class for generic ASN.1 binary format"""
     file_ext = "asn1-binary"
@@ -199,9 +186,6 @@ class GenericAsn1Binary(Binary):
     edam_data = "data_0849"
 
 
-Binary.register_unsniffable_binary_ext("asn1-binary")
-
-
 @dataproviders.decorators.has_dataproviders
 class Bam(Binary):
     """Class describing a BAM binary file"""
@@ -929,9 +913,6 @@ class Scf(Binary):
             return "Binary scf sequence file (%s)" % (nice_size(dataset.get_size()))
 
 
-Binary.register_unsniffable_binary_ext("scf")
-
-
 class Sff(Binary):
     """ Standard Flowgram Format (SFF) """
     edam_format = "format_3284"

lib/galaxy/datatypes/msa.py

@@ -92,9 +92,6 @@ class HmmerPress(Binary):
         self.add_composite_file('model.hmm.h3p', is_binary=True)
 
 
-Binary.register_unsniffable_binary_ext("hmmpress")
-
-
 class Stockholm_1_0(Text):
     edam_data = "data_0863"
     edam_format = "format_1961"

lib/galaxy/datatypes/registry.py

@@ -465,6 +465,10 @@ class Registry(object):
                                         if sniffer_class not in sniffer_elem_classes:
                                             self.sniffer_elems.append(elem)
 
+    def is_extension_unsniffable_binary(self, ext):
+        datatype = self.get_datatype_by_extension(ext)
+        return datatype is not None and isinstance(datatype, binary.Binary) and not hasattr(datatype, 'sniff')
+
     def get_datatype_class_by_name(self, name):
         """
         Return the datatype class where the datatype's `type` attribute

lib/galaxy/datatypes/sniff.py

@@ -16,7 +16,6 @@ import zipfile
 from six import text_type
 
 from galaxy import util
-from galaxy.datatypes.binary import Binary
 from galaxy.util import compression_utils
 from galaxy.util.checkers import (
     check_binary,
@@ -478,7 +477,7 @@ def handle_uploaded_dataset_file(filename, datatypes_registry, ext='auto'):
         ext = guess_ext(filename, sniff_order=datatypes_registry.sniff_order)
 
     if check_binary(filename):
-        if not Binary.is_ext_unsniffable(ext) and not datatypes_registry.get_datatype_by_extension(ext).sniff(filename):
+        if not datatypes_registry.is_extension_unsniffable_binary(ext) and not datatypes_registry.get_datatype_by_extension(ext).sniff(filename):
             raise InappropriateDatasetContentError('The binary uploaded file contains inappropriate content.')
     elif check_html(filename):
         raise InappropriateDatasetContentError('The uploaded file contains inappropriate HTML content.')

tools/data_source/upload.py

@@ -36,11 +36,6 @@ else:
 assert sys.version_info[:2] >= (2, 7)
 
 
-def stop_err(msg, ret=1):
-    sys.stderr.write(msg)
-    sys.exit(ret)
-
-
 def file_err(msg, dataset, json_file):
     json_file.write(dumps(dict(type='dataset',
                                ext='data',
@@ -249,22 +244,18 @@ def add_file(dataset, registry, json_file, output_path):
                             dataset.name = uncompressed_name
                     data_type = 'zip'
             if not data_type:
-                # TODO refactor this logic.  check_binary isn't guaranteed to be
-                # correct since it only looks at whether the first 100 chars are
-                # printable or not.  If someone specifies a known unsniffable
-                # binary datatype and check_binary fails, the file gets mangled.
-                if check_binary(dataset.path) or Binary.is_ext_unsniffable(dataset.file_type):
+                if check_binary(dataset.path) or registry.is_extension_unsniffable_binary(dataset.file_type):
                     # We have a binary dataset, but it is not Bam, Sff or Pdf
                     data_type = 'binary'
-                    # binary_ok = False
                     parts = dataset.name.split(".")
                     if len(parts) > 1:
                         ext = parts[-1].strip().lower()
-                        if check_content and not Binary.is_ext_unsniffable(ext):
+                        is_ext_unsniffable_binary = registry.is_extension_unsniffable_binary(ext)
+                        if check_content and not is_ext_unsniffable_binary:
                             file_err('The uploaded binary file contains inappropriate content', dataset, json_file)
                             return
-                        elif Binary.is_ext_unsniffable(ext) and dataset.file_type != ext:
-                            err_msg = "You must manually set the 'File Format' to '%s' when uploading %s files." % (ext.capitalize(), ext)
+                        elif is_ext_unsniffable_binary and dataset.file_type != ext:
+                            err_msg = "You must manually set the 'File Format' to '%s' when uploading %s files." % (ext, ext)
                             file_err(err_msg, dataset, json_file)
                             return
             if not data_type: