From 0dfe73a54cebb1e4fd7a02d9ff05ca585d034244 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 1 Dec 2017 09:52:22 +0000 Subject: [PATCH] Determine dynamically if an extension corresponds to an unsniffable binary datatype Remove the need to call `Binary.register_unsniffable_binary_ext()` for each unsniffable binary datatype. Fix https://github.com/galaxyproject/galaxy/issues/3441 , where the upload of files of a datatype defined in datatypes_conf.xml as subclass of an unsniffable binary datatype ended up with "The uploaded binary file contains inappropriate content" because it was not possible to register the subclassed datatype as unsniffable. Also remove unused `stop_err()` function in upload.py . --- lib/galaxy/datatypes/binary.py | 23 ++--------------------- lib/galaxy/datatypes/msa.py | 3 --- lib/galaxy/datatypes/registry.py | 4 ++++ lib/galaxy/datatypes/sniff.py | 3 +-- tools/data_source/upload.py | 19 +++++-------------- 5 files changed, 12 insertions(+), 40 deletions(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 3d7703f4280..117b459629c 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -33,7 +33,6 @@ log = logging.getLogger(__name__) class Binary(data.Data): """Binary data""" edam_format = "format_2333" - unsniffable_binary_formats = [] @staticmethod def register_sniffable_binary_format(data_type, ext, type_class): @@ -42,11 +41,8 @@ class Binary(data.Data): @staticmethod def register_unsniffable_binary_ext(ext): - Binary.unsniffable_binary_formats.append(ext.lower()) - - @staticmethod - def is_ext_unsniffable(ext): - return ext in Binary.unsniffable_binary_formats + """Deprecated method.""" + pass def set_peek(self, dataset, is_multi_byte=False): """Set the peek and blurb text""" @@ -93,9 +89,6 @@ class Ab1(Binary): return "Binary ab1 sequence file (%s)" % (nice_size(dataset.get_size())) -Binary.register_unsniffable_binary_ext("ab1") - - class Idat(Binary): """Binary data in idat format""" file_ext = "idat" @@ -164,9 +157,6 @@ class CompressedArchive(Binary): return "Compressed binary file (%s)" % (nice_size(dataset.get_size())) -Binary.register_unsniffable_binary_ext("compressed_archive") - - class CompressedZipArchive(CompressedArchive): """ Class describing an compressed binary file @@ -189,9 +179,6 @@ class CompressedZipArchive(CompressedArchive): return "Compressed zip file (%s)" % (nice_size(dataset.get_size())) -Binary.register_unsniffable_binary_ext("zip") - - class GenericAsn1Binary(Binary): """Class for generic ASN.1 binary format""" file_ext = "asn1-binary" @@ -199,9 +186,6 @@ class GenericAsn1Binary(Binary): edam_data = "data_0849" -Binary.register_unsniffable_binary_ext("asn1-binary") - - @dataproviders.decorators.has_dataproviders class Bam(Binary): """Class describing a BAM binary file""" @@ -929,9 +913,6 @@ class Scf(Binary): return "Binary scf sequence file (%s)" % (nice_size(dataset.get_size())) -Binary.register_unsniffable_binary_ext("scf") - - class Sff(Binary): """ Standard Flowgram Format (SFF) """ edam_format = "format_3284" diff --git a/lib/galaxy/datatypes/msa.py b/lib/galaxy/datatypes/msa.py index 3985ef95488..6e36443bdf6 100644 --- a/lib/galaxy/datatypes/msa.py +++ b/lib/galaxy/datatypes/msa.py @@ -92,9 +92,6 @@ class HmmerPress(Binary): self.add_composite_file('model.hmm.h3p', is_binary=True) -Binary.register_unsniffable_binary_ext("hmmpress") - - class Stockholm_1_0(Text): edam_data = "data_0863" edam_format = "format_1961" diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 103be6c566c..b893f869b73 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -465,6 +465,10 @@ class Registry(object): if sniffer_class not in sniffer_elem_classes: self.sniffer_elems.append(elem) + def is_extension_unsniffable_binary(self, ext): + datatype = self.get_datatype_by_extension(ext) + return datatype is not None and isinstance(datatype, binary.Binary) and not hasattr(datatype, 'sniff') + def get_datatype_class_by_name(self, name): """ Return the datatype class where the datatype's `type` attribute diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 45dc012c0ac..48ac2d46e6e 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -16,7 +16,6 @@ import zipfile from six import text_type from galaxy import util -from galaxy.datatypes.binary import Binary from galaxy.util import compression_utils from galaxy.util.checkers import ( check_binary, @@ -478,7 +477,7 @@ def handle_uploaded_dataset_file(filename, datatypes_registry, ext='auto'): ext = guess_ext(filename, sniff_order=datatypes_registry.sniff_order) if check_binary(filename): - if not Binary.is_ext_unsniffable(ext) and not datatypes_registry.get_datatype_by_extension(ext).sniff(filename): + if not datatypes_registry.is_extension_unsniffable_binary(ext) and not datatypes_registry.get_datatype_by_extension(ext).sniff(filename): raise InappropriateDatasetContentError('The binary uploaded file contains inappropriate content.') elif check_html(filename): raise InappropriateDatasetContentError('The uploaded file contains inappropriate HTML content.') diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py index 6937f5ce831..feb95b5b231 100644 --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -36,11 +36,6 @@ else: assert sys.version_info[:2] >= (2, 7) -def stop_err(msg, ret=1): - sys.stderr.write(msg) - sys.exit(ret) - - def file_err(msg, dataset, json_file): json_file.write(dumps(dict(type='dataset', ext='data', @@ -249,22 +244,18 @@ def add_file(dataset, registry, json_file, output_path): dataset.name = uncompressed_name data_type = 'zip' if not data_type: - # TODO refactor this logic. check_binary isn't guaranteed to be - # correct since it only looks at whether the first 100 chars are - # printable or not. If someone specifies a known unsniffable - # binary datatype and check_binary fails, the file gets mangled. - if check_binary(dataset.path) or Binary.is_ext_unsniffable(dataset.file_type): + if check_binary(dataset.path) or registry.is_extension_unsniffable_binary(dataset.file_type): # We have a binary dataset, but it is not Bam, Sff or Pdf data_type = 'binary' - # binary_ok = False parts = dataset.name.split(".") if len(parts) > 1: ext = parts[-1].strip().lower() - if check_content and not Binary.is_ext_unsniffable(ext): + is_ext_unsniffable_binary = registry.is_extension_unsniffable_binary(ext) + if check_content and not is_ext_unsniffable_binary: file_err('The uploaded binary file contains inappropriate content', dataset, json_file) return - elif Binary.is_ext_unsniffable(ext) and dataset.file_type != ext: - err_msg = "You must manually set the 'File Format' to '%s' when uploading %s files." % (ext.capitalize(), ext) + elif is_ext_unsniffable_binary and dataset.file_type != ext: + err_msg = "You must manually set the 'File Format' to '%s' when uploading %s files." % (ext, ext) file_err(err_msg, dataset, json_file) return if not data_type: