From 5ab4ae4f2f630ea12c8b4e82e90cd8dca2758a6f Mon Sep 17 00:00:00 2001 From: ashok Date: Sun, 26 Jun 2016 12:20:37 +0530 Subject: [PATCH 01/14] Changes for Fastq Compression. --- .../fastqcssangergz_to_fastqcssanger.xml | 13 ++ .../datatypes/converters/fastqgz_to_fastq.xml | 13 ++ .../fastqilluminagz_to_fastqillumina.xml | 13 ++ .../fastqsangergz_to_fastqsanger.xml | 13 ++ .../fastqsolexagz_to_fastqsolexa.xml | 13 ++ lib/galaxy/datatypes/sequence.py | 182 +++++++++++++++++- lib/galaxy/datatypes/sniff.py | 7 +- 7 files changed, 251 insertions(+), 3 deletions(-) create mode 100644 lib/galaxy/datatypes/converters/fastqcssangergz_to_fastqcssanger.xml create mode 100644 lib/galaxy/datatypes/converters/fastqgz_to_fastq.xml create mode 100644 lib/galaxy/datatypes/converters/fastqilluminagz_to_fastqillumina.xml create mode 100644 lib/galaxy/datatypes/converters/fastqsangergz_to_fastqsanger.xml create mode 100644 lib/galaxy/datatypes/converters/fastqsolexagz_to_fastqsolexa.xml diff --git a/lib/galaxy/datatypes/converters/fastqcssangergz_to_fastqcssanger.xml b/lib/galaxy/datatypes/converters/fastqcssangergz_to_fastqcssanger.xml new file mode 100644 index 00000000000..0dffa753a36 --- /dev/null +++ b/lib/galaxy/datatypes/converters/fastqcssangergz_to_fastqcssanger.xml @@ -0,0 +1,13 @@ + diff --git a/lib/galaxy/datatypes/converters/fastqgz_to_fastq.xml b/lib/galaxy/datatypes/converters/fastqgz_to_fastq.xml new file mode 100644 index 00000000000..3d5b8a393a5 --- /dev/null +++ b/lib/galaxy/datatypes/converters/fastqgz_to_fastq.xml @@ -0,0 +1,13 @@ + diff --git a/lib/galaxy/datatypes/converters/fastqilluminagz_to_fastqillumina.xml b/lib/galaxy/datatypes/converters/fastqilluminagz_to_fastqillumina.xml new file mode 100644 index 00000000000..91fa60951c2 --- /dev/null +++ b/lib/galaxy/datatypes/converters/fastqilluminagz_to_fastqillumina.xml @@ -0,0 +1,13 @@ + diff --git a/lib/galaxy/datatypes/converters/fastqsangergz_to_fastqsanger.xml b/lib/galaxy/datatypes/converters/fastqsangergz_to_fastqsanger.xml new file mode 100644 index 00000000000..981b6b50eda --- /dev/null +++ b/lib/galaxy/datatypes/converters/fastqsangergz_to_fastqsanger.xml @@ -0,0 +1,13 @@ + diff --git a/lib/galaxy/datatypes/converters/fastqsolexagz_to_fastqsolexa.xml b/lib/galaxy/datatypes/converters/fastqsolexagz_to_fastqsolexa.xml new file mode 100644 index 00000000000..ed29cc3f9ae --- /dev/null +++ b/lib/galaxy/datatypes/converters/fastqsolexagz_to_fastqsolexa.xml @@ -0,0 +1,13 @@ + diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index d1660145064..0f49fb51c09 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -7,8 +7,10 @@ import json import logging import os import re +import subprocess import string import sys +import tempfile from cgi import escape import bx.align.maf @@ -571,6 +573,9 @@ class Fastq ( Sequence ): data_lines = 0 sequences = 0 seq_counter = 0 # blocks should be 4 lines long + compress = is_gzip(dataset.file_name) + if compress: + self.decompress_fastqgz(dataset) for line in open( dataset.file_name ): line = line.strip() if line and line.startswith( '#' ) and not data_lines: @@ -590,6 +595,156 @@ class Fastq ( Sequence ): dataset.metadata.data_lines = data_lines dataset.metadata.sequences = sequences + def decompress_fastqgz(self,dataset): + print('X Compressing the fastq files') + compress = is_gzip(dataset.file_name) + if compress: + # Rename compressed file + os.rename(dataset.file_name , dataset.file_name + ".gz") + # Decompress fastq file + stderr_name = tempfile.NamedTemporaryFile( prefix="fastq_compress" ).name + command = ['gzip','-df',dataset.file_name+ ".gz"] + try: + exit_code = subprocess.call( args=command, stderr=open( stderr_name, 'wb' ) ) + except Exception as e: + log.warning( '%s, Compression Exception: %s', self, e ) + else: + return False + def sniff( self, filename ): + """ + Determines whether the file is in generic fastq format + For details, see http://maq.sourceforge.net/fastq.shtml + + Note: There are three kinds of FASTQ files, known as "Sanger" (sometimes called "Standard"), Solexa, and Illumina + These differ in the representation of the quality scores + + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname( '1.fastqsanger' ) + >>> Fastq().sniff( fname ) + True + >>> fname = get_test_fname( '2.fastqsanger' ) + >>> Fastq().sniff( fname ) + True + """ + headers = get_headers( filename, None ) + bases_regexp = re.compile( "^[NGTAC]*" ) + # check that first block looks like a fastq block + try: + if len( headers ) >= 4 and headers[0][0] and headers[0][0][0] == "@" and headers[2][0] and headers[2][0][0] == "+" and headers[1][0]: + # Check the sequence line, make sure it contains only G/C/A/T/N + if not bases_regexp.match( headers[1][0] ): + return False + return True + return False + except: + return False + + def split( cls, input_datasets, subdir_generator_function, split_params): + """ + FASTQ files are split on cluster boundaries, in increments of 4 lines + """ + if split_params is None: + return None + + # first, see if there are any associated FQTOC files that will give us the split locations + # if so, we don't need to read the files to do the splitting + toc_file_datasets = [] + for ds in input_datasets: + tmp_ds = ds + fqtoc_file = None + while fqtoc_file is None and tmp_ds is not None: + fqtoc_file = tmp_ds.get_converted_files_by_type('fqtoc') + tmp_ds = tmp_ds.copied_from_library_dataset_dataset_association + + if fqtoc_file is not None: + toc_file_datasets.append(fqtoc_file) + + if len(toc_file_datasets) == len(input_datasets): + return cls.do_fast_split(input_datasets, toc_file_datasets, subdir_generator_function, split_params) + return cls.do_slow_split(input_datasets, subdir_generator_function, split_params) + split = classmethod(split) + + def process_split_file(data): + """ + This is called in the context of an external process launched by a Task (possibly not on the Galaxy machine) + to create the input files for the Task. The parameters: + data - a dict containing the contents of the split file + """ + args = data['args'] + input_name = data['input_name'] + output_name = data['output_name'] + start_sequence = long(args['start_sequence']) + sequence_count = long(args['num_sequences']) + + if 'toc_file' in args: + toc_file = json.load(open(args['toc_file'], 'r')) + commands = Sequence.get_split_commands_with_toc(input_name, output_name, toc_file, start_sequence, sequence_count) + else: + commands = Sequence.get_split_commands_sequential(is_gzip(input_name), input_name, output_name, start_sequence, sequence_count) + for cmd in commands: + if 0 != os.system(cmd): + raise Exception("Executing '%s' failed" % cmd) + return True + process_split_file = staticmethod(process_split_file) + + +class FastqGz ( Sequence ): + """Class representing a generic compressed FASTQ sequence""" + edam_format = "format_1930" + file_ext = "fastq.gz" + + def set_meta( self, dataset, **kwd ): + """ + Set the number of sequences and the number of data lines + in dataset. + FIXME: This does not properly handle line wrapping + """ + if self.max_optional_metadata_filesize >= 0 and dataset.get_size() > self.max_optional_metadata_filesize: + dataset.metadata.data_lines = None + dataset.metadata.sequences = None + return + data_lines = 0 + sequences = 0 + seq_counter = 0 # blocks should be 4 lines long + compress = is_gzip(dataset.file_name) + if not compress: + self.compress_fastq(dataset) + for line in gzip.GzipFile(dataset.file_name, 'r'): + line = line.strip() + if line and line.startswith( '#' ) and not data_lines: + # We don't count comment lines for sequence data types + continue + seq_counter += 1 + data_lines += 1 + if line and line.startswith( '@' ): + if seq_counter >= 4: + # count previous block + # blocks should be 4 lines long + sequences += 1 + seq_counter = 1 + if seq_counter >= 4: + # count final block + sequences += 1 + dataset.metadata.data_lines = data_lines + dataset.metadata.sequences = sequences + + def compress_fastq(self,dataset): + print('X Compressing the fastq files') + compress = is_gzip(dataset.file_name) + if not compress: + # Compress fastq file + stderr_name = tempfile.NamedTemporaryFile( prefix="fastq_compress" ).name + command = ['gzip',dataset.file_name] + try: + exit_code = subprocess.call( args=command, stderr=open( stderr_name, 'wb' ) ) + except Exception as e: + log.warning( '%s, Compression Exception: %s', self, e ) + + # Rename compressed file + os.rename(dataset.file_name + ".gz" , dataset.file_name) + else: + return False + def sniff( self, filename ): """ Determines whether the file is in generic fastq format @@ -674,13 +829,13 @@ class FastqSanger( Fastq ): file_ext = "fastqsanger" -class FastqSolexa( Fastq ): +class FastqSolexaGz( Fastq ): """Class representing a FASTQ sequence ( the Solexa variant )""" edam_format = "format_1933" file_ext = "fastqsolexa" -class FastqIllumina( Fastq ): +class FastqIlluminaGz( Fastq ): """Class representing a FASTQ sequence ( the Illumina 1.3+ variant )""" edam_format = "format_1931" file_ext = "fastqillumina" @@ -691,6 +846,29 @@ class FastqCSSanger( Fastq ): file_ext = "fastqcssanger" +class FastqSangerGz( FastqGz ): + """Class representing a compressed FASTQ sequence ( the Sanger variant )""" + edam_format = "format_1932" + file_ext = "fastqsanger.gz" + + +class FastqSolexa( FastqGz ): + """Class representing a compressed FASTQ sequence ( the Solexa variant )""" + edam_format = "format_1933" + file_ext = "fastqsolexa.gz" + + +class FastqIllumina( FastqGz ): + """Class representing a compressed FASTQ sequence ( the Illumina 1.3+ variant )""" + edam_format = "format_1931" + file_ext = "fastqillumina.gz" + + +class FastqCSSangerGz( FastqGz ): + """Class representing a Color Space compressed FASTQ sequence ( e.g a SOLiD variant )""" + file_ext = "fastqcssanger.gz" + + class Maf( Alignment ): """Class describing a Maf alignment""" edam_format = "format_3008" diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index bacae3210df..ffb71a221ed 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -198,7 +198,12 @@ def get_headers( fname, sep, count=60, is_multi_byte=False ): [['chr7', '127475281', '127491632', 'NM_000230', '0', '+', '127486022', '127488767', '0', '3', '29,172,3225,', '0,10713,13126,'], ['chr7', '127486011', '127488900', 'D49487', '0', '+', '127486022', '127488767', '0', '2', '155,490,', '0,2399']] """ headers = [] - for idx, line in enumerate(open(fname)): + compress = is_gzip(fname) + if compress: + in_file = gzip.GzipFile(fname, 'r') + else: + in_file = open(fname, 'rt') + for idx, line in enumerate(in_file): line = line.rstrip('\n\r') if is_multi_byte: # TODO: fix this - sep is never found in line From 7597da61e5de1796f0cf4ce2f6ea577cb390f311 Mon Sep 17 00:00:00 2001 From: ashok Date: Sun, 26 Jun 2016 21:41:36 +0530 Subject: [PATCH 02/14] Updated sequence.py and datatypes_conf.xml --- config/datatypes_conf.xml.sample | 15 ++++++ lib/galaxy/datatypes/sequence.py | 91 ++------------------------------ 2 files changed, 20 insertions(+), 86 deletions(-) diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index 2676db83e94..cbd72ae28f0 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -72,18 +72,33 @@ + + + + + + + + + + + + + + + diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index 0f49fb51c09..e36465f8867 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -595,21 +595,6 @@ class Fastq ( Sequence ): dataset.metadata.data_lines = data_lines dataset.metadata.sequences = sequences - def decompress_fastqgz(self,dataset): - print('X Compressing the fastq files') - compress = is_gzip(dataset.file_name) - if compress: - # Rename compressed file - os.rename(dataset.file_name , dataset.file_name + ".gz") - # Decompress fastq file - stderr_name = tempfile.NamedTemporaryFile( prefix="fastq_compress" ).name - command = ['gzip','-df',dataset.file_name+ ".gz"] - try: - exit_code = subprocess.call( args=command, stderr=open( stderr_name, 'wb' ) ) - except Exception as e: - log.warning( '%s, Compression Exception: %s', self, e ) - else: - return False def sniff( self, filename ): """ Determines whether the file is in generic fastq format @@ -687,8 +672,7 @@ class Fastq ( Sequence ): return True process_split_file = staticmethod(process_split_file) - -class FastqGz ( Sequence ): +class FastqGz ( Fastq ): """Class representing a generic compressed FASTQ sequence""" edam_format = "format_1930" file_ext = "fastq.gz" @@ -728,23 +712,6 @@ class FastqGz ( Sequence ): dataset.metadata.data_lines = data_lines dataset.metadata.sequences = sequences - def compress_fastq(self,dataset): - print('X Compressing the fastq files') - compress = is_gzip(dataset.file_name) - if not compress: - # Compress fastq file - stderr_name = tempfile.NamedTemporaryFile( prefix="fastq_compress" ).name - command = ['gzip',dataset.file_name] - try: - exit_code = subprocess.call( args=command, stderr=open( stderr_name, 'wb' ) ) - except Exception as e: - log.warning( '%s, Compression Exception: %s', self, e ) - - # Rename compressed file - os.rename(dataset.file_name + ".gz" , dataset.file_name) - else: - return False - def sniff( self, filename ): """ Determines whether the file is in generic fastq format @@ -774,54 +741,6 @@ class FastqGz ( Sequence ): except: return False - def split( cls, input_datasets, subdir_generator_function, split_params): - """ - FASTQ files are split on cluster boundaries, in increments of 4 lines - """ - if split_params is None: - return None - - # first, see if there are any associated FQTOC files that will give us the split locations - # if so, we don't need to read the files to do the splitting - toc_file_datasets = [] - for ds in input_datasets: - tmp_ds = ds - fqtoc_file = None - while fqtoc_file is None and tmp_ds is not None: - fqtoc_file = tmp_ds.get_converted_files_by_type('fqtoc') - tmp_ds = tmp_ds.copied_from_library_dataset_dataset_association - - if fqtoc_file is not None: - toc_file_datasets.append(fqtoc_file) - - if len(toc_file_datasets) == len(input_datasets): - return cls.do_fast_split(input_datasets, toc_file_datasets, subdir_generator_function, split_params) - return cls.do_slow_split(input_datasets, subdir_generator_function, split_params) - split = classmethod(split) - - def process_split_file(data): - """ - This is called in the context of an external process launched by a Task (possibly not on the Galaxy machine) - to create the input files for the Task. The parameters: - data - a dict containing the contents of the split file - """ - args = data['args'] - input_name = data['input_name'] - output_name = data['output_name'] - start_sequence = long(args['start_sequence']) - sequence_count = long(args['num_sequences']) - - if 'toc_file' in args: - toc_file = json.load(open(args['toc_file'], 'r')) - commands = Sequence.get_split_commands_with_toc(input_name, output_name, toc_file, start_sequence, sequence_count) - else: - commands = Sequence.get_split_commands_sequential(is_gzip(input_name), input_name, output_name, start_sequence, sequence_count) - for cmd in commands: - if 0 != os.system(cmd): - raise Exception("Executing '%s' failed" % cmd) - return True - process_split_file = staticmethod(process_split_file) - class FastqSanger( Fastq ): """Class representing a FASTQ sequence ( the Sanger variant )""" @@ -829,13 +748,13 @@ class FastqSanger( Fastq ): file_ext = "fastqsanger" -class FastqSolexaGz( Fastq ): +class FastqSolexa( Fastq ): """Class representing a FASTQ sequence ( the Solexa variant )""" edam_format = "format_1933" file_ext = "fastqsolexa" -class FastqIlluminaGz( Fastq ): +class FastqIllumina( Fastq ): """Class representing a FASTQ sequence ( the Illumina 1.3+ variant )""" edam_format = "format_1931" file_ext = "fastqillumina" @@ -852,13 +771,13 @@ class FastqSangerGz( FastqGz ): file_ext = "fastqsanger.gz" -class FastqSolexa( FastqGz ): +class FastqSolexaGz( FastqGz ): """Class representing a compressed FASTQ sequence ( the Solexa variant )""" edam_format = "format_1933" file_ext = "fastqsolexa.gz" -class FastqIllumina( FastqGz ): +class FastqIlluminaGz( FastqGz ): """Class representing a compressed FASTQ sequence ( the Illumina 1.3+ variant )""" edam_format = "format_1931" file_ext = "fastqillumina.gz" From aabfa4283d396ba01ea42b47ab0b8601122096bd Mon Sep 17 00:00:00 2001 From: ashok Date: Mon, 27 Jun 2016 07:27:07 +0530 Subject: [PATCH 03/14] Changes to keep the compressed fastq file as it is with out decompressing. --- config/datatypes_conf.xml.sample | 1 + lib/galaxy/datatypes/sequence.py | 44 +++++++++++++++++++++++--------- tools/data_source/upload.py | 6 +++++ 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index cbd72ae28f0..93a71255709 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -636,6 +636,7 @@ + diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index e36465f8867..85e324fc916 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -712,7 +712,34 @@ class FastqGz ( Fastq ): dataset.metadata.data_lines = data_lines dataset.metadata.sequences = sequences + def compress_fastq(self,dataset): + print('X Compressing the fastq files') + if dataset.file_name and dataset.file_name.strip(): + compress = is_gzip(dataset.file_name) + if not compress: + # TODO: consider how to created temp file in Galaxy database/tmp folder + compressed_file = tempfile.NamedTemporaryFile(delete=False) + shutil.copyfileobj(open(dataset.file_name), gzip.open(compressed_file.name, 'wb')) + compressed_file.close() + os.unlink(dataset.file_name) + os.rename(compressed_file.name, dataset.file_name) + # # Compress fastq file + # stderr_name = tempfile.NamedTemporaryFile( prefix="fastq_compress" ).name + # command = ['gzip',dataset.file_name] + # try: + # exit_code = subprocess.call( args=command, stderr=open( stderr_name, 'wb' ) ) + # except Exception as e: + # log.warning( '%s, Compression Exception: %s', self, e ) + # + # # Rename compressed file + # os.rename(dataset.file_name + ".gz" , dataset.file_name) + else: + return False + else: + return False + def sniff( self, filename ): + """ Determines whether the file is in generic fastq format For details, see http://maq.sourceforge.net/fastq.shtml @@ -728,19 +755,12 @@ class FastqGz ( Fastq ): >>> Fastq().sniff( fname ) True """ - headers = get_headers( filename, None ) - bases_regexp = re.compile( "^[NGTAC]*" ) - # check that first block looks like a fastq block - try: - if len( headers ) >= 4 and headers[0][0] and headers[0][0][0] == "@" and headers[2][0] and headers[2][0][0] == "+" and headers[1][0]: - # Check the sequence line, make sure it contains only G/C/A/T/N - if not bases_regexp.match( headers[1][0] ): - return False - return True + is_compressed = is_gzip(filename) + is_fastq = super(FastqGz,self).sniff(filename) + if is_fastq and is_compressed: + return True + else: return False - except: - return False - class FastqSanger( Fastq ): """Class representing a FASTQ sequence ( the Sanger variant )""" diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py index 85355c3eb3b..11bb22a26c8 100644 --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -125,6 +125,12 @@ def add_file( dataset, registry, json_file, output_path ): if type_info: data_type = type_info[0] ext = type_info[1] + # Is dataset is compressed Fastq? + is_gzipped, is_valid = check_gzip( dataset.path ) + if is_gzipped and is_valid: + ext = sniff.guess_ext( dataset.path, registry.sniff_order) + if ext: + data_type = ext if not data_type: root_datatype = registry.get_datatype_by_extension( dataset.file_type ) if getattr( root_datatype, 'compressed', False ): From 7d4840ce6b46a2b51f0d5332d72e7b4d956ac808 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 10 Nov 2016 09:36:31 +0100 Subject: [PATCH 04/14] For fastq.gz support inherit from Fastq and Binary and enhance FastQ format to be gzip aware. --- config/datatypes_conf.xml.sample | 28 ++++---- lib/galaxy/datatypes/sequence.py | 116 ++++++------------------------- lib/galaxy/datatypes/sniff.py | 6 +- tools/data_source/upload.py | 6 -- 4 files changed, 38 insertions(+), 118 deletions(-) diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index 93a71255709..a901c9fa98e 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -72,31 +72,31 @@ - - - - - - - - - - - - - + + + + + + + + + + + + + @@ -636,8 +636,8 @@ - + diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index 85e324fc916..c677320c00a 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -7,16 +7,15 @@ import json import logging import os import re -import subprocess import string import sys -import tempfile from cgi import escape import bx.align.maf from galaxy import util from galaxy.datatypes import metadata +from galaxy.datatypes.binary import Binary from galaxy.datatypes.metadata import MetadataElement from galaxy.datatypes.sniff import get_headers from galaxy.util import nice_size @@ -573,10 +572,12 @@ class Fastq ( Sequence ): data_lines = 0 sequences = 0 seq_counter = 0 # blocks should be 4 lines long - compress = is_gzip(dataset.file_name) - if compress: - self.decompress_fastqgz(dataset) - for line in open( dataset.file_name ): + compressed = is_gzip(dataset.file_name) + if compressed: + in_file = gzip.GzipFile(dataset.file_name) + else: + in_file = open(dataset.file_name) + for line in in_file: line = line.strip() if line and line.startswith( '#' ) and not data_lines: # We don't count comment lines for sequence data types @@ -611,6 +612,9 @@ class Fastq ( Sequence ): >>> Fastq().sniff( fname ) True """ + compressed = is_gzip(filename) + if compressed and not isinstance(self, Binary): + return False headers = get_headers( filename, None ) bases_regexp = re.compile( "^[NGTAC]*" ) # check that first block looks like a fastq block @@ -672,95 +676,6 @@ class Fastq ( Sequence ): return True process_split_file = staticmethod(process_split_file) -class FastqGz ( Fastq ): - """Class representing a generic compressed FASTQ sequence""" - edam_format = "format_1930" - file_ext = "fastq.gz" - - def set_meta( self, dataset, **kwd ): - """ - Set the number of sequences and the number of data lines - in dataset. - FIXME: This does not properly handle line wrapping - """ - if self.max_optional_metadata_filesize >= 0 and dataset.get_size() > self.max_optional_metadata_filesize: - dataset.metadata.data_lines = None - dataset.metadata.sequences = None - return - data_lines = 0 - sequences = 0 - seq_counter = 0 # blocks should be 4 lines long - compress = is_gzip(dataset.file_name) - if not compress: - self.compress_fastq(dataset) - for line in gzip.GzipFile(dataset.file_name, 'r'): - line = line.strip() - if line and line.startswith( '#' ) and not data_lines: - # We don't count comment lines for sequence data types - continue - seq_counter += 1 - data_lines += 1 - if line and line.startswith( '@' ): - if seq_counter >= 4: - # count previous block - # blocks should be 4 lines long - sequences += 1 - seq_counter = 1 - if seq_counter >= 4: - # count final block - sequences += 1 - dataset.metadata.data_lines = data_lines - dataset.metadata.sequences = sequences - - def compress_fastq(self,dataset): - print('X Compressing the fastq files') - if dataset.file_name and dataset.file_name.strip(): - compress = is_gzip(dataset.file_name) - if not compress: - # TODO: consider how to created temp file in Galaxy database/tmp folder - compressed_file = tempfile.NamedTemporaryFile(delete=False) - shutil.copyfileobj(open(dataset.file_name), gzip.open(compressed_file.name, 'wb')) - compressed_file.close() - os.unlink(dataset.file_name) - os.rename(compressed_file.name, dataset.file_name) - # # Compress fastq file - # stderr_name = tempfile.NamedTemporaryFile( prefix="fastq_compress" ).name - # command = ['gzip',dataset.file_name] - # try: - # exit_code = subprocess.call( args=command, stderr=open( stderr_name, 'wb' ) ) - # except Exception as e: - # log.warning( '%s, Compression Exception: %s', self, e ) - # - # # Rename compressed file - # os.rename(dataset.file_name + ".gz" , dataset.file_name) - else: - return False - else: - return False - - def sniff( self, filename ): - - """ - Determines whether the file is in generic fastq format - For details, see http://maq.sourceforge.net/fastq.shtml - - Note: There are three kinds of FASTQ files, known as "Sanger" (sometimes called "Standard"), Solexa, and Illumina - These differ in the representation of the quality scores - - >>> from galaxy.datatypes.sniff import get_test_fname - >>> fname = get_test_fname( '1.fastqsanger' ) - >>> Fastq().sniff( fname ) - True - >>> fname = get_test_fname( '2.fastqsanger' ) - >>> Fastq().sniff( fname ) - True - """ - is_compressed = is_gzip(filename) - is_fastq = super(FastqGz,self).sniff(filename) - if is_fastq and is_compressed: - return True - else: - return False class FastqSanger( Fastq ): """Class representing a FASTQ sequence ( the Sanger variant )""" @@ -785,27 +700,38 @@ class FastqCSSanger( Fastq ): file_ext = "fastqcssanger" +class FastqGz ( Fastq, Binary ): + """Class representing a generic compressed FASTQ sequence""" + edam_format = "format_1930" + file_ext = "fastq.gz" +Binary.register_sniffable_binary_format("fastq.gz", "fastq.gz", FastqGz) + + class FastqSangerGz( FastqGz ): """Class representing a compressed FASTQ sequence ( the Sanger variant )""" edam_format = "format_1932" file_ext = "fastqsanger.gz" +Binary.register_sniffable_binary_format("fastqsanger.gz", "fastqsanger.gz", FastqSangerGz) class FastqSolexaGz( FastqGz ): """Class representing a compressed FASTQ sequence ( the Solexa variant )""" edam_format = "format_1933" file_ext = "fastqsolexa.gz" +Binary.register_sniffable_binary_format("fastqsolexa.gz", "fastqsolexa.gz", FastqSolexaGz) class FastqIlluminaGz( FastqGz ): """Class representing a compressed FASTQ sequence ( the Illumina 1.3+ variant )""" edam_format = "format_1931" file_ext = "fastqillumina.gz" +Binary.register_sniffable_binary_format("fastqillumina.gz", "fastqillumina.gz", FastqIlluminaGz) class FastqCSSangerGz( FastqGz ): """Class representing a Color Space compressed FASTQ sequence ( e.g a SOLiD variant )""" file_ext = "fastqcssanger.gz" +Binary.register_sniffable_binary_format("fastqcssanger.gz", "fastqcssanger.gz", FastqCSSangerGz) class Maf( Alignment ): diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index ffb71a221ed..0196971a6f1 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -198,8 +198,8 @@ def get_headers( fname, sep, count=60, is_multi_byte=False ): [['chr7', '127475281', '127491632', 'NM_000230', '0', '+', '127486022', '127488767', '0', '3', '29,172,3225,', '0,10713,13126,'], ['chr7', '127486011', '127488900', 'D49487', '0', '+', '127486022', '127488767', '0', '2', '155,490,', '0,2399']] """ headers = [] - compress = is_gzip(fname) - if compress: + compressed = is_gzip(fname) + if compressed: in_file = gzip.GzipFile(fname, 'r') else: in_file = open(fname, 'rt') @@ -473,7 +473,7 @@ def handle_uploaded_dataset_file( filename, datatypes_registry, ext='auto', is_m AUTO_DETECT_EXTENSIONS = [ 'auto' ] # should 'data' also cause auto detect? DECOMPRESSION_FUNCTIONS = dict( gzip=gzip.GzipFile ) COMPRESSION_CHECK_FUNCTIONS = [ ( 'gzip', is_gzip ) ] -COMPRESSION_DATATYPES = dict( gzip=[ 'bam' ] ) +COMPRESSION_DATATYPES = dict( gzip=[ 'bam', 'fastq.gz', 'fastqsanger.gz', 'fastqillumina.gz', 'fastqsolexa.gz', 'fastqcssanger.gz' ] ) COMPRESSED_EXTENSIONS = [] for exts in COMPRESSION_DATATYPES.values(): COMPRESSED_EXTENSIONS.extend( exts ) diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py index 11bb22a26c8..85355c3eb3b 100644 --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -125,12 +125,6 @@ def add_file( dataset, registry, json_file, output_path ): if type_info: data_type = type_info[0] ext = type_info[1] - # Is dataset is compressed Fastq? - is_gzipped, is_valid = check_gzip( dataset.path ) - if is_gzipped and is_valid: - ext = sniff.guess_ext( dataset.path, registry.sniff_order) - if ext: - data_type = ext if not data_type: root_datatype = registry.get_datatype_by_extension( dataset.file_type ) if getattr( root_datatype, 'compressed', False ): From 10f1bb5459c4d27ca9142c0f4da1c8502ac9f1b2 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 11 Nov 2016 16:02:48 +0100 Subject: [PATCH 05/14] Create a BaseFastq class and add test tools Having a BaseFastq class allows Fastq and FastqGz classes to inherit from BaseFastq. FastqGz is therfor not a Fastq datatype and goes through the converter. Also add 2 tools to demonstrate that compressed fastq will be converted to uncompressed fastq if the tool specifies `format="fastq"`, while `format="fastq.gz"` leaves files compressed. The tool test can be run with: ``` planemo test --galaxy_root . test/functional/tools/compressed_fastq_no_conversion.xml planemo test --galaxy_root . test/functional/tools/compressed_fastq_conversion.xml ``` Note that fastq conversion appears to required a galaxy user session, and planemo testing fails. Interactive testing with `planemo serve` is not affected and works fine. --- lib/galaxy/datatypes/sequence.py | 12 ++++++++--- test-data/1.fastqsanger.gz | Bin 0 -> 161 bytes .../tools/compressed_fastq_conversion.xml | 19 ++++++++++++++++++ .../tools/compressed_fastq_no_conversion.xml | 19 ++++++++++++++++++ 4 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 test-data/1.fastqsanger.gz create mode 100644 test/functional/tools/compressed_fastq_conversion.xml create mode 100644 test/functional/tools/compressed_fastq_no_conversion.xml diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index c677320c00a..87de2f8630a 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -554,8 +554,8 @@ class csFasta( Sequence ): return Sequence.set_meta( self, dataset, **kwd ) -class Fastq ( Sequence ): - """Class representing a generic FASTQ sequence""" +class BaseFastq ( Sequence ): + """Base class for FastQ sequences""" edam_format = "format_1930" file_ext = "fastq" @@ -677,6 +677,12 @@ class Fastq ( Sequence ): process_split_file = staticmethod(process_split_file) +class Fastq( BaseFastq ): + """Class representing a generic FASTQ sequence""" + edam_format = "format_1930" + file_ext = "fastq" + + class FastqSanger( Fastq ): """Class representing a FASTQ sequence ( the Sanger variant )""" edam_format = "format_1932" @@ -700,7 +706,7 @@ class FastqCSSanger( Fastq ): file_ext = "fastqcssanger" -class FastqGz ( Fastq, Binary ): +class FastqGz ( BaseFastq, Binary ): """Class representing a generic compressed FASTQ sequence""" edam_format = "format_1930" file_ext = "fastq.gz" diff --git a/test-data/1.fastqsanger.gz b/test-data/1.fastqsanger.gz new file mode 100644 index 0000000000000000000000000000000000000000..56dbcd413a1419c3fb720bb139ee745307dade43 GIT binary patch literal 161 zcmV;S0ABweiwFo0(j`~||1mCRVRLkGb75|0WpV&b%rOqbAPfc2or928q)8wkLJPsx zAGm<`-2WDVcBzMN%hG>jkKJA&?w8#*zgopC3=u(VXaIzn0d5fVf9s=+Iy1CRDL8{O zb%7c?C+a literal 0 HcmV?d00001 diff --git a/test/functional/tools/compressed_fastq_conversion.xml b/test/functional/tools/compressed_fastq_conversion.xml new file mode 100644 index 00000000000..72cab01c3b9 --- /dev/null +++ b/test/functional/tools/compressed_fastq_conversion.xml @@ -0,0 +1,19 @@ + + + cat '$input1' > '$out_file1' + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/compressed_fastq_no_conversion.xml b/test/functional/tools/compressed_fastq_no_conversion.xml new file mode 100644 index 00000000000..845292869fd --- /dev/null +++ b/test/functional/tools/compressed_fastq_no_conversion.xml @@ -0,0 +1,19 @@ + + + cat '$input1' > '$out_file1' + + + + + + + + + + + + + + + + From b44a977e649e3c6ac7cfc0bc519c242139296789 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 12 Nov 2016 10:09:42 +0100 Subject: [PATCH 06/14] Quote paths in converters --- .../datatypes/converters/fastqcssangergz_to_fastqcssanger.xml | 2 +- lib/galaxy/datatypes/converters/fastqgz_to_fastq.xml | 2 +- .../datatypes/converters/fastqilluminagz_to_fastqillumina.xml | 2 +- .../datatypes/converters/fastqsangergz_to_fastqsanger.xml | 2 +- .../datatypes/converters/fastqsolexagz_to_fastqsolexa.xml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/datatypes/converters/fastqcssangergz_to_fastqcssanger.xml b/lib/galaxy/datatypes/converters/fastqcssangergz_to_fastqcssanger.xml index 0dffa753a36..4bcc85ee056 100644 --- a/lib/galaxy/datatypes/converters/fastqcssangergz_to_fastqcssanger.xml +++ b/lib/galaxy/datatypes/converters/fastqcssangergz_to_fastqcssanger.xml @@ -1,5 +1,5 @@