diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index 2129d9d8979..ceb4b2c285f 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -16,6 +16,10 @@ + + + + @@ -284,6 +288,7 @@ + diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 2b7325a2ed6..f8fbd59b514 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -187,14 +187,11 @@ class GenericAsn1Binary(Binary): edam_data = "data_0849" -@dataproviders.decorators.has_dataproviders -class Bam(Binary): - """Class describing a BAM binary file""" +class BamNative(Binary): + """Class describing a BAM binary file that is not necessarily sorted""" edam_format = "format_2572" edam_data = "data_0863" - file_ext = "bam" - track_type = "ReadTrack" - data_sources = {"data": "bai", "index": "bigwig"} + file_ext = "bam_native" MetadataElement(name="bam_index", desc="BAM Index File", param=metadata.FileParameter, file_ext="bai", readonly=True, no_value=None, visible=False, optional=True) MetadataElement(name="bam_version", default=None, desc="BAM Version", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value=None) @@ -217,80 +214,9 @@ class Bam(Binary): """ pysam.merge('-O', 'BAM', output_file, *split_files) - def dataset_content_needs_grooming(self, file_name): - """ - Check if file_name is a coordinate-sorted BAM file - """ - # The best way to ensure that BAM files are coordinate-sorted and indexable - # is to actually index them. - index_name = tempfile.NamedTemporaryFile(prefix="bam_index").name - try: - # If pysam fails to index a file it will write to stderr, - # and this causes the set_meta script to fail. So instead - # we start another process and discard stderr. - cmd = ['python', '-c', "import pysam; pysam.index('%s', '%s')" % (file_name, index_name)] - with open(os.devnull, 'w') as devnull: - subprocess.check_call(cmd, stderr=devnull, shell=False) - needs_sorting = False - except subprocess.CalledProcessError: - needs_sorting = True - try: - os.unlink(index_name) - except Exception: - pass - return needs_sorting - - def groom_dataset_content(self, file_name): - """ - Ensures that the BAM file contents are sorted. This function is called - on an output dataset after the content is initially generated. - """ - # Use pysam to sort the BAM file - # This command may also creates temporary files .%d.bam when the - # whole alignment cannot fit into memory. - # do this in a unique temp directory, because of possible .%d.bam temp files - if not self.dataset_content_needs_grooming(file_name): - # Don't re-sort if already sorted - return - tmp_dir = tempfile.mkdtemp() - tmp_sorted_dataset_file_name_prefix = os.path.join(tmp_dir, 'sorted') - sorted_file_name = "%s.bam" % tmp_sorted_dataset_file_name_prefix - slots = os.environ.get('GALAXY_SLOTS', 1) - try: - pysam.sort("-@%s" % slots, file_name, '-T', tmp_sorted_dataset_file_name_prefix, '-O', 'BAM', '-o', sorted_file_name) - except Exception: - shutil.rmtree(tmp_dir, ignore_errors=True) - raise - # Move samtools_created_sorted_file_name to our output dataset location - shutil.move(sorted_file_name, file_name) - # Remove temp file and empty temporary directory - os.rmdir(tmp_dir) - def init_meta(self, dataset, copy_from=None): Binary.init_meta(self, dataset, copy_from=copy_from) - def set_meta(self, dataset, overwrite=True, **kwd): - # These metadata values are not accessible by users, always overwrite - index_file = dataset.metadata.bam_index - if not index_file: - index_file = dataset.metadata.spec['bam_index'].param.new_file(dataset=dataset) - pysam.index(dataset.file_name, index_file.file_name) - dataset.metadata.bam_index = index_file - # Now use pysam with BAI index to determine additional metadata - try: - bam_file = pysam.AlignmentFile(dataset.file_name, mode='rb', index_filename=index_file.file_name) - # TODO: Reference names, lengths, read_groups and headers can become very large, truncate when necessary - dataset.metadata.reference_names = list(bam_file.references) - dataset.metadata.reference_lengths = list(bam_file.lengths) - dataset.metadata.bam_header = bam_file.header - dataset.metadata.read_groups = [read_group['ID'] for read_group in dataset.metadata.bam_header.get('RG', []) if 'ID' in read_group] - dataset.metadata.sort_order = bam_file.header.get('HD', {}).get('SO', None) - dataset.metadata.bam_version = bam_file.header.get('HD', {}).get('VN', None) - except Exception: - # Per Dan, don't log here because doing so will cause datasets that - # fail metadata to end in the error state - pass - def sniff(self, filename): # BAM is compressed in the BGZF format, and must not be uncompressed in Galaxy. # The first 4 bytes of any bam file is 'BAM\1', and the file is binary. @@ -302,6 +228,21 @@ class Bam(Binary): except Exception: return False + def set_meta(self, dataset, overwrite=True, **kwd): + try: + bam_file = pysam.AlignmentFile(dataset.file_name, mode='rb') + # TODO: Reference names, lengths, read_groups and headers can become very large, truncate when necessary + dataset.metadata.reference_names = list(bam_file.references) + dataset.metadata.reference_lengths = list(bam_file.lengths) + dataset.metadata.bam_header = bam_file.header + dataset.metadata.read_groups = [read_group['ID'] for read_group in dataset.metadata.bam_header.get('RG', []) if 'ID' in read_group] + dataset.metadata.sort_order = bam_file.header.get('HD', {}).get('SO', None) + dataset.metadata.bam_version = bam_file.header.get('HD', {}).get('VN', None) + except Exception: + # Per Dan, don't log here because doing so will cause datasets that + # fail metadata to end in the error state + pass + def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: dataset.peek = "Binary bam alignments file" @@ -326,10 +267,9 @@ class Bam(Binary): return zip(file_paths, rel_paths) def get_chunk(self, trans, dataset, offset=0, ck_size=None): - index_file = dataset.metadata.bam_index if not offset == -1: try: - with pysam.AlignmentFile(dataset.file_name, "rb", index_filename=index_file.file_name) as bamfile: + with pysam.AlignmentFile(dataset.file_name, "rb") as bamfile: ck_size = 300 # 300 lines ck_data = "" header_line_count = 0 @@ -382,6 +322,90 @@ class Bam(Binary): column_names=column_names, column_types=column_types) + +@dataproviders.decorators.has_dataproviders +class Bam(BamNative): + """Class describing a BAM binary file""" + edam_format = "format_2572" + edam_data = "data_0863" + file_ext = "bam" + track_type = "ReadTrack" + data_sources = {"data": "bai", "index": "bigwig"} + + def dataset_content_needs_grooming(self, file_name): + """ + Check if file_name is a coordinate-sorted BAM file + """ + # The best way to ensure that BAM files are coordinate-sorted and indexable + # is to actually index them. + index_name = tempfile.NamedTemporaryFile(prefix="bam_index").name + try: + # If pysam fails to index a file it will write to stderr, + # and this causes the set_meta script to fail. So instead + # we start another process and discard stderr. + cmd = ['python', '-c', "import pysam; pysam.index('%s', '%s')" % (file_name, index_name)] + with open(os.devnull, 'w') as devnull: + subprocess.check_call(cmd, stderr=devnull, shell=False) + needs_sorting = False + except subprocess.CalledProcessError: + needs_sorting = True + try: + os.unlink(index_name) + except Exception: + pass + return needs_sorting + + def groom_dataset_content(self, file_name): + """ + Ensures that the BAM file contents are sorted. This function is called + on an output dataset after the content is initially generated. + """ + # Use pysam to sort the BAM file + # This command may also creates temporary files .%d.bam when the + # whole alignment cannot fit into memory. + # do this in a unique temp directory, because of possible .%d.bam temp files + if not self.dataset_content_needs_grooming(file_name): + # Don't re-sort if already sorted + return + tmp_dir = tempfile.mkdtemp() + tmp_sorted_dataset_file_name_prefix = os.path.join(tmp_dir, 'sorted') + sorted_file_name = "%s.bam" % tmp_sorted_dataset_file_name_prefix + slots = os.environ.get('GALAXY_SLOTS', 1) + try: + pysam.sort("-@%s" % slots, file_name, '-T', tmp_sorted_dataset_file_name_prefix, '-O', 'BAM', '-o', sorted_file_name) + except Exception: + shutil.rmtree(tmp_dir, ignore_errors=True) + raise + # Move samtools_created_sorted_file_name to our output dataset location + shutil.move(sorted_file_name, file_name) + # Remove temp file and empty temporary directory + os.rmdir(tmp_dir) + + def set_meta(self, dataset, overwrite=True, **kwd): + # These metadata values are not accessible by users, always overwrite + index_file = dataset.metadata.bam_index + if not index_file: + index_file = dataset.metadata.spec['bam_index'].param.new_file(dataset=dataset) + pysam.index(dataset.file_name, index_file.file_name) + dataset.metadata.bam_index = index_file + # Now use pysam with BAI index to determine additional metadata + try: + bam_file = pysam.AlignmentFile(dataset.file_name, mode='rb', index_filename=index_file.file_name) + # TODO: Reference names, lengths, read_groups and headers can become very large, truncate when necessary + dataset.metadata.reference_names = list(bam_file.references) + dataset.metadata.reference_lengths = list(bam_file.lengths) + dataset.metadata.bam_header = bam_file.header + dataset.metadata.read_groups = [read_group['ID'] for read_group in dataset.metadata.bam_header.get('RG', []) if 'ID' in read_group] + dataset.metadata.sort_order = bam_file.header.get('HD', {}).get('SO', None) + dataset.metadata.bam_version = bam_file.header.get('HD', {}).get('VN', None) + except Exception: + # Per Dan, don't log here because doing so will cause datasets that + # fail metadata to end in the error state + pass + + def sniff(self, file_name): + return super(Bam, self).sniff(file_name) and not self.dataset_content_needs_grooming(file_name) + # ------------- Dataproviders # pipe through samtools view # ALSO: (as Sam) diff --git a/lib/galaxy/datatypes/converters/bam_native_to_bam_converter.xml b/lib/galaxy/datatypes/converters/bam_native_to_bam_converter.xml new file mode 100644 index 00000000000..8ac1dd7b633 --- /dev/null +++ b/lib/galaxy/datatypes/converters/bam_native_to_bam_converter.xml @@ -0,0 +1,23 @@ + diff --git a/lib/galaxy/datatypes/converters/bam_to_bigwig_converter.xml b/lib/galaxy/datatypes/converters/bam_to_bigwig_converter.xml index 6307bcf0d64..50b8c1ca14c 100644 --- a/lib/galaxy/datatypes/converters/bam_to_bigwig_converter.xml +++ b/lib/galaxy/datatypes/converters/bam_to_bigwig_converter.xml @@ -17,7 +17,7 @@ > temp.bg && bedGraphToBigWig temp.bg '$chromInfo' '$output']]> - + diff --git a/lib/galaxy/datatypes/converters/sam_to_bam_native.xml b/lib/galaxy/datatypes/converters/sam_to_bam_native.xml new file mode 100644 index 00000000000..5e8475ea52a --- /dev/null +++ b/lib/galaxy/datatypes/converters/sam_to_bam_native.xml @@ -0,0 +1,23 @@ + + + + samtools + + + + + + + + + + + + diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index cf058cf069a..90c914d6f32 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -337,7 +337,7 @@ def guess_ext(fname, sniff_order): 'bam' >>> fname = get_test_fname('3unsorted.bam') >>> guess_ext(fname, sniff_order) - 'bam' + 'bam_native' >>> fname = get_test_fname('test.idpDB') >>> guess_ext(fname, sniff_order) 'idpdb' diff --git a/scripts/functional_tests.py b/scripts/functional_tests.py index 51228a384bb..0493a49bd3b 100644 --- a/scripts/functional_tests.py +++ b/scripts/functional_tests.py @@ -71,6 +71,8 @@ class FrameworkToolsGalaxyTestDriver(DefaultGalaxyTestDriver): """Galaxy-style nose TestDriver for testing framework Galaxy tools.""" framework_tool_and_types = True + conda_auto_init = True + conda_auto_install = True class DataManagersGalaxyTestDriver(driver_util.GalaxyTestDriver): diff --git a/test-data/bam_native_from_sam.bam b/test-data/bam_native_from_sam.bam new file mode 100644 index 00000000000..0052a80bd03 Binary files /dev/null and b/test-data/bam_native_from_sam.bam differ diff --git a/test-data/sam_with_header.sam b/test-data/sam_with_header.sam index 33449b176bc..f2428278d55 100644 --- a/test-data/sam_with_header.sam +++ b/test-data/sam_with_header.sam @@ -1,14 +1,14 @@ @SQ SN:ref LN:45 @SQ SN:ref2 LN:40 +r003 16 ref 29 30 6H5M * 0 0 TAGGC * +r001 83 ref 37 30 9M = 7 -39 CAGCGCCAT * +x2 0 ref2 2 30 21M * 0 0 ggttttataaaacaaataatt ????????????????????? r001 163 ref 7 30 8M4I4M1D3M = 37 39 TTAGATAAAGAGGATACTG * XX:B:S,12561,2,20,112 r002 0 ref 9 30 1S2I6M1P1I1P1I4M2I * 0 0 AAAAGATAAGGGATAAA * r003 0 ref 9 30 5H6M * 0 0 AGCTAA * r004 0 ref 16 30 6M14N1I5M * 0 0 ATAGCTCTCAGC * -r003 16 ref 29 30 6H5M * 0 0 TAGGC * -r001 83 ref 37 30 9M = 7 -39 CAGCGCCAT * -x1 0 ref2 1 30 20M * 0 0 aggttttataaaacaaataa ???????????????????? -x2 0 ref2 2 30 21M * 0 0 ggttttataaaacaaataatt ????????????????????? x3 0 ref2 6 30 9M4I13M * 0 0 ttataaaacAAATaattaagtctaca ?????????????????????????? x4 0 ref2 10 30 25M * 0 0 CaaaTaattaagtctacagagcaac ????????????????????????? x5 0 ref2 12 30 24M * 0 0 aaTaattaagtctacagagcaact ???????????????????????? +x1 0 ref2 1 30 20M * 0 0 aggttttataaaacaaataa ???????????????????? x6 0 ref2 14 30 23M * 0 0 Taattaagtctacagagcaacta ??????????????????????? diff --git a/test/base/driver_util.py b/test/base/driver_util.py index f2beea22f2c..84ae2a2e313 100644 --- a/test/base/driver_util.py +++ b/test/base/driver_util.py @@ -129,6 +129,8 @@ def setup_galaxy_config( update_integrated_tool_panel=False, prefer_template_database=False, log_format=None, + conda_auto_init=False, + conda_auto_install=False ): """Setup environment and build config for test Galaxy instance.""" if not os.path.exists(tmpdir): @@ -188,7 +190,8 @@ def setup_galaxy_config( api_allow_run_as='test@bx.psu.edu', auto_configure_logging=logging_config_file is None, check_migrate_tools=False, - conda_auto_init=False, + conda_auto_init=conda_auto_init, + conda_auto_install=conda_auto_install, cleanup_job='onsuccess', data_manager_config_file=data_manager_config_file, enable_beta_tool_formats=True, @@ -851,6 +854,8 @@ class GalaxyTestDriver(TestDriver): datatypes_conf=datatypes_conf_override, prefer_template_database=getattr(config_object, "prefer_template_database", False), log_format=log_format, + conda_auto_init=getattr(config_object, "conda_auto_init", False), + conda_auto_install=getattr(config_object, "conda_auto_install", False), ) galaxy_config = setup_galaxy_config( galaxy_db_path, diff --git a/test/functional/tools/sam_to_bam_native.xml b/test/functional/tools/sam_to_bam_native.xml new file mode 100644 index 00000000000..aed8f24892c --- /dev/null +++ b/test/functional/tools/sam_to_bam_native.xml @@ -0,0 +1,48 @@ + + + samtools + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/sample_datatypes_conf.xml b/test/functional/tools/sample_datatypes_conf.xml index 8b366098630..c2f8edbe9c6 100644 --- a/test/functional/tools/sample_datatypes_conf.xml +++ b/test/functional/tools/sample_datatypes_conf.xml @@ -14,8 +14,16 @@ - + + + + + + + + + diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml index 6a1070992fc..489eefc1eb2 100644 --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -83,6 +83,7 @@ +