opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5180 from bgruening/bam_unsorted

Browse Source 
Add BamNative datatype
This commit is contained in:
Marius van den Beek
2018-01-19 16:37:36 +01:00
committed by GitHub
parent 33bd54079b 8ecc86cb67
commit 8f65cd46b0
13 changed files with 226 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/datatypes_conf.xml.sample
+5
View File
@@ -16,6 +16,10 @@
<display file="igb/bam.xml" />
<display file="iobio/bam.xml" />
</datatype>
<datatype extension="bam_native" type="galaxy.datatypes.binary:BamNative" mimetype="application/octet-stream" display_in_upload="true" description="A binary file compressed in the BGZF format with a '.bam' file extension." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#BAM">
<converter file="bam_to_bigwig_converter.xml" target_datatype="bigwig"/>
<converter file="bam_native_to_bam_converter.xml" target_datatype="bam"/>
</datatype>
<datatype extension="cram" type="galaxy.datatypes.binary:CRAM" mimetype="application/octet-stream" display_in_upload="true" description="CRAM is a file format for highly efficient and tunable reference-based compression of alignment data." description_url="http://www.ebi.ac.uk/ena/software/cram-usage">
<converter file="cram_to_bam_converter.xml" target_datatype="bam"/>
</datatype>
@@ -284,6 +288,7 @@
<datatype extension="qual454" type="galaxy.datatypes.qualityscore:QualityScore454" display_in_upload="true"/>
<datatype extension="roadmaps" type="galaxy.datatypes.assembly:Roadmaps" display_in_upload="false"/>
<datatype extension="sam" type="galaxy.datatypes.tabular:Sam" display_in_upload="true">
<converter file="sam_to_bam_native.xml" target_datatype="bam_native"/>
<converter file="sam_to_bam.xml" target_datatype="bam"/>
<converter file="sam_to_bigwig_converter.xml" target_datatype="bigwig"/>
</datatype>
lib/galaxy/datatypes/binary.py
+103 -79
View File
@@ -187,14 +187,11 @@ class GenericAsn1Binary(Binary):
edam_data = "data_0849"
@dataproviders.decorators.has_dataproviders
class Bam(Binary):
"""Class describing a BAM binary file"""
class BamNative(Binary):
"""Class describing a BAM binary file that is not necessarily sorted"""
edam_format = "format_2572"
edam_data = "data_0863"
file_ext = "bam"
track_type = "ReadTrack"
data_sources = {"data": "bai", "index": "bigwig"}
file_ext = "bam_native"
MetadataElement(name="bam_index", desc="BAM Index File", param=metadata.FileParameter, file_ext="bai", readonly=True, no_value=None, visible=False, optional=True)
MetadataElement(name="bam_version", default=None, desc="BAM Version", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value=None)
@@ -217,80 +214,9 @@ class Bam(Binary):
"""
pysam.merge('-O', 'BAM', output_file, *split_files)
def dataset_content_needs_grooming(self, file_name):
"""
Check if file_name is a coordinate-sorted BAM file
"""
# The best way to ensure that BAM files are coordinate-sorted and indexable
# is to actually index them.
index_name = tempfile.NamedTemporaryFile(prefix="bam_index").name
try:
# If pysam fails to index a file it will write to stderr,
# and this causes the set_meta script to fail. So instead
# we start another process and discard stderr.
cmd = ['python', '-c', "import pysam; pysam.index('%s', '%s')" % (file_name, index_name)]
with open(os.devnull, 'w') as devnull:
subprocess.check_call(cmd, stderr=devnull, shell=False)
needs_sorting = False
except subprocess.CalledProcessError:
needs_sorting = True
try:
os.unlink(index_name)
except Exception:
pass
return needs_sorting
def groom_dataset_content(self, file_name):
"""
Ensures that the BAM file contents are sorted. This function is called
on an output dataset after the content is initially generated.
"""
# Use pysam to sort the BAM file
# This command may also creates temporary files <out.prefix>.%d.bam when the
# whole alignment cannot fit into memory.
# do this in a unique temp directory, because of possible <out.prefix>.%d.bam temp files
if not self.dataset_content_needs_grooming(file_name):
# Don't re-sort if already sorted
return
tmp_dir = tempfile.mkdtemp()
tmp_sorted_dataset_file_name_prefix = os.path.join(tmp_dir, 'sorted')
sorted_file_name = "%s.bam" % tmp_sorted_dataset_file_name_prefix
slots = os.environ.get('GALAXY_SLOTS', 1)
try:
pysam.sort("-@%s" % slots, file_name, '-T', tmp_sorted_dataset_file_name_prefix, '-O', 'BAM', '-o', sorted_file_name)
except Exception:
shutil.rmtree(tmp_dir, ignore_errors=True)
raise
# Move samtools_created_sorted_file_name to our output dataset location
shutil.move(sorted_file_name, file_name)
# Remove temp file and empty temporary directory
os.rmdir(tmp_dir)
def init_meta(self, dataset, copy_from=None):
Binary.init_meta(self, dataset, copy_from=copy_from)
def set_meta(self, dataset, overwrite=True, **kwd):
# These metadata values are not accessible by users, always overwrite
index_file = dataset.metadata.bam_index
if not index_file:
index_file = dataset.metadata.spec['bam_index'].param.new_file(dataset=dataset)
pysam.index(dataset.file_name, index_file.file_name)
dataset.metadata.bam_index = index_file
# Now use pysam with BAI index to determine additional metadata
try:
bam_file = pysam.AlignmentFile(dataset.file_name, mode='rb', index_filename=index_file.file_name)
# TODO: Reference names, lengths, read_groups and headers can become very large, truncate when necessary
dataset.metadata.reference_names = list(bam_file.references)
dataset.metadata.reference_lengths = list(bam_file.lengths)
dataset.metadata.bam_header = bam_file.header
dataset.metadata.read_groups = [read_group['ID'] for read_group in dataset.metadata.bam_header.get('RG', []) if 'ID' in read_group]
dataset.metadata.sort_order = bam_file.header.get('HD', {}).get('SO', None)
dataset.metadata.bam_version = bam_file.header.get('HD', {}).get('VN', None)
except Exception:
# Per Dan, don't log here because doing so will cause datasets that
# fail metadata to end in the error state
pass
def sniff(self, filename):
# BAM is compressed in the BGZF format, and must not be uncompressed in Galaxy.
# The first 4 bytes of any bam file is 'BAM\1', and the file is binary.
@@ -302,6 +228,21 @@ class Bam(Binary):
except Exception:
return False
def set_meta(self, dataset, overwrite=True, **kwd):
try:
bam_file = pysam.AlignmentFile(dataset.file_name, mode='rb')
# TODO: Reference names, lengths, read_groups and headers can become very large, truncate when necessary
dataset.metadata.reference_names = list(bam_file.references)
dataset.metadata.reference_lengths = list(bam_file.lengths)
dataset.metadata.bam_header = bam_file.header
dataset.metadata.read_groups = [read_group['ID'] for read_group in dataset.metadata.bam_header.get('RG', []) if 'ID' in read_group]
dataset.metadata.sort_order = bam_file.header.get('HD', {}).get('SO', None)
dataset.metadata.bam_version = bam_file.header.get('HD', {}).get('VN', None)
except Exception:
# Per Dan, don't log here because doing so will cause datasets that
# fail metadata to end in the error state
pass
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = "Binary bam alignments file"
@@ -326,10 +267,9 @@ class Bam(Binary):
return zip(file_paths, rel_paths)
def get_chunk(self, trans, dataset, offset=0, ck_size=None):
index_file = dataset.metadata.bam_index
if not offset == -1:
try:
with pysam.AlignmentFile(dataset.file_name, "rb", index_filename=index_file.file_name) as bamfile:
with pysam.AlignmentFile(dataset.file_name, "rb") as bamfile:
ck_size = 300 # 300 lines
ck_data = ""
header_line_count = 0
@@ -382,6 +322,90 @@ class Bam(Binary):
column_names=column_names,
column_types=column_types)
@dataproviders.decorators.has_dataproviders
class Bam(BamNative):
"""Class describing a BAM binary file"""
edam_format = "format_2572"
edam_data = "data_0863"
file_ext = "bam"
track_type = "ReadTrack"
data_sources = {"data": "bai", "index": "bigwig"}
def dataset_content_needs_grooming(self, file_name):
"""
Check if file_name is a coordinate-sorted BAM file
"""
# The best way to ensure that BAM files are coordinate-sorted and indexable
# is to actually index them.
index_name = tempfile.NamedTemporaryFile(prefix="bam_index").name
try:
# If pysam fails to index a file it will write to stderr,
# and this causes the set_meta script to fail. So instead
# we start another process and discard stderr.
cmd = ['python', '-c', "import pysam; pysam.index('%s', '%s')" % (file_name, index_name)]
with open(os.devnull, 'w') as devnull:
subprocess.check_call(cmd, stderr=devnull, shell=False)
needs_sorting = False
except subprocess.CalledProcessError:
needs_sorting = True
try:
os.unlink(index_name)
except Exception:
pass
return needs_sorting
def groom_dataset_content(self, file_name):
"""
Ensures that the BAM file contents are sorted. This function is called
on an output dataset after the content is initially generated.
"""
# Use pysam to sort the BAM file
# This command may also creates temporary files <out.prefix>.%d.bam when the
# whole alignment cannot fit into memory.
# do this in a unique temp directory, because of possible <out.prefix>.%d.bam temp files
if not self.dataset_content_needs_grooming(file_name):
# Don't re-sort if already sorted
return
tmp_dir = tempfile.mkdtemp()
tmp_sorted_dataset_file_name_prefix = os.path.join(tmp_dir, 'sorted')
sorted_file_name = "%s.bam" % tmp_sorted_dataset_file_name_prefix
slots = os.environ.get('GALAXY_SLOTS', 1)
try:
pysam.sort("-@%s" % slots, file_name, '-T', tmp_sorted_dataset_file_name_prefix, '-O', 'BAM', '-o', sorted_file_name)
except Exception:
shutil.rmtree(tmp_dir, ignore_errors=True)
raise
# Move samtools_created_sorted_file_name to our output dataset location
shutil.move(sorted_file_name, file_name)
# Remove temp file and empty temporary directory
os.rmdir(tmp_dir)
def set_meta(self, dataset, overwrite=True, **kwd):
# These metadata values are not accessible by users, always overwrite
index_file = dataset.metadata.bam_index
if not index_file:
index_file = dataset.metadata.spec['bam_index'].param.new_file(dataset=dataset)
pysam.index(dataset.file_name, index_file.file_name)
dataset.metadata.bam_index = index_file
# Now use pysam with BAI index to determine additional metadata
try:
bam_file = pysam.AlignmentFile(dataset.file_name, mode='rb', index_filename=index_file.file_name)
# TODO: Reference names, lengths, read_groups and headers can become very large, truncate when necessary
dataset.metadata.reference_names = list(bam_file.references)
dataset.metadata.reference_lengths = list(bam_file.lengths)
dataset.metadata.bam_header = bam_file.header
dataset.metadata.read_groups = [read_group['ID'] for read_group in dataset.metadata.bam_header.get('RG', []) if 'ID' in read_group]
dataset.metadata.sort_order = bam_file.header.get('HD', {}).get('SO', None)
dataset.metadata.bam_version = bam_file.header.get('HD', {}).get('VN', None)
except Exception:
# Per Dan, don't log here because doing so will cause datasets that
# fail metadata to end in the error state
pass
def sniff(self, file_name):
return super(Bam, self).sniff(file_name) and not self.dataset_content_needs_grooming(file_name)
# ------------- Dataproviders
# pipe through samtools view
# ALSO: (as Sam)
lib/galaxy/datatypes/converters/bam_native_to_bam_converter.xml
+23
View File
@@ -0,0 +1,23 @@
<tool id="CONVERTER_bam_native_to_bam" name="Convert BAM native to BAM" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<requirements>
<requirement type="package" version="1.6">samtools</requirement>
</requirements>
<command><![CDATA[
samtools sort
-@ \${GALAXY_SLOTS:-1}
-o '${output}'
-O bam
-T dataset
'${input}'
]]>
</command>
<inputs>
<param format="bam_native" name="input" type="data" label="Choose a BAM native file"/>
</inputs>
<outputs>
<data format="bam" name="output"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/bam_to_bigwig_converter.xml
+1 -1
View File
@@ -17,7 +17,7 @@
> temp.bg && bedGraphToBigWig temp.bg '$chromInfo' '$output']]>
</command>
<inputs>
<param format="bam" name="input" type="data" label="Choose BAM file"/>
<param format="bam,bam_native" name="input" type="data" label="Choose BAM file"/>
</inputs>
<outputs>
<data format="bigwig" name="output"/>
lib/galaxy/datatypes/converters/sam_to_bam_native.xml
+23
View File
@@ -0,0 +1,23 @@
<tool id="CONVERTER_sam_to_bam_native" name="Convert SAM to BAM native - without sorting" version="1.0.0">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<requirements>
<requirement type="package" version="1.6">samtools</requirement>
</requirements>
<command><![CDATA[
samtools view
-b
-h
-@ \${GALAXY_SLOTS:-2}
-o '${output}'
'$input'
]]>
</command>
<inputs>
<param name="input" type="data" format="sam" label="SAM file"/>
</inputs>
<outputs>
<data name="output" format="bam_native"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/sniff.py
+1 -1
View File
@@ -337,7 +337,7 @@ def guess_ext(fname, sniff_order):
'bam'
>>> fname = get_test_fname('3unsorted.bam')
>>> guess_ext(fname, sniff_order)
'bam'
'bam_native'
>>> fname = get_test_fname('test.idpDB')
>>> guess_ext(fname, sniff_order)
'idpdb'
scripts/functional_tests.py
+2
View File
@@ -71,6 +71,8 @@ class FrameworkToolsGalaxyTestDriver(DefaultGalaxyTestDriver):
"""Galaxy-style nose TestDriver for testing framework Galaxy tools."""
framework_tool_and_types = True
conda_auto_init = True
conda_auto_install = True
class DataManagersGalaxyTestDriver(driver_util.GalaxyTestDriver):
test-data/bam_native_from_sam.bam
BIN
View File
Binary file not shown.
test-data/sam_with_header.sam
+4 -4
View File
@@ -1,14 +1,14 @@
@SQ SN:ref LN:45
@SQ SN:ref2 LN:40
r003 16 ref 29 30 6H5M * 0 0 TAGGC *
r001 83 ref 37 30 9M = 7 -39 CAGCGCCAT *
x2 0 ref2 2 30 21M * 0 0 ggttttataaaacaaataatt ?????????????????????
r001 163 ref 7 30 8M4I4M1D3M = 37 39 TTAGATAAAGAGGATACTG * XX:B:S,12561,2,20,112
r002 0 ref 9 30 1S2I6M1P1I1P1I4M2I * 0 0 AAAAGATAAGGGATAAA *
r003 0 ref 9 30 5H6M * 0 0 AGCTAA *
r004 0 ref 16 30 6M14N1I5M * 0 0 ATAGCTCTCAGC *
r003 16 ref 29 30 6H5M * 0 0 TAGGC *
r001 83 ref 37 30 9M = 7 -39 CAGCGCCAT *
x1 0 ref2 1 30 20M * 0 0 aggttttataaaacaaataa ????????????????????
x2 0 ref2 2 30 21M * 0 0 ggttttataaaacaaataatt ?????????????????????
x3 0 ref2 6 30 9M4I13M * 0 0 ttataaaacAAATaattaagtctaca ??????????????????????????
x4 0 ref2 10 30 25M * 0 0 CaaaTaattaagtctacagagcaac ?????????????????????????
x5 0 ref2 12 30 24M * 0 0 aaTaattaagtctacagagcaact ????????????????????????
x1 0 ref2 1 30 20M * 0 0 aggttttataaaacaaataa ????????????????????
x6 0 ref2 14 30 23M * 0 0 Taattaagtctacagagcaacta ???????????????????????
test/base/driver_util.py
+6 -1
View File
@@ -129,6 +129,8 @@ def setup_galaxy_config(
update_integrated_tool_panel=False,
prefer_template_database=False,
log_format=None,
conda_auto_init=False,
conda_auto_install=False
):
"""Setup environment and build config for test Galaxy instance."""
if not os.path.exists(tmpdir):
@@ -188,7 +190,8 @@ def setup_galaxy_config(
api_allow_run_as='test@bx.psu.edu',
auto_configure_logging=logging_config_file is None,
check_migrate_tools=False,
conda_auto_init=False,
conda_auto_init=conda_auto_init,
conda_auto_install=conda_auto_install,
cleanup_job='onsuccess',
data_manager_config_file=data_manager_config_file,
enable_beta_tool_formats=True,
@@ -851,6 +854,8 @@ class GalaxyTestDriver(TestDriver):
datatypes_conf=datatypes_conf_override,
prefer_template_database=getattr(config_object, "prefer_template_database", False),
log_format=log_format,
conda_auto_init=getattr(config_object, "conda_auto_init", False),
conda_auto_install=getattr(config_object, "conda_auto_install", False),
)
galaxy_config = setup_galaxy_config(
galaxy_db_path,
test/functional/tools/sam_to_bam_native.xml
+48
View File
@@ -0,0 +1,48 @@
<tool id="sam_to_bam_native_conversion" name="Test sam to bam native conversion">
<requirements>
<requirement type="package" version="1.6">samtools</requirement>
</requirements>
<command><![CDATA[
#if $input1:
samtools view
-b
-h
-@ \${GALAXY_SLOTS:-2}
-o '$bam_native_output'
'$input1'
#elif $input2:
cp '$input2' '$bam_native_output'
#elif $input3:
cp '$input3' '$bam_output'
#end if
]]>
</command>
<inputs>
<param name="input1" type="data" format="sam" label="SAM file" optional="true"/>
<param name="input2" type="data" format="bam_native" label="Unsorted BAM file" optional="true"/>
<param name="input3" type="data" format="bam" label="Sorted BAM file" optional="true"/>
</inputs>
<outputs>
<data name="bam_native_output" format="bam_native"/>
<data name="bam_output" format="bam"/>
</outputs>
<tests>
<!-- Test that bam native output won't be sorted-->
<test>
<param name="input1" value="sam_with_header.sam" ftype="sam"/>
<output name="bam_native_output" file="bam_native_from_sam.bam" ftype="bam_native"/>
</test>
<!-- Test that sam input is properly converted to bam native -->
<test>
<param name="input2" value="sam_with_header.sam" ftype="sam"/>
<output name="bam_native_output" file="bam_native_from_sam.bam" ftype="bam_native"/>
</test>
<!-- Test that sam input is properly converted to bam -->
<test>
<param name="input3" value="sam_with_header.sam" ftype="sam"/>
<output name="bam_output" file="bam_from_sam.bam" ftype="bam"/>
</test>
</tests>
<help>
</help>
</tool>
test/functional/tools/sample_datatypes_conf.xml
+9 -1
View File
@@ -14,8 +14,16 @@
<datatype extension="fastqsolexa" type="galaxy.datatypes.sequence:FastqSolexa" display_in_upload="true" />
<datatype extension="fastqcssanger" type="galaxy.datatypes.sequence:FastqCSSanger" display_in_upload="true" />
<datatype extension="fastqillumina" type="galaxy.datatypes.sequence:FastqIllumina" display_in_upload="true" />
<datatype extension="sam" type="galaxy.datatypes.tabular:Sam" display_in_upload="true" />
<datatype extension="sam" type="galaxy.datatypes.tabular:Sam" display_in_upload="true">
<converter file="sam_to_bam_native.xml" target_datatype="bam_native"/>
<converter file="sam_to_bam.xml" target_datatype="bam"/>
<converter file="sam_to_bigwig_converter.xml" target_datatype="bigwig"/>
</datatype>
<datatype extension="bam" type="galaxy.datatypes.binary:Bam" mimetype="application/octet-stream" display_in_upload="true" description="A binary file compressed in the BGZF format with a '.bam' file extension." description_url="https://galaxyproject.org/learn/datatypes/#bam" />
<datatype extension="bam_native" type="galaxy.datatypes.binary:BamNative" mimetype="application/octet-stream" display_in_upload="true" description="A binary file compressed in the BGZF format with a '.bam' file extension." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#BAM">
<converter file="bam_to_bigwig_converter.xml" target_datatype="bigwig"/>
<converter file="bam_native_to_bam_converter.xml" target_datatype="bam"/>
</datatype>
<datatype extension="bcf" type="galaxy.datatypes.binary:Bcf" mimetype="application/octet-stream" display_in_upload="true" description="A binary file compressed in the BGZF format with a '.bcf' file extension." description_url="https://galaxyproject.org/learn/datatypes/#bcf" />
<datatype extension="biom1" type="galaxy.datatypes.text:Biom1" display_in_upload="True" subclass="True" mimetype="application/json"/>
<datatype extension="bed" type="galaxy.datatypes.interval:Bed" display_in_upload="true" description="BED format provides a flexible way to define the data lines that are displayed in an annotation track. BED lines have three required columns and nine additional optional columns. The three required columns are chrom, chromStart and chromEnd." description_url="https://galaxyproject.org/learn/datatypes/#bed">
test/functional/tools/samples_tool_conf.xml
+1
View File
@@ -83,6 +83,7 @@
<tool file="column_multi_param.xml" />
<tool file="special_params.xml" />
<tool file="section.xml" />
<tool file="sam_to_bam_native.xml" />
<tool file="top_level_data.xml" />
<tool file="validation_default.xml" />
<tool file="validation_sanitizer.xml" />