opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #7933 from nsoranzo/release_19.05_fix_vcf.gz_upload

Browse Source 
[19.05] Fix upload of gzipped VCF files
This commit is contained in:
Marius van den Beek
2019-05-12 17:49:18 +02:00
committed by GitHub
parent fb53953a87 196e000191
commit 7602bfae7e
4 changed files with 62 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/datatypes/tabular.py
+18 -2
View File
@@ -4,6 +4,7 @@ Tabular datatype
from __future__ import absolute_import
import abc
import binascii
import csv
import logging
import os
@@ -691,11 +692,11 @@ class BaseVcf(Tabular):
MetadataElement(name="viz_filter_cols", desc="Score column for visualization", default=[5], param=metadata.ColumnParameter, optional=True, multiple=True, visible=False)
MetadataElement(name="sample_names", default=[], desc="Sample names", readonly=True, visible=False, optional=True, no_value=[])
def sniff_prefix(self, file_prefix):
def _sniff(self, fname_or_file_prefix):
# Because this sniffer is run on compressed files that might be BGZF (due to the VcfGz subclass), we should
# handle unicode decode errors. This should ultimately be done in get_headers(), but guess_ext() currently
# relies on get_headers() raising this exception.
headers = get_headers(file_prefix, '\n', count=1)
headers = get_headers(fname_or_file_prefix, '\n', count=1)
return headers[0][0].startswith("##fileformat=VCF")
def display_peek(self, dataset):
@@ -745,14 +746,29 @@ class BaseVcf(Tabular):
class Vcf(BaseVcf):
file_ext = 'vcf'
def sniff_prefix(self, file_prefix):
return self._sniff(file_prefix)
class VcfGz(BaseVcf, binary.Binary):
# This class name is a misnomer, should be VcfBgzip
file_ext = 'vcf_bgzip'
compressed = True
compressed_format = "gzip"
MetadataElement(name="tabix_index", desc="Vcf Index File", param=metadata.FileParameter, file_ext="tbi", readonly=True, no_value=None, visible=False, optional=True)
def sniff(self, filename):
if not self._sniff(filename):
return False
# Check that the file is compressed with bgzip (not gzip), i.e. the
# compressed format is BGZF, as explained in
# http://samtools.github.io/hts-specs/SAMv1.pdf
with open(filename, 'rb') as fh:
fh.seek(-28, 2)
last28 = fh.read()
return binascii.hexlify(last28) == b'1f8b08040000000000ff0600424302001b0003000000000000000000'
def set_meta(self, dataset, **kwd):
super(BaseVcf, self).set_meta(dataset, **kwd)
""" Creates the index for the VCF file. """
lib/galaxy/datatypes/test/vcf_gzipped.vcf.gz
BIN
View File
Binary file not shown.
test/integration/test_datatype_upload.py
+30 -16
View File
@@ -6,12 +6,17 @@ import pytest
from base import integration_util # noqa: I100,I202
from galaxy.datatypes.registry import Registry # noqa: I201
from galaxy.util.checkers import (
is_bz2,
is_gzip,
is_zip
)
from galaxy.util.hash_util import md5_hash_file
from .test_upload_configuration_options import BaseUploadContentConfigurationInstance
SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
TEST_FILE_DIR = '%s/../../lib/galaxy/datatypes/test' % SCRIPT_DIRECTORY
TEST_DATA = collections.namedtuple('UploadDatatypesData', 'path datatype uploadable')
TestData = collections.namedtuple('UploadDatatypesData', 'path datatype uploadable')
GALAXY_ROOT = os.path.abspath('%s/../../' % SCRIPT_DIRECTORY)
DATATYPES_CONFIG = os.path.join(GALAXY_ROOT, 'config/datatypes_conf.xml.sample')
PARENT_SNIFFER_MAP = {'fastqsolexa': 'fastq'}
@@ -26,14 +31,12 @@ def find_datatype(registry, filename):
raise Exception("Couldn't guess datatype for file '%s'" % filename)
def collect_test_data():
registry = Registry()
registry.load_datatypes(root_dir=GALAXY_ROOT, config=DATATYPES_CONFIG)
def collect_test_data(registry):
test_files = os.listdir(TEST_FILE_DIR)
files = [os.path.join(TEST_FILE_DIR, f) for f in test_files]
datatypes = [find_datatype(registry, f) for f in test_files]
uploadable = [datatype.file_ext in registry.upload_file_formats for datatype in datatypes]
test_data_description = [TEST_DATA(*items) for items in zip(files, datatypes, uploadable)]
test_data_description = [TestData(*items) for items in zip(files, datatypes, uploadable)]
return {os.path.basename(data.path): data for data in test_data_description}
@@ -51,11 +54,18 @@ def temp_file():
yield fh
TEST_CASES = collect_test_data()
registry = Registry()
registry.load_datatypes(root_dir=GALAXY_ROOT, config=DATATYPES_CONFIG)
TEST_CASES = collect_test_data(registry)
@pytest.mark.parametrize('test_data', TEST_CASES.values(), ids=list(TEST_CASES.keys()))
def test_upload_datatype_auto(instance, test_data, temp_file):
is_compressed = False
for is_method in (is_bz2, is_gzip, is_zip):
is_compressed = is_method(test_data.path)
if is_compressed:
break
with open(test_data.path, 'rb') as content:
if hasattr(test_data.datatype, 'sniff') or 'false' in test_data.path:
file_type = 'auto'
@@ -75,16 +85,20 @@ def test_upload_datatype_auto(instance, test_data, temp_file):
# state should be OK
assert dataset['state'] == 'ok'
# Check that correct datatype has been detected
file_ext = dataset['file_ext']
if 'false' in test_data.path:
# datasets with false in their name are not of a specific datatype
assert dataset['file_ext'] != PARENT_SNIFFER_MAP.get(expected_file_ext, expected_file_ext)
assert file_ext != PARENT_SNIFFER_MAP.get(expected_file_ext, expected_file_ext)
else:
assert dataset['file_ext'] == PARENT_SNIFFER_MAP.get(expected_file_ext, expected_file_ext)
# download file and verify it hasn't been manipulated
temp_file.write(instance.dataset_populator.get_history_dataset_content(history_id=instance.history_id,
dataset=dataset,
type='bytes',
assert_ok=False,
raw=True))
temp_file.flush()
assert md5_hash_file(test_data.path) == md5_hash_file(temp_file.name)
assert file_ext == PARENT_SNIFFER_MAP.get(expected_file_ext, expected_file_ext)
datatype = registry.datatypes_by_extension[file_ext]
datatype_compressed = getattr(datatype, "compressed", False)
if not is_compressed or datatype_compressed:
# download file and verify it hasn't been manipulated
temp_file.write(instance.dataset_populator.get_history_dataset_content(history_id=instance.history_id,
dataset=dataset,
type='bytes',
assert_ok=False,
raw=True))
temp_file.flush()
assert md5_hash_file(test_data.path) == md5_hash_file(temp_file.name)
test/unit/datatypes/test_vcf.py
+14 -11
View File
@@ -11,19 +11,22 @@ from .util import (
def test_vcf_sniff():
vcf = Vcf()
vcf_gz = VcfGz()
with get_input_files('1.vcf_bgzip', '1.vcf') as input_files:
compressed, uncompressed = input_files
assert vcf_gz.sniff(compressed) is True
assert vcf_gz.sniff(uncompressed) is False
assert vcf.sniff(compressed) is False
assert vcf.sniff(uncompressed) is True
vcf_datatype = Vcf()
vcf_bgzip_datatype = VcfGz()
with get_input_files('1.vcf_bgzip', '1.vcf', 'vcf_gzipped.vcf.gz') as input_files:
vcf_bgzip, uncompressed, compressed = input_files
assert vcf_bgzip_datatype.sniff(vcf_bgzip) is True
assert vcf_bgzip_datatype.sniff(uncompressed) is False
assert vcf_bgzip_datatype.sniff(compressed) is False
assert vcf_datatype.sniff(vcf_bgzip) is False
assert vcf_datatype.sniff(uncompressed) is True
# Cannot sniff the vcf.gz file as vcf directly, works only when
# auto_decompress is enabled
def test_vcf_gz_set_meta():
vcf_gz = VcfGz()
def test_vcf_bgzip_set_meta():
vcf_bgzip_datatype = VcfGz()
with get_input_files('1.vcf_bgzip') as input_files, get_dataset(input_files[0], index_attr='tabix_index') as dataset:
vcf_gz.set_meta(dataset)
vcf_bgzip_datatype.set_meta(dataset)
f = pysam.VariantFile(dataset.file_name, index_filename=dataset.metadata.tabix_index.file_name)
assert isinstance(f.index, pysam.libcbcf.TabixIndex) is True