From 46c85ffa986ec2211bbda7be4f90ab579d3fff69 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 5 Sep 2017 16:54:34 +0200 Subject: [PATCH] Allow working with pbzip2 compressed files This adds supprt for uploading and uncompressing pbzip2 compressed files by exchanging the bz2 module with bz2file (recommended as workaround in https://docs.python.org/2/library/bz2.html). The problem was reported by @alpapan in #4538 and this commit should fix #4538. Depends on https://github.com/galaxyproject/starforge/pull/147 for the bz2file wheel. --- lib/galaxy/datatypes/sniff.py | 4 ++-- lib/galaxy/util/checkers.py | 4 ++-- lib/galaxy/util/compression_utils.py | 5 +++-- lib/tool_shed/util/commit_util.py | 4 ++-- tools/data_source/upload.py | 8 ++++---- 5 files changed, 13 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 10222f6a7d9..194a279c57b 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -3,7 +3,6 @@ File format detector """ from __future__ import absolute_import -import bz2 import codecs import gzip import logging @@ -14,6 +13,7 @@ import sys import tempfile import zipfile +import bz2file from six import text_type from galaxy import util @@ -500,7 +500,7 @@ def handle_uploaded_dataset_file(filename, datatypes_registry, ext='auto', is_mu AUTO_DETECT_EXTENSIONS = ['auto'] # should 'data' also cause auto detect? -DECOMPRESSION_FUNCTIONS = dict(gzip=gzip.GzipFile, bz2=bz2.BZ2File) +DECOMPRESSION_FUNCTIONS = dict(gzip=gzip.GzipFile, bz2=bz2file.BZ2File) COMPRESSION_CHECK_FUNCTIONS = [('gzip', is_gzip), ('bz2', is_bz2)] COMPRESSION_DATATYPES = dict(gzip=['bam', 'fastq.gz', 'fastqsanger.gz', 'fastqillumina.gz', 'fastqsolexa.gz', 'fastqcssanger.gz'], bz2=['fastq.bz2', 'fastqsanger.bz2', 'fastqillumina.bz2', 'fastqsolexa.bz2', 'fastqcssanger.bz2']) COMPRESSED_EXTENSIONS = [] diff --git a/lib/galaxy/util/checkers.py b/lib/galaxy/util/checkers.py index ca22ad97220..2cd81139d92 100644 --- a/lib/galaxy/util/checkers.py +++ b/lib/galaxy/util/checkers.py @@ -1,8 +1,8 @@ -import bz2 import gzip import re import zipfile +import bz2file from six import StringIO from galaxy import util @@ -105,7 +105,7 @@ def check_bz2(file_path, check_content=True): return (True, True) CHUNK_SIZE = 2 ** 15 # reKb - bzipped_file = bz2.BZ2File(file_path, mode='rb') + bzipped_file = bz2file.BZ2File(file_path, mode='rb') chunk = bzipped_file.read(CHUNK_SIZE) bzipped_file.close() # See if we have a compressed HTML file diff --git a/lib/galaxy/util/compression_utils.py b/lib/galaxy/util/compression_utils.py index 6a94c543196..0d5b60baaca 100644 --- a/lib/galaxy/util/compression_utils.py +++ b/lib/galaxy/util/compression_utils.py @@ -1,7 +1,8 @@ -import bz2 import gzip import zipfile +import bz2file + from .checkers import ( is_bz2, is_gzip @@ -27,7 +28,7 @@ def get_fileobj(filename, mode="r", gzip_only=False, bz2_only=False, zip_only=Fa if not bz2_only and not zip_only and is_gzip(filename): return gzip.GzipFile(filename, cmode) if not gzip_only and not zip_only and is_bz2(filename): - return bz2.BZ2File(filename, cmode) + return bz2file.BZ2File(filename, cmode) if not bz2_only and not gzip_only and zipfile.is_zipfile(filename): # Return fileobj for the first file in a zip file. with zipfile.ZipFile(filename, cmode) as zh: diff --git a/lib/tool_shed/util/commit_util.py b/lib/tool_shed/util/commit_util.py index 5be6825d679..84849199d1c 100644 --- a/lib/tool_shed/util/commit_util.py +++ b/lib/tool_shed/util/commit_util.py @@ -1,4 +1,3 @@ -import bz2 import gzip import json import logging @@ -7,6 +6,7 @@ import shutil import tempfile from collections import namedtuple +import bz2file from sqlalchemy.sql.expression import null import tool_shed.repository_types.util as rt_util @@ -131,7 +131,7 @@ def handle_bz2(repository, uploaded_file_name): fd, uncompressed = tempfile.mkstemp(prefix='repo_%d_upload_bunzip2_' % repository.id, dir=os.path.dirname(uploaded_file_name), text=False) - bzipped_file = bz2.BZ2File(uploaded_file_name, 'rb') + bzipped_file = bz2file.BZ2File(uploaded_file_name, 'rb') while 1: try: chunk = bzipped_file.read(basic_util.CHUNK_SIZE) diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py index b47892afd3e..42420f5ae3c 100644 --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -26,9 +26,9 @@ from galaxy.util.image_util import get_image_ext try: - import bz2 + import bz2file except: - bz2 = None + bz2file = None assert sys.version_info[:2] >= (2, 4) @@ -165,7 +165,7 @@ def add_file(dataset, registry, json_file, output_path): os.chmod(dataset.path, 0o644) dataset.name = dataset.name.rstrip('.gz') data_type = 'gzip' - if not data_type and bz2 is not None: + if not data_type and bz2file is not None: # See if we have a bz2 file, much like gzip is_bzipped, is_valid = check_bz2(dataset.path, check_content) if is_bzipped and not is_valid: @@ -176,7 +176,7 @@ def add_file(dataset, registry, json_file, output_path): # We need to uncompress the temp_name file CHUNK_SIZE = 2 ** 20 # 1Mb fd, uncompressed = tempfile.mkstemp(prefix='data_id_%s_upload_bunzip2_' % dataset.dataset_id, dir=os.path.dirname(output_path), text=False) - bzipped_file = bz2.BZ2File(dataset.path, 'rb') + bzipped_file = bz2file.BZ2File(dataset.path, 'rb') while 1: try: chunk = bzipped_file.read(CHUNK_SIZE)