Allow working with pbzip2 compressed files

This adds supprt for uploading and uncompressing pbzip2 compressed
files by exchanging the bz2 module with bz2file (recommended as workaround in
https://docs.python.org/2/library/bz2.html). The problem was reported by
@alpapan in #4538 and this commit should fix #4538.

Depends on https://github.com/galaxyproject/starforge/pull/147 for the
bz2file wheel.

lib/galaxy/datatypes/sniff.py

@@ -3,7 +3,6 @@ File format detector
 """
 from __future__ import absolute_import
 
-import bz2
 import codecs
 import gzip
 import logging
@@ -14,6 +13,7 @@ import sys
 import tempfile
 import zipfile
 
+import bz2file
 from six import text_type
 
 from galaxy import util
@@ -500,7 +500,7 @@ def handle_uploaded_dataset_file(filename, datatypes_registry, ext='auto', is_mu
 
 
 AUTO_DETECT_EXTENSIONS = ['auto']  # should 'data' also cause auto detect?
-DECOMPRESSION_FUNCTIONS = dict(gzip=gzip.GzipFile, bz2=bz2.BZ2File)
+DECOMPRESSION_FUNCTIONS = dict(gzip=gzip.GzipFile, bz2=bz2file.BZ2File)
 COMPRESSION_CHECK_FUNCTIONS = [('gzip', is_gzip), ('bz2', is_bz2)]
 COMPRESSION_DATATYPES = dict(gzip=['bam', 'fastq.gz', 'fastqsanger.gz', 'fastqillumina.gz', 'fastqsolexa.gz', 'fastqcssanger.gz'], bz2=['fastq.bz2', 'fastqsanger.bz2', 'fastqillumina.bz2', 'fastqsolexa.bz2', 'fastqcssanger.bz2'])
 COMPRESSED_EXTENSIONS = []

lib/galaxy/util/checkers.py

@@ -1,8 +1,8 @@
-import bz2
 import gzip
 import re
 import zipfile
 
+import bz2file
 from six import StringIO
 
 from galaxy import util
@@ -105,7 +105,7 @@ def check_bz2(file_path, check_content=True):
         return (True, True)
 
     CHUNK_SIZE = 2 ** 15  # reKb
-    bzipped_file = bz2.BZ2File(file_path, mode='rb')
+    bzipped_file = bz2file.BZ2File(file_path, mode='rb')
     chunk = bzipped_file.read(CHUNK_SIZE)
     bzipped_file.close()
     # See if we have a compressed HTML file

lib/galaxy/util/compression_utils.py

@@ -1,7 +1,8 @@
-import bz2
 import gzip
 import zipfile
 
+import bz2file
+
 from .checkers import (
     is_bz2,
     is_gzip
@@ -27,7 +28,7 @@ def get_fileobj(filename, mode="r", gzip_only=False, bz2_only=False, zip_only=Fa
     if not bz2_only and not zip_only and is_gzip(filename):
         return gzip.GzipFile(filename, cmode)
     if not gzip_only and not zip_only and is_bz2(filename):
-        return bz2.BZ2File(filename, cmode)
+        return bz2file.BZ2File(filename, cmode)
     if not bz2_only and not gzip_only and zipfile.is_zipfile(filename):
         # Return fileobj for the first file in a zip file.
         with zipfile.ZipFile(filename, cmode) as zh:

lib/tool_shed/util/commit_util.py

@@ -1,4 +1,3 @@
-import bz2
 import gzip
 import json
 import logging
@@ -7,6 +6,7 @@ import shutil
 import tempfile
 from collections import namedtuple
 
+import bz2file
 from sqlalchemy.sql.expression import null
 
 import tool_shed.repository_types.util as rt_util
@@ -131,7 +131,7 @@ def handle_bz2(repository, uploaded_file_name):
     fd, uncompressed = tempfile.mkstemp(prefix='repo_%d_upload_bunzip2_' % repository.id,
                                         dir=os.path.dirname(uploaded_file_name),
                                         text=False)
-    bzipped_file = bz2.BZ2File(uploaded_file_name, 'rb')
+    bzipped_file = bz2file.BZ2File(uploaded_file_name, 'rb')
     while 1:
         try:
             chunk = bzipped_file.read(basic_util.CHUNK_SIZE)

tools/data_source/upload.py

@@ -26,9 +26,9 @@ from galaxy.util.image_util import get_image_ext
 
 
 try:
-    import bz2
+    import bz2file
 except:
-    bz2 = None
+    bz2file = None
 
 assert sys.version_info[:2] >= (2, 4)
 
@@ -165,7 +165,7 @@ def add_file(dataset, registry, json_file, output_path):
                     os.chmod(dataset.path, 0o644)
                 dataset.name = dataset.name.rstrip('.gz')
                 data_type = 'gzip'
-            if not data_type and bz2 is not None:
+            if not data_type and bz2file is not None:
                 # See if we have a bz2 file, much like gzip
                 is_bzipped, is_valid = check_bz2(dataset.path, check_content)
                 if is_bzipped and not is_valid:
@@ -176,7 +176,7 @@ def add_file(dataset, registry, json_file, output_path):
                         # We need to uncompress the temp_name file
                         CHUNK_SIZE = 2 ** 20  # 1Mb
                         fd, uncompressed = tempfile.mkstemp(prefix='data_id_%s_upload_bunzip2_' % dataset.dataset_id, dir=os.path.dirname(output_path), text=False)
-                        bzipped_file = bz2.BZ2File(dataset.path, 'rb')
+                        bzipped_file = bz2file.BZ2File(dataset.path, 'rb')
                         while 1:
                             try:
                                 chunk = bzipped_file.read(CHUNK_SIZE)