From cccf79ac5f1d4ef8dcd6d66585b5cc7966847bca Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 5 Sep 2017 18:28:08 +0200 Subject: [PATCH] Make bz2file dependency conditional on python being below version 3.3 --- lib/galaxy/datatypes/sniff.py | 8 ++++++-- lib/galaxy/dependencies/__init__.py | 4 ++++ .../dependencies/conditional-requirements.txt | 3 +++ lib/galaxy/dependencies/pinned-requirements.txt | 1 - lib/galaxy/dependencies/requirements.txt | 1 - lib/galaxy/util/checkers.py | 9 +++++++-- lib/galaxy/util/compression_utils.py | 10 +++++++--- lib/tool_shed/util/commit_util.py | 9 +++++++-- tools/data_source/upload.py | 14 +++++++------- 9 files changed, 41 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 194a279c57b..95621d6b294 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -13,7 +13,6 @@ import sys import tempfile import zipfile -import bz2file from six import text_type from galaxy import util @@ -30,6 +29,11 @@ from galaxy.util.checkers import ( is_gzip ) +if sys.version_info < (3, 3): + import bz2file as bz2 +else: + import bz2 + log = logging.getLogger(__name__) @@ -500,7 +504,7 @@ def handle_uploaded_dataset_file(filename, datatypes_registry, ext='auto', is_mu AUTO_DETECT_EXTENSIONS = ['auto'] # should 'data' also cause auto detect? -DECOMPRESSION_FUNCTIONS = dict(gzip=gzip.GzipFile, bz2=bz2file.BZ2File) +DECOMPRESSION_FUNCTIONS = dict(gzip=gzip.GzipFile, bz2=bz2.BZ2File) COMPRESSION_CHECK_FUNCTIONS = [('gzip', is_gzip), ('bz2', is_bz2)] COMPRESSION_DATATYPES = dict(gzip=['bam', 'fastq.gz', 'fastqsanger.gz', 'fastqillumina.gz', 'fastqsolexa.gz', 'fastqcssanger.gz'], bz2=['fastq.bz2', 'fastqsanger.bz2', 'fastqillumina.bz2', 'fastqsolexa.bz2', 'fastqcssanger.bz2']) COMPRESSED_EXTENSIONS = [] diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index 78bf347b9ce..d005d7fe8a9 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -1,6 +1,7 @@ """ Determine what optional dependencies are needed. """ +import sys from os.path import dirname, join from xml.etree import ElementTree @@ -112,6 +113,9 @@ class ConditionalDependencies(object): def check_kamaki(self): return 'pithos' in self.object_stores + def check_bz2file(self): + return sys.version < (3, 3) + def optional(config_file): rval = [] diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index 173a16fecb6..3066aa91482 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -16,3 +16,6 @@ python-ldap==2.4.27 # Synnefo / Pithos+ object store client kamaki + +# bz2file is not required on python >= 3.3 +bz2file diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index f2b56b3842a..417ed83b26c 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -13,7 +13,6 @@ uWSGI==2.0.15 #python_lzo==1.8 # pure Python packages -bz2file==0.98 Paste==2.0.2 PasteDeploy==1.5.2 docutils==0.12 diff --git a/lib/galaxy/dependencies/requirements.txt b/lib/galaxy/dependencies/requirements.txt index 9f3d9b9ecd2..8a3141a5f48 100644 --- a/lib/galaxy/dependencies/requirements.txt +++ b/lib/galaxy/dependencies/requirements.txt @@ -13,7 +13,6 @@ pycrypto #python_lzo # pure Python packages -bz2file Paste PasteDeploy docutils diff --git a/lib/galaxy/util/checkers.py b/lib/galaxy/util/checkers.py index 2cd81139d92..467d136d6f8 100644 --- a/lib/galaxy/util/checkers.py +++ b/lib/galaxy/util/checkers.py @@ -1,13 +1,18 @@ import gzip import re +import sys import zipfile -import bz2file from six import StringIO from galaxy import util from galaxy.util.image_util import image_type +if sys.version_info < (3, 3): + import bz2file as bz2 +else: + import bz2 + HTML_CHECK_LINES = 100 @@ -105,7 +110,7 @@ def check_bz2(file_path, check_content=True): return (True, True) CHUNK_SIZE = 2 ** 15 # reKb - bzipped_file = bz2file.BZ2File(file_path, mode='rb') + bzipped_file = bz2.BZ2File(file_path, mode='rb') chunk = bzipped_file.read(CHUNK_SIZE) bzipped_file.close() # See if we have a compressed HTML file diff --git a/lib/galaxy/util/compression_utils.py b/lib/galaxy/util/compression_utils.py index 0d5b60baaca..8d46e80f8b4 100644 --- a/lib/galaxy/util/compression_utils.py +++ b/lib/galaxy/util/compression_utils.py @@ -1,13 +1,17 @@ import gzip +import sys import zipfile -import bz2file - from .checkers import ( is_bz2, is_gzip ) +if sys.version_info < (3, 3): + import bz2file as bz2 +else: + import bz2 + def get_fileobj(filename, mode="r", gzip_only=False, bz2_only=False, zip_only=False): """ @@ -28,7 +32,7 @@ def get_fileobj(filename, mode="r", gzip_only=False, bz2_only=False, zip_only=Fa if not bz2_only and not zip_only and is_gzip(filename): return gzip.GzipFile(filename, cmode) if not gzip_only and not zip_only and is_bz2(filename): - return bz2file.BZ2File(filename, cmode) + return bz2.BZ2File(filename, cmode) if not bz2_only and not gzip_only and zipfile.is_zipfile(filename): # Return fileobj for the first file in a zip file. with zipfile.ZipFile(filename, cmode) as zh: diff --git a/lib/tool_shed/util/commit_util.py b/lib/tool_shed/util/commit_util.py index 84849199d1c..86b66009500 100644 --- a/lib/tool_shed/util/commit_util.py +++ b/lib/tool_shed/util/commit_util.py @@ -3,10 +3,10 @@ import json import logging import os import shutil +import sys import tempfile from collections import namedtuple -import bz2file from sqlalchemy.sql.expression import null import tool_shed.repository_types.util as rt_util @@ -14,6 +14,11 @@ from galaxy.util import checkers, safe_relpath from tool_shed.tools import data_table_manager from tool_shed.util import basic_util, hg_util, shed_util_common as suc +if sys.version_info < (3, 3): + import bz2file as bz2 +else: + import bz2 + log = logging.getLogger(__name__) UNDESIRABLE_DIRS = ['.hg', '.svn', '.git', '.cvs'] @@ -131,7 +136,7 @@ def handle_bz2(repository, uploaded_file_name): fd, uncompressed = tempfile.mkstemp(prefix='repo_%d_upload_bunzip2_' % repository.id, dir=os.path.dirname(uploaded_file_name), text=False) - bzipped_file = bz2file.BZ2File(uploaded_file_name, 'rb') + bzipped_file = bz2.BZ2File(uploaded_file_name, 'rb') while 1: try: chunk = bzipped_file.read(basic_util.CHUNK_SIZE) diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py index 42420f5ae3c..7993d4101d1 100644 --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -25,12 +25,12 @@ from galaxy.util.checkers import check_binary, check_bz2, check_gzip, check_html from galaxy.util.image_util import get_image_ext -try: - import bz2file -except: - bz2file = None +if sys.version_info < (3, 3): + import bz2file as bz2 +else: + import bz2 -assert sys.version_info[:2] >= (2, 4) +assert sys.version_info[:2] >= (2, 7) def stop_err(msg, ret=1): @@ -165,7 +165,7 @@ def add_file(dataset, registry, json_file, output_path): os.chmod(dataset.path, 0o644) dataset.name = dataset.name.rstrip('.gz') data_type = 'gzip' - if not data_type and bz2file is not None: + if not data_type: # See if we have a bz2 file, much like gzip is_bzipped, is_valid = check_bz2(dataset.path, check_content) if is_bzipped and not is_valid: @@ -176,7 +176,7 @@ def add_file(dataset, registry, json_file, output_path): # We need to uncompress the temp_name file CHUNK_SIZE = 2 ** 20 # 1Mb fd, uncompressed = tempfile.mkstemp(prefix='data_id_%s_upload_bunzip2_' % dataset.dataset_id, dir=os.path.dirname(output_path), text=False) - bzipped_file = bz2file.BZ2File(dataset.path, 'rb') + bzipped_file = bz2.BZ2File(dataset.path, 'rb') while 1: try: chunk = bzipped_file.read(CHUNK_SIZE)