diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 653f3c635aa..a437a6d24df 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -38,6 +38,7 @@ from six.moves.urllib import ( parse as urlparse, request as urlrequest ) +from six.moves.urllib.request import urlopen try: import docutils.core as docutils_core @@ -1481,6 +1482,17 @@ def url_get( base_url, password_mgr=None, pathspec=None, params=None ): return content +def download_to_file(url, dest_file_path, timeout=30, chunk_size=2 ** 20): + """Download a URL to a file in chunks.""" + src = urlopen(url, timeout=timeout) + with open(dest_file_path, 'wb') as f: + while True: + chunk = src.read(chunk_size) + if not chunk: + break + f.write(chunk) + + def safe_relpath(path): """ Given what we expect to be a relative path, determine whether the path diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py index 938048f4122..3548b65249c 100644 --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py @@ -6,15 +6,16 @@ import shutil import stat import tarfile import tempfile -import time -import urllib2 import zipfile from string import Template # TODO: eliminate the use of fabric here. from fabric.api import lcd, settings -from galaxy.util import asbool +from galaxy.util import ( + asbool, + download_to_file +) from galaxy.util.template import fill_template from tool_shed.galaxy_install.tool_dependencies.env_manager import EnvManager from tool_shed.util import basic_util, tool_dependency_util @@ -157,33 +158,11 @@ class Download( object ): If the checksum does not match an exception is thrown. """ file_path = os.path.join( install_dir, downloaded_file_name ) - src = None - dst = None - - # Set a timer so we don't sit here forever. - start_time = time.time() try: - src = urllib2.urlopen( download_url ) - dst = open( file_path, 'wb' ) - while True: - chunk = src.read( basic_util.CHUNK_SIZE ) - if chunk: - dst.write( chunk ) - else: - break - time_taken = time.time() - start_time - if time_taken > basic_util.NO_OUTPUT_TIMEOUT: - err_msg = 'Downloading from URL %s took longer than the defined timeout period of %.1f seconds.' % \ - ( str( download_url ), basic_util.NO_OUTPUT_TIMEOUT ) - raise Exception( err_msg ) + download_to_file( download_url, file_path, chunk_size=basic_util.CHUNK_SIZE) except Exception as e: - err_msg = err_msg = 'Error downloading from URL\n%s:\n%s' % ( str( download_url ), str( e ) ) + err_msg = 'Error downloading from URL %s : %s' % ( str( download_url ), str( e ) ) raise Exception( err_msg ) - finally: - if src: - src.close() - if dst: - dst.close() if 'sha256sum' in checksums or '#sha256#' in download_url: downloaded_checksum = hashlib.sha256(open(file_path, 'rb').read()).hexdigest().lower() @@ -228,7 +207,7 @@ class Download( object ): return rval def get_dict_checksums( self, dct ): - return dict(filter(lambda i: i[0] in ['md5sum', 'sha256sum'], dct.iteritems())) + return dict(i for i in dct.items() if i[0] in ['md5sum', 'sha256sum']) class RecipeStep( object ): diff --git a/test/base/driver_util.py b/test/base/driver_util.py index 0b2e8b5585b..974abc691a8 100644 --- a/test/base/driver_util.py +++ b/test/base/driver_util.py @@ -17,12 +17,11 @@ import nose.config import nose.core import nose.loader import nose.plugins.manager -import requests from paste import httpserver from functional import database_contexts from galaxy.app import UniverseApplication as GalaxyUniverseApplication -from galaxy.util import asbool +from galaxy.util import asbool, download_to_file from galaxy.util.properties import load_app_properties from galaxy.web import buildapp from galaxy.webapps.tool_shed.app import UniverseApplication as ToolshedUniverseApplication @@ -254,9 +253,7 @@ def copy_database_template( source, db_path ): shutil.copy(source, db_path) assert os.path.exists(db_path) elif source.lower().startswith(("http://", "https://", "ftp://")): - r = requests.get(source) - with open(db_path, 'w') as f: - f.write(r.content) + download_to_file(source, db_path) else: raise Exception( "Failed to copy database template from source %s" % source )