From 288b44391ba43d85d9dc4365d5b0aabebcd92c4f Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 5 Aug 2020 18:10:50 -0400 Subject: [PATCH 1/4] Use archive_format instead of 'format', since that's https://docs.python.org/3/library/functions.html\#format --- .../webapps/galaxy/api/library_datasets.py | 36 +++++++++---------- lib/galaxy/webapps/galaxy/buildapp.py | 2 +- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/library_datasets.py b/lib/galaxy/webapps/galaxy/api/library_datasets.py index 041d68b38a6..8d0f1866fbe 100644 --- a/lib/galaxy/webapps/galaxy/api/library_datasets.py +++ b/lib/galaxy/webapps/galaxy/api/library_datasets.py @@ -512,19 +512,19 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin, Libra @web.expose # TODO convert to expose_api - def download(self, trans, format, **kwd): + def download(self, trans, archive_format, **kwd): """ - GET /api/libraries/datasets/download/{format} - POST /api/libraries/datasets/download/{format} + GET /api/libraries/datasets/download/{archive_format} + POST /api/libraries/datasets/download/{archive_format} - Download requested datasets (identified by encoded IDs) in requested format. + Download requested datasets (identified by encoded IDs) in requested archive_format. example: ``GET localhost:8080/api/libraries/datasets/download/tbz?ld_ids%255B%255D=a0d84b45643a2678&ld_ids%255B%255D=fe38c84dcd46c828`` - .. note:: supported format values are: 'zip', 'tgz', 'tbz', 'uncompressed' + .. note:: supported archive_format values are: 'zip', 'tgz', 'tbz', 'uncompressed' - :param format: string representing requested archive format - :type format: string + :param archive_format: string representing requested archive archive_format + :type archive_format: string :param ld_ids[]: an array of encoded dataset ids :type ld_ids[]: an array :param folder_ids[]: an array of encoded folder ids @@ -586,30 +586,30 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin, Libra if not library_datasets: raise exceptions.RequestParameterMissingException('Request has to contain a list of dataset ids or folder ids to download.') - if format in ['zip', 'tgz', 'tbz']: + if archive_format in ['zip', 'tgz', 'tbz']: # error = False killme = string.punctuation + string.whitespace trantab = maketrans(killme, '_' * len(killme)) try: outext = 'zip' - if format == 'zip': + if archive_format == 'zip': # Can't use mkstemp - the file must not exist first tmpd = tempfile.mkdtemp() util.umask_fix_perms(tmpd, trans.app.config.umask, 0o777, self.app.config.gid) - tmpf = os.path.join(tmpd, 'library_download.' + format) + tmpf = os.path.join(tmpd, 'library_download.' + archive_format) if trans.app.config.upstream_gzip: archive = zipfile.ZipFile(tmpf, 'w', zipfile.ZIP_STORED, True) else: archive = zipfile.ZipFile(tmpf, 'w', zipfile.ZIP_DEFLATED, True) archive.add = lambda x, y: archive.write(x, y.encode('CP437')) - elif format == 'tgz': + elif archive_format == 'tgz': if trans.app.config.upstream_gzip: archive = StreamBall('w|') outext = 'tar' else: archive = StreamBall('w|gz') outext = 'tgz' - elif format == 'tbz': + elif archive_format == 'tbz': archive = StreamBall('w|bz2') outext = 'tbz2' except (OSError, zipfile.BadZipfile): @@ -646,7 +646,7 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin, Libra if zpathext == '': zpath = '%s.html' % zpath # fake the real nature of the html file try: - if format == 'zip': + if archive_format == 'zip': archive.add(ldda.dataset.file_name, zpath) # add the primary of a composite set else: archive.add(ldda.dataset.file_name, zpath, check_file=True) # add the primary of a composite set @@ -666,7 +666,7 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin, Libra if fname > '': fname = fname.translate(trantab) try: - if format == 'zip': + if archive_format == 'zip': archive.add(fpath, fname) else: archive.add(fpath, fname, check_file=True) @@ -681,7 +681,7 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin, Libra raise exceptions.InternalServerError("Unable to add dataset to temporary library download archive . " + util.unicodify(e)) else: try: - if format == 'zip': + if archive_format == 'zip': archive.add(ldda.dataset.file_name, path) else: archive.add(ldda.dataset.file_name, path, check_file=True) @@ -696,7 +696,7 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin, Libra raise exceptions.InternalServerError("Unknown error. " + util.unicodify(e)) lname = 'selected_dataset' fname = lname.replace(' ', '_') + '_files' - if format == 'zip': + if archive_format == 'zip': archive.close() trans.response.set_content_type("application/octet-stream") trans.response.headers["Content-Disposition"] = 'attachment; filename="%s.%s"' % (fname, outext) @@ -710,7 +710,7 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin, Libra archive.wsgi_status = trans.response.wsgi_status() archive.wsgi_headeritems = trans.response.wsgi_headeritems() return archive.stream - elif format == 'uncompressed': + elif archive_format == 'uncompressed': if len(library_datasets) != 1: raise exceptions.RequestParameterInvalidException("You can download only one uncompressed file at once.") else: @@ -728,4 +728,4 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin, Libra except Exception: raise exceptions.InternalServerError("This dataset contains no content.") else: - raise exceptions.RequestParameterInvalidException("Wrong format parameter specified") + raise exceptions.RequestParameterInvalidException("Wrong archive_format parameter specified") diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 69cbc8a274c..f1cb8de405f 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -869,7 +869,7 @@ def populate_api_routes(webapp, app): conditions=dict(method=["DELETE"])) webapp.mapper.connect('download_ld_items', - '/api/libraries/datasets/download/{format}', + '/api/libraries/datasets/download/{archive_format}', controller='library_datasets', action='download', conditions=dict(method=["POST", "GET"])) From 867ce47e983a409824f75204ac82c70aedf161fb Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 5 Aug 2020 18:30:53 -0400 Subject: [PATCH 2/4] Update maketrans usage --- lib/galaxy/webapps/galaxy/api/library_datasets.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/library_datasets.py b/lib/galaxy/webapps/galaxy/api/library_datasets.py index 8d0f1866fbe..1c06dc991f7 100644 --- a/lib/galaxy/webapps/galaxy/api/library_datasets.py +++ b/lib/galaxy/webapps/galaxy/api/library_datasets.py @@ -42,11 +42,6 @@ from galaxy.webapps.base.controller import ( UsesVisualizationMixin, ) -try: - maketrans = str.maketrans -except AttributeError: - from string import maketrans - log = logging.getLogger(__name__) @@ -589,7 +584,7 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin, Libra if archive_format in ['zip', 'tgz', 'tbz']: # error = False killme = string.punctuation + string.whitespace - trantab = maketrans(killme, '_' * len(killme)) + trantab = str.maketrans(killme, '_' * len(killme)) try: outext = 'zip' if archive_format == 'zip': From 1914001c75abd4c554d0d80a4feef1100fe69802 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 5 Aug 2020 18:31:38 -0400 Subject: [PATCH 3/4] streamball.ZipBall tweaks --- lib/galaxy/util/streamball.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/util/streamball.py b/lib/galaxy/util/streamball.py index 20806319f2f..b6ffcce97fe 100644 --- a/lib/galaxy/util/streamball.py +++ b/lib/galaxy/util/streamball.py @@ -48,12 +48,13 @@ class ZipBall(object): def __init__(self, tmpf, tmpd): self._tmpf = tmpf self._tmpd = tmpd + self.wsgi_status = None + self.wsgi_headeritems = None def stream(self, environ, start_response): response_write = start_response(self.wsgi_status, self.wsgi_headeritems) - tmpfh = open(self._tmpf) - response_write(tmpfh.read()) - tmpfh.close() + with open(self._tmpf, 'rb') as tmpfh: + response_write(tmpfh.read()) try: os.unlink(self._tmpf) os.rmdir(self._tmpd) From 152d1daae35c09ab38ce774cbc6d279b77152cf2 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 5 Aug 2020 18:38:03 -0400 Subject: [PATCH 4/4] Python3 compat fix for zipfile creation. xref https://bugs.python.org/issue10757, https://bugs.python.org/issue24110 --- lib/galaxy/webapps/galaxy/api/library_datasets.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/library_datasets.py b/lib/galaxy/webapps/galaxy/api/library_datasets.py index 1c06dc991f7..890f64bfbeb 100644 --- a/lib/galaxy/webapps/galaxy/api/library_datasets.py +++ b/lib/galaxy/webapps/galaxy/api/library_datasets.py @@ -596,7 +596,18 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin, Libra archive = zipfile.ZipFile(tmpf, 'w', zipfile.ZIP_STORED, True) else: archive = zipfile.ZipFile(tmpf, 'w', zipfile.ZIP_DEFLATED, True) - archive.add = lambda x, y: archive.write(x, y.encode('CP437')) + + def zipfile_add(fpath, arcname): + encoded_arcname = arcname.encode('CP437') + try: + archive.write(fpath, encoded_arcname) + except TypeError: + # Despite documenting the need for CP437 encoded arcname, + # python 3 actually needs this to be a unicode string ... + # https://bugs.python.org/issue24110 + archive.write(fpath, arcname) + archive.add = zipfile_add + elif archive_format == 'tgz': if trans.app.config.upstream_gzip: archive = StreamBall('w|')