diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 32a221b214d..7c2aca4865b 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -1960,14 +1960,30 @@ :Type: str -~~~~~~~~~~~~~~~~ -``upstream_zip`` -~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~ +``upstream_gzip`` +~~~~~~~~~~~~~~~~~ + +:Description: + If using compression in the upstream proxy server, use this option + to disable gzipping of dataset collection and library archives, + since the upstream server will do it faster on the fly. To enable + compression add ``application/zip`` to the proxy's compressable + mimetypes. +:Default: ``false`` +:Type: bool + + +~~~~~~~~~~~~~~~~~~~~ +``upstream_mod_zip`` +~~~~~~~~~~~~~~~~~~~~ :Description: If using the mod-zip module in nginx, use this option to assemble - zip archives in nginx. Requires setting up internal nginx - locations to all paths that can be archived. See + zip archives in nginx. This is preferable over the upstream_gzip + option as Galaxy does not need to serve the archive. Requires + setting up internal nginx locations to all paths that can be + archived. See https://docs.galaxyproject.org/en/master/admin/nginx.html#creating-archives-with-mod-zip for details. :Default: ``false`` diff --git a/lib/galaxy/config/config_manage.py b/lib/galaxy/config/config_manage.py index 32f25470d79..3217a09de0b 100644 --- a/lib/galaxy/config/config_manage.py +++ b/lib/galaxy/config/config_manage.py @@ -317,7 +317,6 @@ OPTION_ACTIONS = { 'communication_server_port': _DeprecatedAndDroppedAction(), 'persistent_communication_rooms': _DeprecatedAndDroppedAction(), 'legacy_eager_objectstore_initialization': _DeprecatedAndDroppedAction(), - 'upstream_gzip': _DeprecatedAndDroppedAction(), } diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index a42fa1d8047..915f05e3eed 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -1020,12 +1020,21 @@ galaxy: # files (see documentation linked above). #nginx_x_accel_redirect_base: null + # If using compression in the upstream proxy server, use this option + # to disable gzipping of dataset collection and library archives, + # since the upstream server will do it faster on the fly. To enable + # compression add ``application/zip`` to the proxy's compressable + # mimetypes. + #upstream_gzip: false + # If using the mod-zip module in nginx, use this option to assemble - # zip archives in nginx. Requires setting up internal nginx locations - # to all paths that can be archived. See + # zip archives in nginx. This is preferable over the upstream_gzip + # option as Galaxy does not need to serve the archive. Requires + # setting up internal nginx locations to all paths that can be + # archived. See # https://docs.galaxyproject.org/en/master/admin/nginx.html#creating-archives-with-mod-zip # for details. - #upstream_zip: false + #upstream_mod_zip: false # The following default adds a header to web request responses that # will cause modern web browsers to not allow Galaxy to be embedded in diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index e4d2454db3f..d4db30d926e 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -275,7 +275,11 @@ class Data(metaclass=DataMeta): # save a composite object into a compressed archive for downloading outfname = data.name[0:150] outfname = ''.join(c in FILENAME_VALID_CHARS and c or '_' for c in outfname) - archive = ZipstreamWrapper(upstream_zip=trans.app.config.upstream_zip, archive_name=outfname) + archive = ZipstreamWrapper( + archive_name=outfname, + upstream_mod_zip=trans.app.config.upstream_mod_zip, + upstream_gzip=trans.app.config.upstream_gzip + ) error = False msg = '' ext = data.extension diff --git a/lib/galaxy/managers/hdcas.py b/lib/galaxy/managers/hdcas.py index 57f6dfe616d..5f003549c90 100644 --- a/lib/galaxy/managers/hdcas.py +++ b/lib/galaxy/managers/hdcas.py @@ -22,9 +22,13 @@ from galaxy.util.zipstream import ZipstreamWrapper log = logging.getLogger(__name__) -def stream_dataset_collection(dataset_collection_instance, upstream_zip=False): +def stream_dataset_collection(dataset_collection_instance, upstream_mod_zip=False, upstream_gzip=False): archive_name = f"{dataset_collection_instance.hid}: {dataset_collection_instance.name}" - archive = ZipstreamWrapper(upstream_zip=upstream_zip, archive_name=archive_name) + archive = ZipstreamWrapper( + archive_name=archive_name, + upstream_mod_zip=upstream_mod_zip, + upstream_gzip=upstream_gzip, + ) names, hdas = get_hda_and_element_identifiers(dataset_collection_instance) for name, hda in zip(names, hdas): if hda.state != hda.states.OK: diff --git a/lib/galaxy/util/zipstream.py b/lib/galaxy/util/zipstream.py index 122d5e2751e..f6d70650868 100644 --- a/lib/galaxy/util/zipstream.py +++ b/lib/galaxy/util/zipstream.py @@ -8,16 +8,16 @@ from .path import safe_walk class ZipstreamWrapper: - def __init__(self, upstream_zip=False, archive_name=None): - self.upstream_zip = upstream_zip + def __init__(self, archive_name=None, upstream_mod_zip=False, upstream_gzip=False): + self.upstream_mod_zip = upstream_mod_zip self.archive_name = archive_name - if not self.upstream_zip: - self.archive = zipstream.ZipFile(allowZip64=True, compression=zipstream.ZIP_DEFLATED) + if not self.upstream_mod_zip: + self.archive = zipstream.ZipFile(allowZip64=True, compression=zipstream.ZIP_STORED if upstream_gzip else zipstream.ZIP_DEFLATED) self.files = [] self.size = 0 def response(self): - if self.upstream_zip: + if self.upstream_mod_zip: yield "\n".join(self.files).encode() else: yield iter(self.archive) @@ -26,7 +26,7 @@ class ZipstreamWrapper: headers = {} if self.archive_name: headers['Content-Disposition'] = f'attachment; filename="{self.archive_name}.zip"' - if self.upstream_zip: + if self.upstream_mod_zip: headers['X-Archive-Files'] = 'zip' else: headers['Content-Type'] = 'application/x-zip-compressed' @@ -34,7 +34,7 @@ class ZipstreamWrapper: def add_path(self, path, archive_name): size = int(os.stat(path).st_size) - if self.upstream_zip: + if self.upstream_mod_zip: # calculating crc32 would defeat the point of using mod-zip, but if we ever calculate hashsums we should consider this crc32 = "-" line = f"{crc32} {size} {quote(path)} {archive_name}" diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py index e1437e01c69..f81f8f35bcb 100644 --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -296,7 +296,7 @@ class HistoryContentsController(BaseAPIController, UsesLibraryMixin, UsesLibrary return {'error': util.unicodify(e)} def __stream_dataset_collection(self, trans, dataset_collection_instance): - archive = hdcas.stream_dataset_collection(dataset_collection_instance=dataset_collection_instance, upstream_zip=trans.app.config.upstream_zip) + archive = hdcas.stream_dataset_collection(dataset_collection_instance=dataset_collection_instance, upstream_mod_zip=trans.app.config.upstream_mod_zip) trans.response.headers.update(archive.get_headers()) return archive.response() @@ -1019,8 +1019,11 @@ class HistoryContentsController(BaseAPIController, UsesLibraryMixin, UsesLibrary return safe_dumps(paths_and_files) # create the archive, add the dataset files, then stream the archive as a download - archive = ZipstreamWrapper(upstream_zip=self.app.config.upstream_zip, archive_name=archive_base_name) - + archive = ZipstreamWrapper( + archive_name=archive_base_name, + upstream_mod_zip=self.app.config.upstream_mod_zip, + upstream_gzip=self.app.config.upstream_gzip, + ) for file_path, archive_path in paths_and_files: archive.write(file_path, archive_path) diff --git a/lib/galaxy/webapps/galaxy/api/library_datasets.py b/lib/galaxy/webapps/galaxy/api/library_datasets.py index 9211550d3bf..50d4fe1d050 100644 --- a/lib/galaxy/webapps/galaxy/api/library_datasets.py +++ b/lib/galaxy/webapps/galaxy/api/library_datasets.py @@ -583,7 +583,11 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin, Libra raise exceptions.RequestParameterMissingException('Request has to contain a list of dataset ids or folder ids to download.') if archive_format == 'zip': - archive = ZipstreamWrapper(upstream_zip=self.app.config.upstream_zip, archive_name="selected_library_files") + archive = ZipstreamWrapper( + archive_name="selected_library_files", + upstream_mod_zip=self.app.config.upstream_mod_zip, + upstream_gzip=self.app.config.upstream_gzip, + ) killme = string.punctuation + string.whitespace trantab = str.maketrans(killme, '_' * len(killme)) seen = [] diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index 533d877a59c..53130252f2c 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -156,8 +156,12 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin): trans.response.headers["Content-Disposition"] = f'attachment; filename="{filename}"' return open(path, mode='rb') elif os.path.isdir(path): - # Set upstream_zip to false, otherwise tool data must be among allowed internal routes - archive = ZipstreamWrapper(upstream_zip=False, archive_name=filename) + # Set upstream_mod_zip to false, otherwise tool data must be among allowed internal routes + archive = ZipstreamWrapper( + upstream_mod_zip=False, + upstream_gzip=self.app.config.upstream_gzip, + archive_name=filename, + ) archive.write(path) trans.response.headers.update(archive.get_headers()) return archive.response() diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index de4e1e7113c..05fb0225b2b 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -1424,14 +1424,25 @@ mapping: should be set to the path defined in the nginx config as an internal redirect with access to Galaxy's data files (see documentation linked above). - upstream_zip: + upstream_gzip: + type: bool + default: false + required: false + desc: | + If using compression in the upstream proxy server, use this option to disable + gzipping of dataset collection and library archives, since the upstream server + will do it faster on the fly. To enable compression add ``application/zip`` + to the proxy's compressable mimetypes. + + upstream_mod_zip: type: bool default: false required: false desc: | If using the mod-zip module in nginx, use this option to assemble - zip archives in nginx. Requires setting up internal nginx locations - to all paths that can be archived. + zip archives in nginx. This is preferable over the upstream_gzip option + as Galaxy does not need to serve the archive. + Requires setting up internal nginx locations to all paths that can be archived. See https://docs.galaxyproject.org/en/master/admin/nginx.html#creating-archives-with-mod-zip for details.