Restore upstream_gzip option, rename upstream_zip to upstream_mod_zip

doc/source/admin/galaxy_options.rst

@@ -1960,14 +1960,30 @@
 :Type: str
 
 
-~~~~~~~~~~~~~~~~
-``upstream_zip``
-~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~
+``upstream_gzip``
+~~~~~~~~~~~~~~~~~
+
+:Description:
+    If using compression in the upstream proxy server, use this option
+    to disable gzipping of dataset collection and library archives,
+    since the upstream server will do it faster on the fly. To enable
+    compression add ``application/zip`` to the proxy's compressable
+    mimetypes.
+:Default: ``false``
+:Type: bool
+
+
+~~~~~~~~~~~~~~~~~~~~
+``upstream_mod_zip``
+~~~~~~~~~~~~~~~~~~~~
 
 :Description:
     If using the mod-zip module in nginx, use this option to assemble
-    zip archives in nginx. Requires setting up internal nginx
-    locations to all paths that can be archived. See
+    zip archives in nginx. This is preferable over the upstream_gzip
+    option as Galaxy does not need to serve the archive. Requires
+    setting up internal nginx locations to all paths that can be
+    archived. See
     https://docs.galaxyproject.org/en/master/admin/nginx.html#creating-archives-with-mod-zip
     for details.
 :Default: ``false``

lib/galaxy/config/config_manage.py

@@ -317,7 +317,6 @@ OPTION_ACTIONS = {
     'communication_server_port': _DeprecatedAndDroppedAction(),
     'persistent_communication_rooms': _DeprecatedAndDroppedAction(),
     'legacy_eager_objectstore_initialization': _DeprecatedAndDroppedAction(),
-    'upstream_gzip': _DeprecatedAndDroppedAction(),
 }
 
 

lib/galaxy/config/sample/galaxy.yml.sample

@@ -1020,12 +1020,21 @@ galaxy:
   # files (see documentation linked above).
   #nginx_x_accel_redirect_base: null
 
+  # If using compression in the upstream proxy server, use this option
+  # to disable gzipping of dataset collection and library archives,
+  # since the upstream server will do it faster on the fly. To enable
+  # compression add ``application/zip`` to the proxy's compressable
+  # mimetypes.
+  #upstream_gzip: false
+
   # If using the mod-zip module in nginx, use this option to assemble
-  # zip archives in nginx. Requires setting up internal nginx locations
-  # to all paths that can be archived. See
+  # zip archives in nginx. This is preferable over the upstream_gzip
+  # option as Galaxy does not need to serve the archive. Requires
+  # setting up internal nginx locations to all paths that can be
+  # archived. See
   # https://docs.galaxyproject.org/en/master/admin/nginx.html#creating-archives-with-mod-zip
   # for details.
-  #upstream_zip: false
+  #upstream_mod_zip: false
 
   # The following default adds a header to web request responses that
   # will cause modern web browsers to not allow Galaxy to be embedded in

lib/galaxy/datatypes/data.py

@@ -275,7 +275,11 @@ class Data(metaclass=DataMeta):
         # save a composite object into a compressed archive for downloading
         outfname = data.name[0:150]
         outfname = ''.join(c in FILENAME_VALID_CHARS and c or '_' for c in outfname)
-        archive = ZipstreamWrapper(upstream_zip=trans.app.config.upstream_zip, archive_name=outfname)
+        archive = ZipstreamWrapper(
+            archive_name=outfname,
+            upstream_mod_zip=trans.app.config.upstream_mod_zip,
+            upstream_gzip=trans.app.config.upstream_gzip
+        )
         error = False
         msg = ''
         ext = data.extension

lib/galaxy/managers/hdcas.py

@@ -22,9 +22,13 @@ from galaxy.util.zipstream import ZipstreamWrapper
 log = logging.getLogger(__name__)
 
 
-def stream_dataset_collection(dataset_collection_instance, upstream_zip=False):
+def stream_dataset_collection(dataset_collection_instance, upstream_mod_zip=False, upstream_gzip=False):
     archive_name = f"{dataset_collection_instance.hid}: {dataset_collection_instance.name}"
-    archive = ZipstreamWrapper(upstream_zip=upstream_zip, archive_name=archive_name)
+    archive = ZipstreamWrapper(
+        archive_name=archive_name,
+        upstream_mod_zip=upstream_mod_zip,
+        upstream_gzip=upstream_gzip,
+    )
     names, hdas = get_hda_and_element_identifiers(dataset_collection_instance)
     for name, hda in zip(names, hdas):
         if hda.state != hda.states.OK:

lib/galaxy/util/zipstream.py

@@ -8,16 +8,16 @@ from .path import safe_walk
 
 class ZipstreamWrapper:
 
-    def __init__(self, upstream_zip=False, archive_name=None):
-        self.upstream_zip = upstream_zip
+    def __init__(self, archive_name=None, upstream_mod_zip=False, upstream_gzip=False):
+        self.upstream_mod_zip = upstream_mod_zip
         self.archive_name = archive_name
-        if not self.upstream_zip:
-            self.archive = zipstream.ZipFile(allowZip64=True, compression=zipstream.ZIP_DEFLATED)
+        if not self.upstream_mod_zip:
+            self.archive = zipstream.ZipFile(allowZip64=True, compression=zipstream.ZIP_STORED if upstream_gzip else zipstream.ZIP_DEFLATED)
         self.files = []
         self.size = 0
 
     def response(self):
-        if self.upstream_zip:
+        if self.upstream_mod_zip:
             yield "\n".join(self.files).encode()
         else:
             yield iter(self.archive)
@@ -26,7 +26,7 @@ class ZipstreamWrapper:
         headers = {}
         if self.archive_name:
             headers['Content-Disposition'] = f'attachment; filename="{self.archive_name}.zip"'
-        if self.upstream_zip:
+        if self.upstream_mod_zip:
             headers['X-Archive-Files'] = 'zip'
         else:
             headers['Content-Type'] = 'application/x-zip-compressed'
@@ -34,7 +34,7 @@ class ZipstreamWrapper:
 
     def add_path(self, path, archive_name):
         size = int(os.stat(path).st_size)
-        if self.upstream_zip:
+        if self.upstream_mod_zip:
             # calculating crc32 would defeat the point of using mod-zip, but if we ever calculate hashsums we should consider this
             crc32 = "-"
             line = f"{crc32} {size} {quote(path)} {archive_name}"

lib/galaxy/webapps/galaxy/api/history_contents.py

@@ -296,7 +296,7 @@ class HistoryContentsController(BaseAPIController, UsesLibraryMixin, UsesLibrary
             return {'error': util.unicodify(e)}
 
     def __stream_dataset_collection(self, trans, dataset_collection_instance):
-        archive = hdcas.stream_dataset_collection(dataset_collection_instance=dataset_collection_instance, upstream_zip=trans.app.config.upstream_zip)
+        archive = hdcas.stream_dataset_collection(dataset_collection_instance=dataset_collection_instance, upstream_mod_zip=trans.app.config.upstream_mod_zip)
         trans.response.headers.update(archive.get_headers())
         return archive.response()
 
@@ -1019,8 +1019,11 @@ class HistoryContentsController(BaseAPIController, UsesLibraryMixin, UsesLibrary
             return safe_dumps(paths_and_files)
 
         # create the archive, add the dataset files, then stream the archive as a download
-        archive = ZipstreamWrapper(upstream_zip=self.app.config.upstream_zip, archive_name=archive_base_name)
-
+        archive = ZipstreamWrapper(
+            archive_name=archive_base_name,
+            upstream_mod_zip=self.app.config.upstream_mod_zip,
+            upstream_gzip=self.app.config.upstream_gzip,
+        )
         for file_path, archive_path in paths_and_files:
             archive.write(file_path, archive_path)
 

lib/galaxy/webapps/galaxy/api/library_datasets.py

@@ -583,7 +583,11 @@ class LibraryDatasetsController(BaseAPIController, UsesVisualizationMixin, Libra
             raise exceptions.RequestParameterMissingException('Request has to contain a list of dataset ids or folder ids to download.')
 
         if archive_format == 'zip':
-            archive = ZipstreamWrapper(upstream_zip=self.app.config.upstream_zip, archive_name="selected_library_files")
+            archive = ZipstreamWrapper(
+                archive_name="selected_library_files",
+                upstream_mod_zip=self.app.config.upstream_mod_zip,
+                upstream_gzip=self.app.config.upstream_gzip,
+            )
             killme = string.punctuation + string.whitespace
             trantab = str.maketrans(killme, '_' * len(killme))
             seen = []

lib/galaxy/webapps/galaxy/api/tools.py

@@ -156,8 +156,12 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin):
                 trans.response.headers["Content-Disposition"] = f'attachment; filename="{filename}"'
                 return open(path, mode='rb')
             elif os.path.isdir(path):
-                # Set upstream_zip to false, otherwise tool data must be among allowed internal routes
-                archive = ZipstreamWrapper(upstream_zip=False, archive_name=filename)
+                # Set upstream_mod_zip to false, otherwise tool data must be among allowed internal routes
+                archive = ZipstreamWrapper(
+                    upstream_mod_zip=False,
+                    upstream_gzip=self.app.config.upstream_gzip,
+                    archive_name=filename,
+                )
                 archive.write(path)
                 trans.response.headers.update(archive.get_headers())
                 return archive.response()

lib/galaxy/webapps/galaxy/config_schema.yml

@@ -1424,14 +1424,25 @@ mapping:
           should be set to the path defined in the nginx config as an internal redirect
           with access to Galaxy's data files (see documentation linked above).
 
-      upstream_zip:
+      upstream_gzip:
+        type: bool
+        default: false
+        required: false
+        desc: |
+          If using compression in the upstream proxy server, use this option to disable
+          gzipping of dataset collection and library archives, since the upstream server
+          will do it faster on the fly. To enable compression add ``application/zip``
+          to the proxy's compressable mimetypes.
+
+      upstream_mod_zip:
         type: bool
         default: false
         required: false
         desc: |
           If using the mod-zip module in nginx, use this option to assemble
-          zip archives in nginx. Requires setting up internal nginx locations
-          to all paths that can be archived.
+          zip archives in nginx. This is preferable over the upstream_gzip option
+          as Galaxy does not need to serve the archive.
+          Requires setting up internal nginx locations to all paths that can be archived.
           See https://docs.galaxyproject.org/en/master/admin/nginx.html#creating-archives-with-mod-zip
           for details.