diff --git a/doc/source/admin/nginx.md b/doc/source/admin/nginx.md index f30b3517d1e..01cb555953f 100644 --- a/doc/source/admin/nginx.md +++ b/doc/source/admin/nginx.md @@ -442,7 +442,7 @@ galaxy: Galaxy creates zip archives when downloading multiple datasets from a history or a dataset library. While this works fine for small datasets and few users, nginx can handle the creation of zip archives more efficiently using [mod-zip](https://www.nginx.com/resources/wiki/modules/zip/). -To use this feature, install nginx with mod-zip enabled, provide the file locations from which +To use this feature, install nginx with mod-zip enabled (requires or a newer), provide the file locations from which nginx should serve files and edit `galaxy.yml` and make the following changes before restarting Galaxy: ```yaml diff --git a/lib/galaxy/util/zipstream.py b/lib/galaxy/util/zipstream.py index 53dd88d5707..5bddbb68ab5 100644 --- a/lib/galaxy/util/zipstream.py +++ b/lib/galaxy/util/zipstream.py @@ -1,10 +1,14 @@ import os +import zlib from urllib.parse import quote import zipstream from .path import safe_walk +CRC32_MIN = 1444 +CRC32_MAX = 1459 + class ZipstreamWrapper: def __init__(self, archive_name=None, upstream_mod_zip=False, upstream_gzip=False): @@ -15,11 +19,13 @@ class ZipstreamWrapper: allowZip64=True, compression=zipstream.ZIP_STORED if upstream_gzip else zipstream.ZIP_DEFLATED ) self.files = [] + self.directories = set() self.size = 0 def response(self): if self.upstream_mod_zip: - yield "\n".join(self.files).encode() + dir_lines = [f"0 0 @directory {directory}" for directory in self.directories] + yield "\n".join(dir_lines + self.files).encode() else: yield from iter(self.archive) @@ -38,7 +44,16 @@ class ZipstreamWrapper: if self.upstream_mod_zip: # calculating crc32 would defeat the point of using mod-zip, but if we ever calculate hashsums we should consider this crc32 = "-" + # We do have to calculate the crc32 for files that are between 1444 and 1459 bytes in size, xref: https://github.com/evanmiller/mod_zip/issues/44#issuecomment-656660686 + # Oddly that seems to be only true for usegalaxy.org (nginx version 1.12.2), and works fine locally (nginx 1.19.10). + # May have been fixed in nginx 1.17.0 + if CRC32_MIN <= os.path.getsize(path) <= CRC32_MAX: + with open(path, "rb") as contents: + crc32 = hex(zlib.crc32(contents.read()))[2:] line = f"{crc32} {size} {quote(path)} {archive_name}" + head, tail = os.path.split(archive_name) + if head: + self.directories.add(head) self.files.append(line) else: self.size += size