From f83aefd82ac82b433486b0742b91bdc3e269e6e5 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 9 Aug 2022 11:01:39 +0200 Subject: [PATCH 1/2] Calculate CRC32 for files in size between 1444 and 1459 bytes --- lib/galaxy/util/zipstream.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/galaxy/util/zipstream.py b/lib/galaxy/util/zipstream.py index 53dd88d5707..e148be5c2cb 100644 --- a/lib/galaxy/util/zipstream.py +++ b/lib/galaxy/util/zipstream.py @@ -1,10 +1,14 @@ import os +import zlib from urllib.parse import quote import zipstream from .path import safe_walk +CRC32_MIN = 1444 +CRC32_MAX = 1459 + class ZipstreamWrapper: def __init__(self, archive_name=None, upstream_mod_zip=False, upstream_gzip=False): @@ -38,6 +42,12 @@ class ZipstreamWrapper: if self.upstream_mod_zip: # calculating crc32 would defeat the point of using mod-zip, but if we ever calculate hashsums we should consider this crc32 = "-" + # We do have to calculate the crc32 for files that are between 1444 and 1459 bytes in size, xref: https://github.com/evanmiller/mod_zip/issues/44#issuecomment-656660686 + # Oddly that seems to be only true for usegalaxy.org (nginx version 1.12.2), and works fine locally (nginx 1.19.10). + # May have been fixed in nginx 1.17.0 + if CRC32_MIN <= os.path.getsize(path) <= CRC32_MAX: + with open(path, "rb") as contents: + crc32 = hex(zlib.crc32(contents.read()))[2:] line = f"{crc32} {size} {quote(path)} {archive_name}" self.files.append(line) else: From 728ceebc8aa712be6b702778303dbc3a1fdacea3 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 10 Aug 2022 15:30:20 +0200 Subject: [PATCH 2/2] Fix nested archive unzipping on windows We need the directory markers in order for the default windows zip utility to be able to extract the archives. Also requires https://github.com/evanmiller/mod_zip/commit/51cf45d3e9f51e02224af017b235d1d30fbf28fb or newer on the mod_zip side. --- doc/source/admin/nginx.md | 2 +- lib/galaxy/util/zipstream.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/doc/source/admin/nginx.md b/doc/source/admin/nginx.md index f30b3517d1e..01cb555953f 100644 --- a/doc/source/admin/nginx.md +++ b/doc/source/admin/nginx.md @@ -442,7 +442,7 @@ galaxy: Galaxy creates zip archives when downloading multiple datasets from a history or a dataset library. While this works fine for small datasets and few users, nginx can handle the creation of zip archives more efficiently using [mod-zip](https://www.nginx.com/resources/wiki/modules/zip/). -To use this feature, install nginx with mod-zip enabled, provide the file locations from which +To use this feature, install nginx with mod-zip enabled (requires or a newer), provide the file locations from which nginx should serve files and edit `galaxy.yml` and make the following changes before restarting Galaxy: ```yaml diff --git a/lib/galaxy/util/zipstream.py b/lib/galaxy/util/zipstream.py index e148be5c2cb..5bddbb68ab5 100644 --- a/lib/galaxy/util/zipstream.py +++ b/lib/galaxy/util/zipstream.py @@ -19,11 +19,13 @@ class ZipstreamWrapper: allowZip64=True, compression=zipstream.ZIP_STORED if upstream_gzip else zipstream.ZIP_DEFLATED ) self.files = [] + self.directories = set() self.size = 0 def response(self): if self.upstream_mod_zip: - yield "\n".join(self.files).encode() + dir_lines = [f"0 0 @directory {directory}" for directory in self.directories] + yield "\n".join(dir_lines + self.files).encode() else: yield from iter(self.archive) @@ -49,6 +51,9 @@ class ZipstreamWrapper: with open(path, "rb") as contents: crc32 = hex(zlib.crc32(contents.read()))[2:] line = f"{crc32} {size} {quote(path)} {archive_name}" + head, tail = os.path.split(archive_name) + if head: + self.directories.add(head) self.files.append(line) else: self.size += size