Merge pull request #14417 from mvdbeek/crc32_for_specific_file_sizes

[22.05] Calculate CRC32 for files in size between 1444 and 1459 bytes

doc/source/admin/nginx.md

@@ -442,7 +442,7 @@ galaxy:
 Galaxy creates zip archives when downloading multiple datasets from a history or a dataset library.
 While this works fine for small datasets and few users, nginx can handle the creation of zip archives
 more efficiently using [mod-zip](https://www.nginx.com/resources/wiki/modules/zip/).
-To use this feature, install nginx with mod-zip enabled, provide the file locations from which
+To use this feature, install nginx with mod-zip enabled (requires <https://github.com/evanmiller/mod_zip/commit/51cf45d3e9f51e02224af017b235d1d30fbf28fb> or a newer), provide the file locations from which
 nginx should serve files and edit `galaxy.yml` and make the following changes before restarting Galaxy:
 
 ```yaml

lib/galaxy/util/zipstream.py

@@ -1,10 +1,14 @@
 import os
+import zlib
 from urllib.parse import quote
 
 import zipstream
 
 from .path import safe_walk
 
+CRC32_MIN = 1444
+CRC32_MAX = 1459
+
 
 class ZipstreamWrapper:
     def __init__(self, archive_name=None, upstream_mod_zip=False, upstream_gzip=False):
@@ -15,11 +19,13 @@ class ZipstreamWrapper:
                 allowZip64=True, compression=zipstream.ZIP_STORED if upstream_gzip else zipstream.ZIP_DEFLATED
             )
         self.files = []
+        self.directories = set()
         self.size = 0
 
     def response(self):
         if self.upstream_mod_zip:
-            yield "\n".join(self.files).encode()
+            dir_lines = [f"0 0 @directory {directory}" for directory in self.directories]
+            yield "\n".join(dir_lines + self.files).encode()
         else:
             yield from iter(self.archive)
 
@@ -38,7 +44,16 @@ class ZipstreamWrapper:
         if self.upstream_mod_zip:
             # calculating crc32 would defeat the point of using mod-zip, but if we ever calculate hashsums we should consider this
             crc32 = "-"
+            # We do have to calculate the crc32 for files that are between 1444 and 1459 bytes in size, xref: https://github.com/evanmiller/mod_zip/issues/44#issuecomment-656660686
+            # Oddly that seems to be only true for usegalaxy.org (nginx version 1.12.2), and works fine locally (nginx 1.19.10).
+            # May have been fixed in nginx 1.17.0
+            if CRC32_MIN <= os.path.getsize(path) <= CRC32_MAX:
+                with open(path, "rb") as contents:
+                    crc32 = hex(zlib.crc32(contents.read()))[2:]
             line = f"{crc32} {size} {quote(path)} {archive_name}"
+            head, tail = os.path.split(archive_name)
+            if head:
+                self.directories.add(head)
             self.files.append(line)
         else:
             self.size += size