From a237237c490903c0e31f10f6332c27cf48ac4bab Mon Sep 17 00:00:00 2001 From: Peter van Heusden Date: Sun, 15 Jun 2025 12:34:59 +0200 Subject: [PATCH 1/8] Add Auspice JSON datatype --- .../config/sample/datatypes_conf.xml.sample | 2 + lib/galaxy/datatypes/test/1.auspicejson | 14 ++++++ lib/galaxy/datatypes/text.py | 49 +++++++++++++++++++ 3 files changed, 65 insertions(+) create mode 100644 lib/galaxy/datatypes/test/1.auspicejson diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index 49270c49811..0ef9ae55312 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -148,6 +148,7 @@ + @@ -1364,6 +1365,7 @@ + diff --git a/lib/galaxy/datatypes/test/1.auspicejson b/lib/galaxy/datatypes/test/1.auspicejson new file mode 100644 index 00000000000..541dcdaefb2 --- /dev/null +++ b/lib/galaxy/datatypes/test/1.auspicejson @@ -0,0 +1,14 @@ +{ + "version": "v2", + "meta": { + "title": "Minimal AuspiceJSON", + "updated": "2025-02-05", + "panels": ["tree"] + }, + "tree": { + "name": "1", + "node_attrs": { + "div": 1 + } + } +} diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index bf4f51e4753..9398c1991ce 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -700,6 +700,55 @@ class VitessceJson(Json): return False +@build_sniff_from_prefix +class AuspiceJson(Json): + """ + Auspice is a visualization tool for phylogenetic trees and associated data. + It uses JSON format to represent the tree structure and metadata. + """ + + file_ext = "auspicejson" + + def set_peek(self, dataset: DatasetProtocol, **kwd) -> None: + super().set_peek(dataset) + if not dataset.dataset.purged: + dataset.blurb = "AuspiceJSON" + + def sniff_prefix(self, file_prefix: FilePrefix) -> bool: + """ + Determines whether the file is in Auspice v2 JSON by looking for keys + like "version", "meta" and "updated" that are both required by the + https://docs.nextstrain.org/projects/auspice/en/stable/releases/v2.html format + and also will be in the first part of the file + + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname( '1.json' ) + >>> AuspiceJson().sniff( fname ) + False + >>> fname = get_test_fname( '1.auspicejson' ) + >>> AuspiceJson().sniff( fname ) + True + """ + is_auspicejson = False + if self._looks_like_json(file_prefix): + is_auspicejson = self._looks_like_is_auspicejson(file_prefix) + return is_auspicejson + + def _looks_like_is_auspicejson(self, file_prefix: FilePrefix, load_size: int = 20000) -> bool: + """ + Expects 'meta', 'tree', and 'nodes' to be present as keys in the JSON structure. + """ + try: + with open(file_prefix.filename) as fh: + segment_str = fh.read(load_size) + + if all(x in segment_str for x in ["version", "meta", "updated"]): + return True + except Exception: + pass + return False + + @build_sniff_from_prefix class Obo(Text): """ From d415fd4f9f5e8ddf047da1073309fd47aea90b74 Mon Sep 17 00:00:00 2001 From: Peter van Heusden Date: Sat, 21 Jun 2025 08:43:52 +0200 Subject: [PATCH 2/8] Add "panels" as required key for AuspiceJSON --- lib/galaxy/datatypes/text.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index 9398c1991ce..4cd69385202 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -742,7 +742,7 @@ class AuspiceJson(Json): with open(file_prefix.filename) as fh: segment_str = fh.read(load_size) - if all(x in segment_str for x in ["version", "meta", "updated"]): + if all(x in segment_str for x in ["version", "meta", "updated", "panels"]): return True except Exception: pass From 719cf891641fd47a9606fcae047296077001774b Mon Sep 17 00:00:00 2001 From: Peter van Heusden Date: Sun, 22 Jun 2025 14:09:36 +0200 Subject: [PATCH 3/8] Expect Auspice JSON to start with "{" --- lib/galaxy/datatypes/text.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index 4cd69385202..55fc19a1fb6 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -736,13 +736,13 @@ class AuspiceJson(Json): def _looks_like_is_auspicejson(self, file_prefix: FilePrefix, load_size: int = 20000) -> bool: """ - Expects 'meta', 'tree', and 'nodes' to be present as keys in the JSON structure. + Expects JSON to start with { and 'meta', 'tree', 'updated' and 'nodes' to be present as keys in the JSON structure. """ try: with open(file_prefix.filename) as fh: segment_str = fh.read(load_size) - if all(x in segment_str for x in ["version", "meta", "updated", "panels"]): + if segment_str.startswith('{') and all(x in segment_str for x in ["version", "meta", "updated", "panels"]): return True except Exception: pass From e9bb3767e4c0c9be7eb1bf840e54cfb1eab482d4 Mon Sep 17 00:00:00 2001 From: Peter van Heusden Date: Sun, 22 Jun 2025 14:18:03 +0200 Subject: [PATCH 4/8] Satisfy black linting --- lib/galaxy/datatypes/text.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index 55fc19a1fb6..b477a0b7b66 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -742,7 +742,9 @@ class AuspiceJson(Json): with open(file_prefix.filename) as fh: segment_str = fh.read(load_size) - if segment_str.startswith('{') and all(x in segment_str for x in ["version", "meta", "updated", "panels"]): + if segment_str.startswith("{") and all( + x in segment_str for x in ["version", "meta", "updated", "panels"] + ): return True except Exception: pass From 182f6c2bb67c301577651df2b12471183508dee3 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 1 Oct 2025 11:15:50 +0200 Subject: [PATCH 5/8] Fix type, use auspice.json extension --- lib/galaxy/config/sample/datatypes_conf.xml.sample | 2 +- lib/galaxy/datatypes/text.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index 0ef9ae55312..1389cd97bca 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -148,7 +148,7 @@ - + diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index b477a0b7b66..01c4ac06ac4 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -707,7 +707,7 @@ class AuspiceJson(Json): It uses JSON format to represent the tree structure and metadata. """ - file_ext = "auspicejson" + file_ext = "auspice.json" def set_peek(self, dataset: DatasetProtocol, **kwd) -> None: super().set_peek(dataset) From c2899e91cef2fdcbe802164966e73160acddca6f Mon Sep 17 00:00:00 2001 From: guerler Date: Wed, 1 Oct 2025 17:14:05 +0300 Subject: [PATCH 6/8] Ensure that conversion_key is defined --- lib/galaxy/model/store/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/model/store/__init__.py b/lib/galaxy/model/store/__init__.py index b5120d07dc2..eaac5720ae2 100644 --- a/lib/galaxy/model/store/__init__.py +++ b/lib/galaxy/model/store/__init__.py @@ -2030,14 +2030,13 @@ class DirectoryModelExportStore(ModelExportStore): as_dict["extra_files_path"] = extra_files_path return + conversion = self.dataset_implicit_conversions.get(dataset) + conversion_key = self.serialization_options.get_identifier(self.security, conversion) if conversion else None + if file_name: if not os.path.exists(dir_path): os.makedirs(dir_path) - conversion = self.dataset_implicit_conversions.get(dataset) - conversion_key = ( - self.serialization_options.get_identifier(self.security, conversion) if conversion else None - ) target_filename = get_export_dataset_filename( as_dict["name"], as_dict["extension"], as_dict["encoded_id"], conversion_key=conversion_key ) From 456cfca57a2d79dc3035a5fcd76588b4f2214fde Mon Sep 17 00:00:00 2001 From: caroott <39764934+caroott@users.noreply.github.com> Date: Fri, 3 Oct 2025 15:12:26 +0200 Subject: [PATCH 7/8] Improves zip file type detection for uploads Recognizes both "application/zip" and "application/x-zip-compressed" MIME types to support windows uploads and prevent false negatives when validating local zip files. --- client/src/composables/zipExplorer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/composables/zipExplorer.ts b/client/src/composables/zipExplorer.ts index c7184801b5e..f857e46c155 100644 --- a/client/src/composables/zipExplorer.ts +++ b/client/src/composables/zipExplorer.ts @@ -310,7 +310,7 @@ export function validateLocalZipFile(file?: File | null): string { } export function isLocalZipFile(file?: File | null): boolean { - return Boolean(file) && file?.type === "application/zip"; + return Boolean(file) && (file?.type === "application/zip" || file?.type === "application/x-zip-compressed"); } export async function isRemoteZipFile(url: string): Promise { From 9896e9b4977852f21ae6b078a504c82ed746832c Mon Sep 17 00:00:00 2001 From: Amirhossein Naghsh Nilchi Date: Sat, 9 Aug 2025 15:56:46 +0200 Subject: [PATCH 8/8] add safetensors remove safetensor dependency keeping only sniff update description test header and add more comments and info about data add test test files match the files in galaxy-test-data add test files Update lib/galaxy/config/sample/datatypes_conf.xml.sample Co-authored-by: M Bernt remove subclass=true add Wolfgang's suggestion black reformater json check using same function for json and safetensors fix lint reformat fix isort lint Rename test dataset rename test file try to use FilePrefix add if for when the header is big fix lint Update lib/galaxy/datatypes/util/json.py Co-authored-by: Nicola Soranzo Update lib/galaxy/datatypes/binary.py Co-authored-by: Nicola Soranzo remove unused function Update lib/galaxy/datatypes/binary.py Co-authored-by: Nicola Soranzo Restore text.py and remove json.py check if header is dict Update lib/galaxy/config/sample/datatypes_conf.xml.sample Co-authored-by: Nicola Soranzo remove pth file Update lib/galaxy/datatypes/binary.py Co-authored-by: Nicola Soranzo Update lib/galaxy/datatypes/binary.py Co-authored-by: Nicola Soranzo Update lib/galaxy/datatypes/binary.py Co-authored-by: Nicola Soranzo correct test file name --- .../config/sample/datatypes_conf.xml.sample | 1 + lib/galaxy/datatypes/binary.py | 99 ++++++++++++++++++ .../cellpose_model_safetensors.safetensors | Bin 0 -> 76 bytes 3 files changed, 100 insertions(+) create mode 100644 lib/galaxy/datatypes/test/cellpose_model_safetensors.safetensors diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index 3e75bce62ff..f155b9d17e8 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -1172,6 +1172,7 @@ + diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 575616c183d..db4fe53cb73 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -4847,3 +4847,102 @@ class Hic(Binary): with open(dataset.get_file_name(), "rb") as handle: header_bytes = handle.read(8) dataset.metadata.version = struct.unpack(" bool: + """ + Determining if the file is in safetensors format + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname('cellpose_model_safetensors.safetensors') + >>> Safetensors().sniff(fname) + True + >>> fname = get_test_fname('test_charmm.vel') + >>> Safetensors().sniff(fname) + False + """ + try: + # Safetensors files start with an 8-byte little-endian integer + # indicating the size of the JSON header + if len(file_prefix.contents_header_bytes) < 8: + return False + + header_size = int.from_bytes(file_prefix.contents_header_bytes[:8], "little") + + # Currently, there's a limit on the size of the header of 100MB to prevent parsing extremely large JSON headers + # In practice, safetensors headers are typically just a few KB to MB + # (containing tensor names, shapes, dtypes, and offsets - rarely exceeds 1-10MB even for large models) + # But in theory it is possible to have 100 MB header + # more info here: https://github.com/huggingface/safetensors?tab=readme-ov-file#benefits + if header_size == 0 or header_size > 10**8: # 100MB max for JSON header + return False + + # Check if file is large enough to contain the full header + if file_prefix.file_size < 8 + header_size: + return False + + # CRITICAL: Check if header begins with '{' character (0x7B) as per safetensors spec + # This is required by the format and helps distinguish from other binary formats + # Only check 1 byte to avoid issues with malicious header_size values + # more info here: https://github.com/huggingface/safetensors?tab=readme-ov-file#format + if file_prefix.contents_header_bytes[8] != 0x7B: + return False + + # Check if header ends with '}' character (0x7D) as per safetensors spec + # This requires reading more data if header extends beyond the prefix + header_end_pos = 8 + header_size - 1 + if header_end_pos < len(file_prefix.contents_header_bytes): + # Header end is within the prefix + if file_prefix.contents_header_bytes[header_end_pos] != 0x7D: + return False + else: + # Header extends beyond prefix, need to check from file + with open(file_prefix.filename, "rb") as f: + f.seek(header_end_pos) + last_header_byte = f.read(1) + if len(last_header_byte) != 1 or last_header_byte[0] != 0x7D: + return False + + # Read the full header for JSON parsing + if 8 + header_size <= len(file_prefix.contents_header_bytes): + # Entire header is in the prefix + header_bytes = file_prefix.contents_header_bytes[8 : 8 + header_size] + else: + # Need to read full header from file + with open(file_prefix.filename, "rb") as f: + f.seek(8) + header_bytes = f.read(header_size) + + if len(header_bytes) != header_size: + return False + + # Parse the validated JSON header + header = json.loads(header_bytes.decode("utf-8")) + # check if header is a dict + if not isinstance(header, dict): + return False + # Basic validation: check if it looks like safetensors metadata + # Safetensors headers should have entries with data_offsets + has_valid_entries = False + for key, value in header.items(): + if key == "__metadata__": # Special metadata key + continue + if isinstance(value, dict) and "data_offsets" in value: + has_valid_entries = True + break + + return has_valid_entries + + except Exception: + # Any exception during parsing means it's not a valid safetensors file + return False diff --git a/lib/galaxy/datatypes/test/cellpose_model_safetensors.safetensors b/lib/galaxy/datatypes/test/cellpose_model_safetensors.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d24e51dc49acbcd046539062cb8b4068696cf0ea GIT binary patch literal 76 zcmZ=@fPiYHl+483_?*O~)SO}^D