From 28423d27efcc2981c23a4356a3ddbf3f5052fb06 Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Thu, 13 Jan 2022 15:19:10 -0500 Subject: [PATCH 1/2] Clear the cache before fetching a file from irods object store, to verify file is fetched from the object store, not from the cache --- .../objectstore/test_objectstore_datatype_upload.py | 2 +- test/integration/test_datatype_upload.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/test/integration/objectstore/test_objectstore_datatype_upload.py b/test/integration/objectstore/test_objectstore_datatype_upload.py index 52178898be8..de0a24a2091 100644 --- a/test/integration/objectstore/test_objectstore_datatype_upload.py +++ b/test/integration/objectstore/test_objectstore_datatype_upload.py @@ -183,7 +183,7 @@ def test_upload_datatype_dos_disk_and_disk(distributed_instance, test_data, temp @pytest.mark.parametrize('test_data', TEST_CASES.values(), ids=list(TEST_CASES.keys())) def test_upload_datatype_irods(irods_instance, test_data, temp_file): - upload_datatype_helper(irods_instance, test_data, temp_file) + upload_datatype_helper(irods_instance, test_data, temp_file, True) @pytest.mark.parametrize('test_data', TEST_CASES.values(), ids=list(TEST_CASES.keys())) diff --git a/test/integration/test_datatype_upload.py b/test/integration/test_datatype_upload.py index e8c46ae17db..2e2a0485ed6 100644 --- a/test/integration/test_datatype_upload.py +++ b/test/integration/test_datatype_upload.py @@ -1,5 +1,6 @@ import collections import os +import shutil import pytest @@ -59,7 +60,7 @@ def test_upload_datatype_auto(instance, test_data, temp_file): upload_datatype_helper(instance, test_data, temp_file) -def upload_datatype_helper(instance, test_data, temp_file): +def upload_datatype_helper(instance, test_data, temp_file, delete_cache_dir=False): is_compressed = False for is_method in (is_bz2, is_gzip, is_zip): is_compressed = is_method(test_data.path) @@ -93,6 +94,14 @@ def upload_datatype_helper(instance, test_data, temp_file): datatype = registry.datatypes_by_extension[file_ext] datatype_compressed = getattr(datatype, "compressed", False) if not is_compressed or datatype_compressed: + if delete_cache_dir: + # Delete cache directory and then re-create it. This way we confirm + # that dataset is fetched from the object store, not from the cache + temp_dir = instance.get_object_store_kwargs()['temp_directory'] + cache_dir = temp_dir + '/object_store_cache' + shutil.rmtree(cache_dir) + os.mkdir(cache_dir) + # download file and verify it hasn't been manipulated temp_file.write(instance.dataset_populator.get_history_dataset_content(history_id=instance.history_id, dataset=dataset, From 2b70e34a61e024ea6bdb0b08ec7bd51998559b90 Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Thu, 6 Jan 2022 09:41:28 -0500 Subject: [PATCH 2/2] Revised code based on code review --- lib/galaxy/objectstore/irods.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/objectstore/irods.py b/lib/galaxy/objectstore/irods.py index eb6841f6c51..9b9bfa4bf5c 100644 --- a/lib/galaxy/objectstore/irods.py +++ b/lib/galaxy/objectstore/irods.py @@ -429,7 +429,7 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin): log.debug("Pushing cache file '%s' of size %s bytes to collection '%s'", source_file, os.path.getsize(source_file), rel_path) # Add the source file to the irods collection - self.session.data_objects.put(source_file, f"{collection_path}/", **options) + self.session.data_objects.put(source_file, data_object_path, **options) end_time = datetime.now() log.debug("Pushed cache file '%s' to collection '%s' (%s bytes transfered in %s sec)",