From 540c39298b0e27c261bf1526a762128367ddb494 Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Fri, 17 Dec 2021 12:14:31 -0500 Subject: [PATCH 1/3] Fix issue with uploading cahced file to irods --- lib/galaxy/objectstore/irods.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/objectstore/irods.py b/lib/galaxy/objectstore/irods.py index 15997a7158b..65293ec9b95 100644 --- a/lib/galaxy/objectstore/irods.py +++ b/lib/galaxy/objectstore/irods.py @@ -662,6 +662,7 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin): # FIXME? Should this be a `move`? shutil.copy2(source_file, cache_file) self._fix_permissions(cache_file) + source_file = cache_file except OSError: log.exception("Trouble copying source file '%s' to cache '%s'", source_file, cache_file) else: From 638ba6e4be1e6902134fa04947c44502c4378517 Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Thu, 6 Jan 2022 09:41:28 -0500 Subject: [PATCH 2/3] Revised code based on code review --- lib/galaxy/objectstore/irods.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/galaxy/objectstore/irods.py b/lib/galaxy/objectstore/irods.py index 65293ec9b95..57cf93c082f 100644 --- a/lib/galaxy/objectstore/irods.py +++ b/lib/galaxy/objectstore/irods.py @@ -420,7 +420,7 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin): log.debug("Pushing cache file '%s' of size %s bytes to collection '%s'", source_file, os.path.getsize(source_file), rel_path) # Add the source file to the irods collection - self.session.data_objects.put(source_file, f"{collection_path}/", **options) + self.session.data_objects.put(source_file, data_object_path, **options) end_time = datetime.now() log.debug("Pushed cache file '%s' to collection '%s' (%s bytes transfered in %s sec)", @@ -662,7 +662,6 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin): # FIXME? Should this be a `move`? shutil.copy2(source_file, cache_file) self._fix_permissions(cache_file) - source_file = cache_file except OSError: log.exception("Trouble copying source file '%s' to cache '%s'", source_file, cache_file) else: From 85ade3df4f445312b69914d96d90a1719e9fae2b Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Thu, 13 Jan 2022 15:19:10 -0500 Subject: [PATCH 3/3] Clear the cache before fetching a file from irods object store, to verify file is fetched from the object store, not from the cache --- .../objectstore/test_objectstore_datatype_upload.py | 2 +- test/integration/test_datatype_upload.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/test/integration/objectstore/test_objectstore_datatype_upload.py b/test/integration/objectstore/test_objectstore_datatype_upload.py index 52178898be8..de0a24a2091 100644 --- a/test/integration/objectstore/test_objectstore_datatype_upload.py +++ b/test/integration/objectstore/test_objectstore_datatype_upload.py @@ -183,7 +183,7 @@ def test_upload_datatype_dos_disk_and_disk(distributed_instance, test_data, temp @pytest.mark.parametrize('test_data', TEST_CASES.values(), ids=list(TEST_CASES.keys())) def test_upload_datatype_irods(irods_instance, test_data, temp_file): - upload_datatype_helper(irods_instance, test_data, temp_file) + upload_datatype_helper(irods_instance, test_data, temp_file, True) @pytest.mark.parametrize('test_data', TEST_CASES.values(), ids=list(TEST_CASES.keys())) diff --git a/test/integration/test_datatype_upload.py b/test/integration/test_datatype_upload.py index e8c46ae17db..2e2a0485ed6 100644 --- a/test/integration/test_datatype_upload.py +++ b/test/integration/test_datatype_upload.py @@ -1,5 +1,6 @@ import collections import os +import shutil import pytest @@ -59,7 +60,7 @@ def test_upload_datatype_auto(instance, test_data, temp_file): upload_datatype_helper(instance, test_data, temp_file) -def upload_datatype_helper(instance, test_data, temp_file): +def upload_datatype_helper(instance, test_data, temp_file, delete_cache_dir=False): is_compressed = False for is_method in (is_bz2, is_gzip, is_zip): is_compressed = is_method(test_data.path) @@ -93,6 +94,14 @@ def upload_datatype_helper(instance, test_data, temp_file): datatype = registry.datatypes_by_extension[file_ext] datatype_compressed = getattr(datatype, "compressed", False) if not is_compressed or datatype_compressed: + if delete_cache_dir: + # Delete cache directory and then re-create it. This way we confirm + # that dataset is fetched from the object store, not from the cache + temp_dir = instance.get_object_store_kwargs()['temp_directory'] + cache_dir = temp_dir + '/object_store_cache' + shutil.rmtree(cache_dir) + os.mkdir(cache_dir) + # download file and verify it hasn't been manipulated temp_file.write(instance.dataset_populator.get_history_dataset_content(history_id=instance.history_id, dataset=dataset,