From 23eb63323c38e2efde73d077a3e0ab7b36b18b1a Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Mon, 13 Apr 2020 10:33:09 -0400 Subject: [PATCH 01/10] Enabled various object store types to store data objects by uuid. --- lib/galaxy/objectstore/azure_blob.py | 10 +++++----- lib/galaxy/objectstore/cloud.py | 10 +++++----- lib/galaxy/objectstore/irods.py | 10 +++++----- lib/galaxy/objectstore/pithos.py | 10 +++++----- test/integration/test_datatype_upload_irods.py | 1 + 5 files changed, 21 insertions(+), 20 deletions(-) diff --git a/lib/galaxy/objectstore/azure_blob.py b/lib/galaxy/objectstore/azure_blob.py index 93c8c335d8c..7b12f255df1 100644 --- a/lib/galaxy/objectstore/azure_blob.py +++ b/lib/galaxy/objectstore/azure_blob.py @@ -173,7 +173,7 @@ class AzureBlobObjectStore(ConcreteObjectStore): # follow them, so if they are valid we normalize them out alt_name = os.path.normpath(alt_name) - rel_path = os.path.join(*directory_hash_id(obj.id)) + rel_path = os.path.join(*directory_hash_id(self._get_object_id(obj))) if extra_dir is not None: if extra_dir_at_root: @@ -183,7 +183,7 @@ class AzureBlobObjectStore(ConcreteObjectStore): # for JOB_WORK directory if obj_dir: - rel_path = os.path.join(rel_path, str(obj.id)) + rel_path = os.path.join(rel_path, str(self._get_object_id(obj))) if base_dir: base = self.extra_dirs.get(base_dir) return os.path.join(base, rel_path) @@ -192,7 +192,7 @@ class AzureBlobObjectStore(ConcreteObjectStore): # rel_path = '%s/' % rel_path # assume for now we don't need this in Azure blob storage. if not dir_only: - rel_path = os.path.join(rel_path, alt_name if alt_name else "dataset_%s.dat" % obj.id) + rel_path = os.path.join(rel_path, alt_name if alt_name else "dataset_%s.dat" % self._get_object_id(obj)) return rel_path @@ -368,7 +368,7 @@ class AzureBlobObjectStore(ConcreteObjectStore): alt_name = kwargs.get('alt_name', None) # Construct hashed path - rel_path = os.path.join(*directory_hash_id(obj.id)) + rel_path = os.path.join(*directory_hash_id(self._get_object_id(obj))) # Optionally append extra_dir if extra_dir is not None: @@ -389,7 +389,7 @@ class AzureBlobObjectStore(ConcreteObjectStore): # self._push_to_os(s3_dir, from_string='') # If instructed, create the dataset in cache & in S3 if not dir_only: - rel_path = os.path.join(rel_path, alt_name if alt_name else "dataset_%s.dat" % obj.id) + rel_path = os.path.join(rel_path, alt_name if alt_name else "dataset_%s.dat" % self._get_object_id(obj)) open(os.path.join(self.staging_path, rel_path), 'w').close() self._push_to_os(rel_path, from_string='') diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index 49d1fa778e7..a52d6eccbc0 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -351,7 +351,7 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): # alt_name can contain parent directory references, but S3 will not # follow them, so if they are valid we normalize them out alt_name = os.path.normpath(alt_name) - rel_path = os.path.join(*directory_hash_id(obj.id)) + rel_path = os.path.join(*directory_hash_id(self._get_object_id(obj))) if extra_dir is not None: if extra_dir_at_root: rel_path = os.path.join(extra_dir, rel_path) @@ -360,7 +360,7 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): # for JOB_WORK directory if obj_dir: - rel_path = os.path.join(rel_path, str(obj.id)) + rel_path = os.path.join(rel_path, str(self._get_object_id(obj))) if base_dir: base = self.extra_dirs.get(base_dir) return os.path.join(base, rel_path) @@ -369,7 +369,7 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): rel_path = '%s/' % rel_path if not dir_only: - rel_path = os.path.join(rel_path, alt_name if alt_name else "dataset_%s.dat" % obj.id) + rel_path = os.path.join(rel_path, alt_name if alt_name else "dataset_%s.dat" % self._get_object_id(obj)) return rel_path def _get_cache_path(self, rel_path): @@ -553,7 +553,7 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): alt_name = kwargs.get('alt_name', None) # Construct hashed path - rel_path = os.path.join(*directory_hash_id(obj.id)) + rel_path = os.path.join(*directory_hash_id(self._get_object_id(obj))) # Optionally append extra_dir if extra_dir is not None: @@ -568,7 +568,7 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): os.makedirs(cache_dir) if not dir_only: - rel_path = os.path.join(rel_path, alt_name if alt_name else "dataset_%s.dat" % obj.id) + rel_path = os.path.join(rel_path, alt_name if alt_name else "dataset_%s.dat" % self._get_object_id(obj)) open(os.path.join(self.staging_path, rel_path), 'w').close() self._push_to_os(rel_path, from_string='') diff --git a/lib/galaxy/objectstore/irods.py b/lib/galaxy/objectstore/irods.py index 06d49254a84..41cc160a777 100644 --- a/lib/galaxy/objectstore/irods.py +++ b/lib/galaxy/objectstore/irods.py @@ -258,7 +258,7 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin): # alt_name can contain parent directory references, but S3 will not # follow them, so if they are valid we normalize them out alt_name = os.path.normpath(alt_name) - rel_path = os.path.join(*directory_hash_id(obj.id)) + rel_path = os.path.join(*directory_hash_id(self._get_object_id(obj))) if extra_dir is not None: if extra_dir_at_root: rel_path = os.path.join(extra_dir, rel_path) @@ -267,13 +267,13 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin): # for JOB_WORK directory if obj_dir: - rel_path = os.path.join(rel_path, str(obj.id)) + rel_path = os.path.join(rel_path, str(self._get_object_id(obj))) if base_dir: base = self.extra_dirs.get(base_dir) return os.path.join(base, rel_path) if not dir_only: - rel_path = os.path.join(rel_path, alt_name if alt_name else "dataset_%s.dat" % obj.id) + rel_path = os.path.join(rel_path, alt_name if alt_name else "dataset_%s.dat" % self._get_object_id(obj)) return rel_path def _get_cache_path(self, rel_path): @@ -460,7 +460,7 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin): alt_name = kwargs.get('alt_name', None) # Construct hashed path - rel_path = os.path.join(*directory_hash_id(obj.id)) + rel_path = os.path.join(*directory_hash_id(self._get_object_id(obj))) # Optionally append extra_dir if extra_dir is not None: @@ -475,7 +475,7 @@ class IRODSObjectStore(DiskObjectStore, CloudConfigMixin): os.makedirs(cache_dir) if not dir_only: - rel_path = os.path.join(rel_path, alt_name if alt_name else "dataset_%s.dat" % obj.id) + rel_path = os.path.join(rel_path, alt_name if alt_name else "dataset_%s.dat" % self._get_object_id(obj)) open(os.path.join(self.staging_path, rel_path), 'w').close() self._push_to_irods(rel_path, from_string='') diff --git a/lib/galaxy/objectstore/pithos.py b/lib/galaxy/objectstore/pithos.py index 2d658f0133b..573ab053e84 100644 --- a/lib/galaxy/objectstore/pithos.py +++ b/lib/galaxy/objectstore/pithos.py @@ -163,7 +163,7 @@ class PithosObjectStore(ConcreteObjectStore): # alt_name can contain parent directory references, but S3 will not # follow them, so if they are valid we normalize them out alt_name = os.path.normpath(alt_name) - rel_path = os.path.join(*directory_hash_id(obj.id)) + rel_path = os.path.join(*directory_hash_id(self._get_object_id(obj))) if extra_dir is not None: if extra_dir_at_root: rel_path = os.path.join(extra_dir, rel_path) @@ -172,7 +172,7 @@ class PithosObjectStore(ConcreteObjectStore): # for JOB_WORK directory if obj_dir: - rel_path = os.path.join(rel_path, str(obj.id)) + rel_path = os.path.join(rel_path, str(self._get_object_id(obj))) if base_dir: base = self.extra_dirs.get(base_dir) return os.path.join(base, rel_path) @@ -181,7 +181,7 @@ class PithosObjectStore(ConcreteObjectStore): rel_path = '{0}/'.format(rel_path) if not dir_only: - an = alt_name if alt_name else 'dataset_{0}.dat'.format(obj.id) + an = alt_name if alt_name else 'dataset_{0}.dat'.format(self._get_object_id(obj)) rel_path = os.path.join(rel_path, an) return rel_path @@ -263,7 +263,7 @@ class PithosObjectStore(ConcreteObjectStore): alt_name = kwargs.get('alt_name', None) # Construct hashed path - rel_path = os.path.join(*directory_hash_id(obj.id)) + rel_path = os.path.join(*directory_hash_id(self._get_object_id(obj))) # Optionally append extra_dir if extra_dir is not None: @@ -283,7 +283,7 @@ class PithosObjectStore(ConcreteObjectStore): else: rel_path = os.path.join( rel_path, - alt_name if alt_name else 'dataset_{0}.dat'.format(obj.id)) + alt_name if alt_name else 'dataset_{0}.dat'.format(self._get_object_id(obj))) new_file = os.path.join(self.staging_path, rel_path) open(new_file, 'w').close() self.pithos.upload_from_string(rel_path, '') diff --git a/test/integration/test_datatype_upload_irods.py b/test/integration/test_datatype_upload_irods.py index 3001da5d3a1..0567e3e6cbb 100644 --- a/test/integration/test_datatype_upload_irods.py +++ b/test/integration/test_datatype_upload_irods.py @@ -78,6 +78,7 @@ class UploadTestDatatypeDataTestCase(UploadTestDatatypeDataTestCase): # config["metadata_strategy"] = "extended" config["outpus_to_working_dir"] = True config["retry_metadata_internally"] = False + config["object_store_store_by"] = "uuid" with open(config_path, "w") as f: f.write( OBJECT_STORE_CONFIG.safe_substitute( From 6af5b99153cc8d5fd8e5a3c54fa5c3d54617fed9 Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Mon, 13 Apr 2020 17:37:10 -0400 Subject: [PATCH 02/10] Added integration test for Distributed Object Store (DOS), before I start changing it. --- test/integration/test_datatype_upload_dos.py | 64 +++++++++++++++++++ .../integration/test_datatype_upload_irods.py | 4 +- 2 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 test/integration/test_datatype_upload_dos.py diff --git a/test/integration/test_datatype_upload_dos.py b/test/integration/test_datatype_upload_dos.py new file mode 100644 index 00000000000..de1ca7e7e00 --- /dev/null +++ b/test/integration/test_datatype_upload_dos.py @@ -0,0 +1,64 @@ + +import os +import string + +import pytest + +from galaxy_test.driver import integration_util +from .test_datatype_upload import ( + TEST_CASES, + upload_datatype_helper, + UploadTestDatatypeDataTestCase +) + +SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) +# Run test for only the first 10 test files +DOS_TEST_CASES = dict(list(TEST_CASES.items())[0:10]) +OBJECT_STORE_CONFIG = string.Template(""" + + + + + + + + + + + + + + +""") + + +class UploadTestDatatypeDataTestCaseDos(UploadTestDatatypeDataTestCase): + + @classmethod + def handle_galaxy_config_kwds(cls, config): + temp_directory = cls._test_driver.mkdtemp() + print('temp_directory:', temp_directory) + cls.object_stores_parent = temp_directory + config_path = os.path.join(temp_directory, "object_store_conf.xml") + # This doesn't quite work yet, fails with extra_files_path + # config["metadata_strategy"] = "extended" + config["outpus_to_working_dir"] = True + config["retry_metadata_internally"] = False + config["object_store_store_by"] = "uuid" + with open(config_path, "w") as f: + f.write( + OBJECT_STORE_CONFIG.safe_substitute( + { + "temp_directory": temp_directory, + } + ) + ) + config["object_store_config_file"] = config_path + + +instance = integration_util.integration_module_instance(UploadTestDatatypeDataTestCaseDos) + + +@pytest.mark.parametrize('test_data', DOS_TEST_CASES.values(), ids=list(DOS_TEST_CASES.keys())) +def test_upload_datatype_dos(instance, test_data, temp_file): + upload_datatype_helper(instance, test_data, temp_file) diff --git a/test/integration/test_datatype_upload_irods.py b/test/integration/test_datatype_upload_irods.py index 0567e3e6cbb..088c8c758b5 100644 --- a/test/integration/test_datatype_upload_irods.py +++ b/test/integration/test_datatype_upload_irods.py @@ -56,7 +56,7 @@ def stop_irods(container_name): subprocess.check_call(['docker', 'rm', '-f', container_name]) -class UploadTestDatatypeDataTestCase(UploadTestDatatypeDataTestCase): +class UploadTestDatatypeDataTestCaseIrods(UploadTestDatatypeDataTestCase): @classmethod def setUpClass(cls): @@ -97,7 +97,7 @@ class UploadTestDatatypeDataTestCase(UploadTestDatatypeDataTestCase): config["object_store_config_file"] = config_path -instance = integration_util.integration_module_instance(UploadTestDatatypeDataTestCase) +instance = integration_util.integration_module_instance(UploadTestDatatypeDataTestCaseIrods) @pytest.mark.parametrize('test_data', IRODS_TEST_CASES.values(), ids=list(IRODS_TEST_CASES.keys())) From 63f7c85c166263909ac4faab885abfe419878c39 Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Tue, 14 Apr 2020 16:38:26 -0400 Subject: [PATCH 03/10] Changed Distributed Object Store's constructor to accept non-disk-type object stores. Moved Mercurial imports into function to fix issue loading Pytests in VS code. --- lib/galaxy/objectstore/__init__.py | 16 ++++++---------- lib/tool_shed/webapp/model/__init__.py | 9 ++++----- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index 6bbbeda7d12..25c1ffefea4 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -669,26 +669,22 @@ class DistributedObjectStore(NestedObjectStore): self.global_max_percent_full = config_dict.get("global_max_percent_full", 0) random.seed() - backends_def = config_dict["backends"] - for backend_def in backends_def: + for backend_def in config_dict["backends"]: backened_id = backend_def["id"] - file_path = backend_def["files_dir"] - extra_dirs = backend_def.get("extra_dirs", []) maxpctfull = backend_def.get("max_percent_full", 0) weight = backend_def["weight"] - store_by = backend_def.get("store_by") - disk_config_dict = dict(files_dir=file_path, extra_dirs=extra_dirs) - if store_by is not None: - disk_config_dict['store_by'] = store_by - self.backends[backened_id] = DiskObjectStore(config, disk_config_dict) + + backend = build_object_store_from_config(config, config_dict=backend_def, fsmon=fsmon) + + self.backends[backened_id] = backend self.max_percent_full[backened_id] = maxpctfull - log.debug("Loaded disk backend '%s' with weight %s and file_path: %s" % (backened_id, weight, file_path)) for i in range(0, weight): # The simplest way to do weighting: add backend ids to a # sequence the number of times equalling weight, then randomly # choose a backend from that sequence at creation self.weighted_backend_ids.append(backened_id) + self.original_weighted_backend_ids = self.weighted_backend_ids self.sleeper = None diff --git a/lib/tool_shed/webapp/model/__init__.py b/lib/tool_shed/webapp/model/__init__.py index d1981ed661a..f087bcfcef3 100644 --- a/lib/tool_shed/webapp/model/__init__.py +++ b/lib/tool_shed/webapp/model/__init__.py @@ -6,11 +6,6 @@ from datetime import ( timedelta ) -from mercurial import ( - hg, - ui -) - import tool_shed.repository_types.util as rt_util from galaxy import util from galaxy.model.orm.now import now @@ -202,6 +197,10 @@ class Repository(Dictifiable): @property def hg_repo(self): + from mercurial import ( + hg, + ui + ) if not WEAK_HG_REPO_CACHE.get(self): WEAK_HG_REPO_CACHE[self] = hg.cachedlocalrepo(hg.repository(ui.ui(), self.repo_path().encode('utf-8'))) return WEAK_HG_REPO_CACHE[self].fetch()[0] From f6a5b2f4793975efbe8e3a61baf1b9e60018652e Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Wed, 15 Apr 2020 13:52:00 -0400 Subject: [PATCH 04/10] Updated DOS to support various backend object stores. Added a couple of DOS tests for 2 disk backends and 1 disk/1 irods backends. --- lib/galaxy/objectstore/__init__.py | 41 +++---- test/integration/test_datatype_upload_dos.py | 109 ++++++++++++++++++- 2 files changed, 118 insertions(+), 32 deletions(-) diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index 25c1ffefea4..cfddfc0ce4b 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -708,33 +708,22 @@ class DistributedObjectStore(NestedObjectStore): 'backends': backends, } - for elem in [e for e in backends_root if e.tag == 'backend']: - id = elem.get('id') - weight = int(elem.get('weight', 1)) - maxpctfull = float(elem.get('maxpctfull', 0)) - elem_type = elem.get('type', 'disk') - store_by = elem.get('store_by', None) - if elem_type: - path = None - extra_dirs = [] - for sub in elem: - if sub.tag == 'files_dir': - path = sub.get('path') - elif sub.tag == 'extra_dir': - type = sub.get('type') - extra_dirs.append({"type": type, "path": sub.get('path')}) + for b in config_xml.find('backends'): + store_id = b.get("id") + store_weight = int(b.get("weight", 1)) + store_maxpctfull = float(b.get('maxpctfull', 0)) + store_type = b.get("type", "disk") + store_by = b.get('store_by', None) - backend_dict = { - 'id': id, - 'weight': weight, - 'max_percent_full': maxpctfull, - 'files_dir': path, - 'extra_dirs': extra_dirs, - 'type': elem_type, - } - if store_by is not None: - backend_dict['store_by'] = store_by - backends.append(backend_dict) + objectstore_class, _ = type_to_object_store_class(store_type) + backend_config_dict = objectstore_class.parse_xml(b) + backend_config_dict["id"] = store_id + backend_config_dict["weight"] = store_weight + backend_config_dict["max_percent_full"] = store_maxpctfull + backend_config_dict["type"] = store_type + if store_by is not None: + backend_config_dict["store_by"] = store_by + backends.append(backend_config_dict) return config_dict diff --git a/test/integration/test_datatype_upload_dos.py b/test/integration/test_datatype_upload_dos.py index de1ca7e7e00..e91a5add7de 100644 --- a/test/integration/test_datatype_upload_dos.py +++ b/test/integration/test_datatype_upload_dos.py @@ -1,6 +1,8 @@ import os import string +import subprocess +import time import pytest @@ -12,9 +14,16 @@ from .test_datatype_upload import ( ) SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) +IRODS_OBJECT_STORE_HOST = os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_HOST', 'localhost') +IRODS_OBJECT_STORE_PORT = int(os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_PORT', 1247)) +IRODS_OBJECT_STORE_TIMEOUT = int(os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_TIMEOUT', 30)) +IRODS_OBJECT_STORE_USERNAME = os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_USERNAME', 'rods') +IRODS_OBJECT_STORE_PASSWORD = os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_PASSWORD', 'rods') +IRODS_OBJECT_STORE_RESOURCE = os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_RESOURCE', 'demoResc') +IRODS_OBJECT_STORE_ZONE = os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_ZONE', 'tempZone') # Run test for only the first 10 test files DOS_TEST_CASES = dict(list(TEST_CASES.items())[0:10]) -OBJECT_STORE_CONFIG = string.Template(""" +DISK_OBJECT_STORE_CONFIG = string.Template(""" @@ -30,9 +39,49 @@ OBJECT_STORE_CONFIG = string.Template(""" """) +IRODS__OBJECT_STORE_CONFIG = string.Template(""" + + + + + + + + + + + + + + + + + + +""") -class UploadTestDatatypeDataTestCaseDos(UploadTestDatatypeDataTestCase): +def start_irods(container_name): + irods_start_args = [ + 'docker', + 'run', + '-p', + '1247:1247', + '-d', + '--name', + container_name, + 'kxk302/irods-server:0.1'] + subprocess.check_call(irods_start_args) + + # Sleep so integration test's iRODS instance is up and running + time.sleep(20) + + +def stop_irods(container_name): + subprocess.check_call(['docker', 'rm', '-f', container_name]) + + +class UploadTestDosDiskAndDiskTestCase(UploadTestDatatypeDataTestCase): @classmethod def handle_galaxy_config_kwds(cls, config): @@ -47,7 +96,7 @@ class UploadTestDatatypeDataTestCaseDos(UploadTestDatatypeDataTestCase): config["object_store_store_by"] = "uuid" with open(config_path, "w") as f: f.write( - OBJECT_STORE_CONFIG.safe_substitute( + DISK_OBJECT_STORE_CONFIG.safe_substitute( { "temp_directory": temp_directory, } @@ -56,9 +105,57 @@ class UploadTestDatatypeDataTestCaseDos(UploadTestDatatypeDataTestCase): config["object_store_config_file"] = config_path -instance = integration_util.integration_module_instance(UploadTestDatatypeDataTestCaseDos) +class UploadTestDosIrodsAndDiskTestCase(UploadTestDatatypeDataTestCase): + + @classmethod + def handle_galaxy_config_kwds(cls, config): + temp_directory = cls._test_driver.mkdtemp() + print('temp_directory:', temp_directory) + cls.object_stores_parent = temp_directory + config_path = os.path.join(temp_directory, "object_store_conf.xml") + # This doesn't quite work yet, fails with extra_files_path + # config["metadata_strategy"] = "extended" + config["outpus_to_working_dir"] = True + config["retry_metadata_internally"] = False + config["object_store_store_by"] = "uuid" + with open(config_path, "w") as f: + f.write( + IRODS__OBJECT_STORE_CONFIG.safe_substitute( + { + "temp_directory": temp_directory, + "host": IRODS_OBJECT_STORE_HOST, + "port": IRODS_OBJECT_STORE_PORT, + "timeout": IRODS_OBJECT_STORE_TIMEOUT, + "username": IRODS_OBJECT_STORE_USERNAME, + "password": IRODS_OBJECT_STORE_PASSWORD, + "resource": IRODS_OBJECT_STORE_RESOURCE, + "zone": IRODS_OBJECT_STORE_ZONE + } + ) + ) + config["object_store_config_file"] = config_path + + @classmethod + def setUpClass(cls): + cls.container_name = "%s_container" % cls.__name__ + start_irods(cls.container_name) + super(UploadTestDosIrodsAndDiskTestCase, cls).setUpClass() + + @classmethod + def tearDownClass(cls): + # stop_irods(cls.container_name) + super(UploadTestDosIrodsAndDiskTestCase, cls).tearDownClass() + + +instance1 = integration_util.integration_module_instance(UploadTestDosDiskAndDiskTestCase) +instance2 = integration_util.integration_module_instance(UploadTestDosIrodsAndDiskTestCase) @pytest.mark.parametrize('test_data', DOS_TEST_CASES.values(), ids=list(DOS_TEST_CASES.keys())) -def test_upload_datatype_dos(instance, test_data, temp_file): - upload_datatype_helper(instance, test_data, temp_file) +def test_upload_datatype_dos_disk_and_disk(instance1, test_data, temp_file): + upload_datatype_helper(instance1, test_data, temp_file) + + +@pytest.mark.parametrize('test_data', DOS_TEST_CASES.values(), ids=list(DOS_TEST_CASES.keys())) +def test_upload_datatype_dos_irods_and_disk(instance2, test_data, temp_file): + upload_datatype_helper(instance2, test_data, temp_file) From 85ddda85d4f3f339ee415caaab57c2fdb7794cc8 Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Wed, 15 Apr 2020 13:53:10 -0400 Subject: [PATCH 05/10] Uncommented line that stops irods after test is finished. --- test/integration/test_datatype_upload_dos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/test_datatype_upload_dos.py b/test/integration/test_datatype_upload_dos.py index e91a5add7de..552c3d6707e 100644 --- a/test/integration/test_datatype_upload_dos.py +++ b/test/integration/test_datatype_upload_dos.py @@ -143,7 +143,7 @@ class UploadTestDosIrodsAndDiskTestCase(UploadTestDatatypeDataTestCase): @classmethod def tearDownClass(cls): - # stop_irods(cls.container_name) + stop_irods(cls.container_name) super(UploadTestDosIrodsAndDiskTestCase, cls).tearDownClass() From 4d7a6dbca364139b645d76bd8ff9eccadabf1b45 Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Wed, 15 Apr 2020 14:55:47 -0400 Subject: [PATCH 06/10] Deleted from_xml method in DOS class, as it seems it was redundant. Test pass with the default (super class) implementation. --- lib/galaxy/objectstore/__init__.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index cfddfc0ce4b..3a6fafde9e3 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -727,25 +727,6 @@ class DistributedObjectStore(NestedObjectStore): return config_dict - @classmethod - def from_xml(clazz, config, config_xml, fsmon=False): - legacy = False - if config_xml is None: - distributed_config = config.distributed_object_store_config_file - assert distributed_config is not None, \ - "distributed object store ('object_store = distributed') " \ - "requires a config file, please set one in " \ - "'distributed_object_store_config_file')" - - log.debug('Loading backends for distributed object store from %s', distributed_config) - config_xml = ElementTree.parse(distributed_config).getroot() - legacy = True - else: - log.debug('Loading backends for distributed object store from %s', config_xml.get('id')) - - config_dict = clazz.parse_xml(config_xml, legacy=legacy) - return clazz(config, config_dict, fsmon=fsmon) - def to_dict(self): as_dict = super(DistributedObjectStore, self).to_dict() as_dict["global_max_percent_full"] = self.global_max_percent_full From 15d3d6ad8bf04fc4a9d4396c0efe5b377360eb82 Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Fri, 17 Apr 2020 15:20:16 -0400 Subject: [PATCH 07/10] Changes for code review comments. --- lib/galaxy/objectstore/__init__.py | 21 +++++++++- test/integration/test_datatype_upload_dos.py | 38 ++----------------- .../integration/test_datatype_upload_irods.py | 4 +- 3 files changed, 26 insertions(+), 37 deletions(-) diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index 3a6fafde9e3..7a47dbffeb7 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -708,7 +708,7 @@ class DistributedObjectStore(NestedObjectStore): 'backends': backends, } - for b in config_xml.find('backends'): + for b in [e for e in backends_root if e.tag == 'backend']: store_id = b.get("id") store_weight = int(b.get("weight", 1)) store_maxpctfull = float(b.get('maxpctfull', 0)) @@ -727,6 +727,25 @@ class DistributedObjectStore(NestedObjectStore): return config_dict + @classmethod + def from_xml(clazz, config, config_xml, fsmon=False): + legacy = False + if config_xml is None: + distributed_config = config.distributed_object_store_config_file + assert distributed_config is not None, \ + "distributed object store ('object_store = distributed') " \ + "requires a config file, please set one in " \ + "'distributed_object_store_config_file')" + + log.debug('Loading backends for distributed object store from %s', distributed_config) + config_xml = ElementTree.parse(distributed_config).getroot() + legacy = True + else: + log.debug('Loading backends for distributed object store from %s', config_xml.get('id')) + + config_dict = clazz.parse_xml(config_xml, legacy=legacy) + return clazz(config, config_dict, fsmon=fsmon) + def to_dict(self): as_dict = super(DistributedObjectStore, self).to_dict() as_dict["global_max_percent_full"] = self.global_max_percent_full diff --git a/test/integration/test_datatype_upload_dos.py b/test/integration/test_datatype_upload_dos.py index 552c3d6707e..748841fce9d 100644 --- a/test/integration/test_datatype_upload_dos.py +++ b/test/integration/test_datatype_upload_dos.py @@ -1,8 +1,6 @@ import os import string -import subprocess -import time import pytest @@ -12,6 +10,9 @@ from .test_datatype_upload import ( upload_datatype_helper, UploadTestDatatypeDataTestCase ) +from .test_datatype_upload_irods import ( + IrodsUploadTestDatatypeDataTestCase +) SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) IRODS_OBJECT_STORE_HOST = os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_HOST', 'localhost') @@ -61,26 +62,6 @@ IRODS__OBJECT_STORE_CONFIG = string.Template(""" """) -def start_irods(container_name): - irods_start_args = [ - 'docker', - 'run', - '-p', - '1247:1247', - '-d', - '--name', - container_name, - 'kxk302/irods-server:0.1'] - subprocess.check_call(irods_start_args) - - # Sleep so integration test's iRODS instance is up and running - time.sleep(20) - - -def stop_irods(container_name): - subprocess.check_call(['docker', 'rm', '-f', container_name]) - - class UploadTestDosDiskAndDiskTestCase(UploadTestDatatypeDataTestCase): @classmethod @@ -105,7 +86,7 @@ class UploadTestDosDiskAndDiskTestCase(UploadTestDatatypeDataTestCase): config["object_store_config_file"] = config_path -class UploadTestDosIrodsAndDiskTestCase(UploadTestDatatypeDataTestCase): +class UploadTestDosIrodsAndDiskTestCase(IrodsUploadTestDatatypeDataTestCase): @classmethod def handle_galaxy_config_kwds(cls, config): @@ -135,17 +116,6 @@ class UploadTestDosIrodsAndDiskTestCase(UploadTestDatatypeDataTestCase): ) config["object_store_config_file"] = config_path - @classmethod - def setUpClass(cls): - cls.container_name = "%s_container" % cls.__name__ - start_irods(cls.container_name) - super(UploadTestDosIrodsAndDiskTestCase, cls).setUpClass() - - @classmethod - def tearDownClass(cls): - stop_irods(cls.container_name) - super(UploadTestDosIrodsAndDiskTestCase, cls).tearDownClass() - instance1 = integration_util.integration_module_instance(UploadTestDosDiskAndDiskTestCase) instance2 = integration_util.integration_module_instance(UploadTestDosIrodsAndDiskTestCase) diff --git a/test/integration/test_datatype_upload_irods.py b/test/integration/test_datatype_upload_irods.py index 088c8c758b5..c7cbf6f9cec 100644 --- a/test/integration/test_datatype_upload_irods.py +++ b/test/integration/test_datatype_upload_irods.py @@ -56,7 +56,7 @@ def stop_irods(container_name): subprocess.check_call(['docker', 'rm', '-f', container_name]) -class UploadTestDatatypeDataTestCaseIrods(UploadTestDatatypeDataTestCase): +class IrodsUploadTestDatatypeDataTestCase(UploadTestDatatypeDataTestCase): @classmethod def setUpClass(cls): @@ -97,7 +97,7 @@ class UploadTestDatatypeDataTestCaseIrods(UploadTestDatatypeDataTestCase): config["object_store_config_file"] = config_path -instance = integration_util.integration_module_instance(UploadTestDatatypeDataTestCaseIrods) +instance = integration_util.integration_module_instance(IrodsUploadTestDatatypeDataTestCase) @pytest.mark.parametrize('test_data', IRODS_TEST_CASES.values(), ids=list(IRODS_TEST_CASES.keys())) From 56346f8705424a168edcd7ba631643abcc47d04a Mon Sep 17 00:00:00 2001 From: kxk302 Date: Tue, 21 Apr 2020 09:51:37 -0400 Subject: [PATCH 08/10] Update test/integration/test_datatype_upload_dos.py Co-Authored-By: Marius van den Beek --- test/integration/test_datatype_upload_dos.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/integration/test_datatype_upload_dos.py b/test/integration/test_datatype_upload_dos.py index 748841fce9d..4c160d14979 100644 --- a/test/integration/test_datatype_upload_dos.py +++ b/test/integration/test_datatype_upload_dos.py @@ -1,4 +1,3 @@ - import os import string From 98bdc8ddcb21666eb6a19d591c55d0209478862c Mon Sep 17 00:00:00 2001 From: Kaivan Kamali Date: Tue, 21 Apr 2020 12:58:44 -0400 Subject: [PATCH 09/10] Test clean up per code review feedback. --- test/integration/test_datatype_upload.py | 2 ++ test/integration/test_datatype_upload_dos.py | 34 ++++++++----------- .../integration/test_datatype_upload_irods.py | 11 +++--- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/test/integration/test_datatype_upload.py b/test/integration/test_datatype_upload.py index 0250d3d3a6d..c90ce258302 100644 --- a/test/integration/test_datatype_upload.py +++ b/test/integration/test_datatype_upload.py @@ -42,6 +42,8 @@ def collect_test_data(registry): class UploadTestDatatypeDataTestCase(BaseUploadContentConfigurationInstance): framework_tool_and_types = False datatypes_conf_override = DATATYPES_CONFIG + object_store_config = None + object_store_config_path = None instance = integration_util.integration_module_instance(UploadTestDatatypeDataTestCase) diff --git a/test/integration/test_datatype_upload_dos.py b/test/integration/test_datatype_upload_dos.py index 748841fce9d..650bd736639 100644 --- a/test/integration/test_datatype_upload_dos.py +++ b/test/integration/test_datatype_upload_dos.py @@ -40,7 +40,7 @@ DISK_OBJECT_STORE_CONFIG = string.Template(""" """) -IRODS__OBJECT_STORE_CONFIG = string.Template(""" +IRODS_OBJECT_STORE_CONFIG = string.Template(""" @@ -64,46 +64,41 @@ IRODS__OBJECT_STORE_CONFIG = string.Template(""" class UploadTestDosDiskAndDiskTestCase(UploadTestDatatypeDataTestCase): + UploadTestDatatypeDataTestCase.object_store_config = DISK_OBJECT_STORE_CONFIG + @classmethod def handle_galaxy_config_kwds(cls, config): + super(UploadTestDosDiskAndDiskTestCase, cls).handle_galaxy_config_kwds(config) temp_directory = cls._test_driver.mkdtemp() - print('temp_directory:', temp_directory) cls.object_stores_parent = temp_directory - config_path = os.path.join(temp_directory, "object_store_conf.xml") + cls.object_store_config_path = os.path.join(temp_directory, "object_store_conf.xml") # This doesn't quite work yet, fails with extra_files_path # config["metadata_strategy"] = "extended" config["outpus_to_working_dir"] = True config["retry_metadata_internally"] = False config["object_store_store_by"] = "uuid" - with open(config_path, "w") as f: + with open(cls.object_store_config_path, "w") as f: f.write( - DISK_OBJECT_STORE_CONFIG.safe_substitute( + UploadTestDatatypeDataTestCase.object_store_config.safe_substitute( { - "temp_directory": temp_directory, + "temp_directory": cls.object_stores_parent, } ) ) - config["object_store_config_file"] = config_path class UploadTestDosIrodsAndDiskTestCase(IrodsUploadTestDatatypeDataTestCase): + UploadTestDatatypeDataTestCase.object_store_config = IRODS_OBJECT_STORE_CONFIG + @classmethod def handle_galaxy_config_kwds(cls, config): - temp_directory = cls._test_driver.mkdtemp() - print('temp_directory:', temp_directory) - cls.object_stores_parent = temp_directory - config_path = os.path.join(temp_directory, "object_store_conf.xml") - # This doesn't quite work yet, fails with extra_files_path - # config["metadata_strategy"] = "extended" - config["outpus_to_working_dir"] = True - config["retry_metadata_internally"] = False - config["object_store_store_by"] = "uuid" - with open(config_path, "w") as f: + super(UploadTestDosIrodsAndDiskTestCase, cls).handle_galaxy_config_kwds(config) + with open(cls.object_store_config_path, "w") as f: f.write( - IRODS__OBJECT_STORE_CONFIG.safe_substitute( + UploadTestDatatypeDataTestCase.object_store_config.safe_substitute( { - "temp_directory": temp_directory, + "temp_directory": cls.object_stores_parent, "host": IRODS_OBJECT_STORE_HOST, "port": IRODS_OBJECT_STORE_PORT, "timeout": IRODS_OBJECT_STORE_TIMEOUT, @@ -114,7 +109,6 @@ class UploadTestDosIrodsAndDiskTestCase(IrodsUploadTestDatatypeDataTestCase): } ) ) - config["object_store_config_file"] = config_path instance1 = integration_util.integration_module_instance(UploadTestDosDiskAndDiskTestCase) diff --git a/test/integration/test_datatype_upload_irods.py b/test/integration/test_datatype_upload_irods.py index c7cbf6f9cec..18fabad3aa4 100644 --- a/test/integration/test_datatype_upload_irods.py +++ b/test/integration/test_datatype_upload_irods.py @@ -58,6 +58,8 @@ def stop_irods(container_name): class IrodsUploadTestDatatypeDataTestCase(UploadTestDatatypeDataTestCase): + UploadTestDatatypeDataTestCase.object_store_config = OBJECT_STORE_CONFIG + @classmethod def setUpClass(cls): cls.container_name = "%s_container" % cls.__name__ @@ -71,17 +73,18 @@ class IrodsUploadTestDatatypeDataTestCase(UploadTestDatatypeDataTestCase): @classmethod def handle_galaxy_config_kwds(cls, config): + super(IrodsUploadTestDatatypeDataTestCase, cls).handle_galaxy_config_kwds(config) temp_directory = cls._test_driver.mkdtemp() cls.object_stores_parent = temp_directory - config_path = os.path.join(temp_directory, "object_store_conf.xml") + cls.object_store_config_path = os.path.join(temp_directory, "object_store_conf.xml") # This doesn't quite work yet, fails with extra_files_path # config["metadata_strategy"] = "extended" config["outpus_to_working_dir"] = True config["retry_metadata_internally"] = False config["object_store_store_by"] = "uuid" - with open(config_path, "w") as f: + with open(cls.object_store_config_path, "w") as f: f.write( - OBJECT_STORE_CONFIG.safe_substitute( + UploadTestDatatypeDataTestCase.object_store_config.safe_substitute( { "temp_directory": temp_directory, "host": OBJECT_STORE_HOST, @@ -94,7 +97,7 @@ class IrodsUploadTestDatatypeDataTestCase(UploadTestDatatypeDataTestCase): } ) ) - config["object_store_config_file"] = config_path + config["object_store_config_file"] = cls.object_store_config_path instance = integration_util.integration_module_instance(IrodsUploadTestDatatypeDataTestCase) From 4ba2ea27698a8c777598a3f3ab93e3264a6b017e Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 21 Apr 2020 21:19:15 +0200 Subject: [PATCH 10/10] Move object store test upload test cases to test/integration/objectstore --- .../test_objectstore_datatype_upload.py | 188 ++++++++++++++++++ test/integration/test_datatype_upload_dos.py | 124 ------------ .../integration/test_datatype_upload_irods.py | 108 ---------- 3 files changed, 188 insertions(+), 232 deletions(-) create mode 100644 test/integration/objectstore/test_objectstore_datatype_upload.py delete mode 100644 test/integration/test_datatype_upload_dos.py delete mode 100644 test/integration/test_datatype_upload_irods.py diff --git a/test/integration/objectstore/test_objectstore_datatype_upload.py b/test/integration/objectstore/test_objectstore_datatype_upload.py new file mode 100644 index 00000000000..8651813f8c4 --- /dev/null +++ b/test/integration/objectstore/test_objectstore_datatype_upload.py @@ -0,0 +1,188 @@ +"""Tests objectstores by exercising the datatype upload integration tests.""" +import os +import string +import subprocess +import time + +import pytest + +from galaxy_test.driver import integration_util +from ..test_datatype_upload import ( + TEST_CASES, + upload_datatype_helper, + UploadTestDatatypeDataTestCase +) + +SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) +IRODS_OBJECT_STORE_HOST = os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_HOST', 'localhost') +IRODS_OBJECT_STORE_PORT = int(os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_PORT', 1247)) +IRODS_OBJECT_STORE_TIMEOUT = int(os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_TIMEOUT', 30)) +IRODS_OBJECT_STORE_USERNAME = os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_USERNAME', 'rods') +IRODS_OBJECT_STORE_PASSWORD = os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_PASSWORD', 'rods') +IRODS_OBJECT_STORE_RESOURCE = os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_RESOURCE', 'demoResc') +IRODS_OBJECT_STORE_ZONE = os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_ZONE', 'tempZone') +# Run test for only the first 10 test files +TEST_CASES = dict(list(TEST_CASES.items())[0:10]) +DISTRIBUTED_OBJECT_STORE_CONFIG = string.Template(""" + + + + + + + + + + + + + + +""") +DISTRIBUTED_IRODS_OBJECT_STORE_CONFIG = string.Template(""" + + + + + + + + + + + + + + + + + + +""") +IRODS_OBJECT_STORE_CONFIG = string.Template(""" + + + + + + + + +""") + + +def check_container_active(container_name): + return subprocess.call([ + 'docker', + 'inspect', + '-f', + '{{.State.Running}}', + container_name + ]) == 0 + + +def start_irods(container_name): + if not check_container_active(container_name): + irods_start_args = [ + 'docker', + 'run', + '-p', + '1247:1247', + '-d', + '--name', + container_name, + 'kxk302/irods-server:0.1'] + subprocess.check_call(irods_start_args) + + # Sleep so integration test's iRODS instance is up and running + time.sleep(20) + + +def stop_irods(container_name): + if check_container_active(container_name): + subprocess.check_call(['docker', 'rm', '-f', container_name]) + + +class BaseObjectstoreUploadTest(UploadTestDatatypeDataTestCase): + + object_store_template = None + + @classmethod + def handle_galaxy_config_kwds(cls, config): + temp_directory = cls._test_driver.mkdtemp() + cls.object_stores_parent = temp_directory + cls.object_store_config_path = os.path.join(temp_directory, "object_store_conf.xml") + # This doesn't quite work yet, fails with extra_files_path + # config["metadata_strategy"] = "extended" + config["outpus_to_working_dir"] = True + config["retry_metadata_internally"] = False + config["object_store_store_by"] = "uuid" + with open(cls.object_store_config_path, "w") as f: + f.write(cls.object_store_template.safe_substitute(**cls.get_object_store_kwargs())) + + @classmethod + def get_object_store_kwargs(cls): + return {} + + +class IrodsUploadTestDatatypeDataTestCase(BaseObjectstoreUploadTest): + + object_store_template = IRODS_OBJECT_STORE_CONFIG + + @classmethod + def setUpClass(cls): + cls.container_name = "irods_integration_container" + start_irods(cls.container_name) + super(UploadTestDatatypeDataTestCase, cls).setUpClass() + + @classmethod + def tearDownClass(cls): + stop_irods(cls.container_name) + super(UploadTestDatatypeDataTestCase, cls).tearDownClass() + + @classmethod + def get_object_store_kwargs(cls): + return { + "temp_directory": cls.object_stores_parent, + "host": IRODS_OBJECT_STORE_HOST, + "port": IRODS_OBJECT_STORE_PORT, + "timeout": IRODS_OBJECT_STORE_TIMEOUT, + "username": IRODS_OBJECT_STORE_USERNAME, + "password": IRODS_OBJECT_STORE_PASSWORD, + "resource": IRODS_OBJECT_STORE_RESOURCE, + "zone": IRODS_OBJECT_STORE_ZONE + } + + +class UploadTestDosDiskAndDiskTestCase(BaseObjectstoreUploadTest): + + object_store_template = DISTRIBUTED_OBJECT_STORE_CONFIG + + @classmethod + def get_object_store_kwargs(cls): + return {'temp_directory': cls.object_stores_parent} + + +class UploadTestDosIrodsAndDiskTestCase(IrodsUploadTestDatatypeDataTestCase): + + object_store_template = DISTRIBUTED_IRODS_OBJECT_STORE_CONFIG + + +distributed_instance = integration_util.integration_module_instance(UploadTestDosDiskAndDiskTestCase) +irods_instance = integration_util.integration_module_instance(IrodsUploadTestDatatypeDataTestCase) +distributed_and_irods_instance = integration_util.integration_module_instance(UploadTestDosIrodsAndDiskTestCase) + + +@pytest.mark.parametrize('test_data', TEST_CASES.values(), ids=list(TEST_CASES.keys())) +def test_upload_datatype_dos_disk_and_disk(distributed_instance, test_data, temp_file): + upload_datatype_helper(distributed_instance, test_data, temp_file) + + +@pytest.mark.parametrize('test_data', TEST_CASES.values(), ids=list(TEST_CASES.keys())) +def test_upload_datatype_irods(irods_instance, test_data, temp_file): + upload_datatype_helper(irods_instance, test_data, temp_file) + + +@pytest.mark.parametrize('test_data', TEST_CASES.values(), ids=list(TEST_CASES.keys())) +def test_upload_datatype_dos_irods_and_disk(distributed_and_irods_instance, test_data, temp_file): + upload_datatype_helper(distributed_and_irods_instance, test_data, temp_file) diff --git a/test/integration/test_datatype_upload_dos.py b/test/integration/test_datatype_upload_dos.py deleted file mode 100644 index bbabbdb228c..00000000000 --- a/test/integration/test_datatype_upload_dos.py +++ /dev/null @@ -1,124 +0,0 @@ -import os -import string - -import pytest - -from galaxy_test.driver import integration_util -from .test_datatype_upload import ( - TEST_CASES, - upload_datatype_helper, - UploadTestDatatypeDataTestCase -) -from .test_datatype_upload_irods import ( - IrodsUploadTestDatatypeDataTestCase -) - -SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) -IRODS_OBJECT_STORE_HOST = os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_HOST', 'localhost') -IRODS_OBJECT_STORE_PORT = int(os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_PORT', 1247)) -IRODS_OBJECT_STORE_TIMEOUT = int(os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_TIMEOUT', 30)) -IRODS_OBJECT_STORE_USERNAME = os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_USERNAME', 'rods') -IRODS_OBJECT_STORE_PASSWORD = os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_PASSWORD', 'rods') -IRODS_OBJECT_STORE_RESOURCE = os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_RESOURCE', 'demoResc') -IRODS_OBJECT_STORE_ZONE = os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_ZONE', 'tempZone') -# Run test for only the first 10 test files -DOS_TEST_CASES = dict(list(TEST_CASES.items())[0:10]) -DISK_OBJECT_STORE_CONFIG = string.Template(""" - - - - - - - - - - - - - - -""") -IRODS_OBJECT_STORE_CONFIG = string.Template(""" - - - - - - - - - - - - - - - - - - -""") - - -class UploadTestDosDiskAndDiskTestCase(UploadTestDatatypeDataTestCase): - - UploadTestDatatypeDataTestCase.object_store_config = DISK_OBJECT_STORE_CONFIG - - @classmethod - def handle_galaxy_config_kwds(cls, config): - super(UploadTestDosDiskAndDiskTestCase, cls).handle_galaxy_config_kwds(config) - temp_directory = cls._test_driver.mkdtemp() - cls.object_stores_parent = temp_directory - cls.object_store_config_path = os.path.join(temp_directory, "object_store_conf.xml") - # This doesn't quite work yet, fails with extra_files_path - # config["metadata_strategy"] = "extended" - config["outpus_to_working_dir"] = True - config["retry_metadata_internally"] = False - config["object_store_store_by"] = "uuid" - with open(cls.object_store_config_path, "w") as f: - f.write( - UploadTestDatatypeDataTestCase.object_store_config.safe_substitute( - { - "temp_directory": cls.object_stores_parent, - } - ) - ) - - -class UploadTestDosIrodsAndDiskTestCase(IrodsUploadTestDatatypeDataTestCase): - - UploadTestDatatypeDataTestCase.object_store_config = IRODS_OBJECT_STORE_CONFIG - - @classmethod - def handle_galaxy_config_kwds(cls, config): - super(UploadTestDosIrodsAndDiskTestCase, cls).handle_galaxy_config_kwds(config) - with open(cls.object_store_config_path, "w") as f: - f.write( - UploadTestDatatypeDataTestCase.object_store_config.safe_substitute( - { - "temp_directory": cls.object_stores_parent, - "host": IRODS_OBJECT_STORE_HOST, - "port": IRODS_OBJECT_STORE_PORT, - "timeout": IRODS_OBJECT_STORE_TIMEOUT, - "username": IRODS_OBJECT_STORE_USERNAME, - "password": IRODS_OBJECT_STORE_PASSWORD, - "resource": IRODS_OBJECT_STORE_RESOURCE, - "zone": IRODS_OBJECT_STORE_ZONE - } - ) - ) - - -instance1 = integration_util.integration_module_instance(UploadTestDosDiskAndDiskTestCase) -instance2 = integration_util.integration_module_instance(UploadTestDosIrodsAndDiskTestCase) - - -@pytest.mark.parametrize('test_data', DOS_TEST_CASES.values(), ids=list(DOS_TEST_CASES.keys())) -def test_upload_datatype_dos_disk_and_disk(instance1, test_data, temp_file): - upload_datatype_helper(instance1, test_data, temp_file) - - -@pytest.mark.parametrize('test_data', DOS_TEST_CASES.values(), ids=list(DOS_TEST_CASES.keys())) -def test_upload_datatype_dos_irods_and_disk(instance2, test_data, temp_file): - upload_datatype_helper(instance2, test_data, temp_file) diff --git a/test/integration/test_datatype_upload_irods.py b/test/integration/test_datatype_upload_irods.py deleted file mode 100644 index 18fabad3aa4..00000000000 --- a/test/integration/test_datatype_upload_irods.py +++ /dev/null @@ -1,108 +0,0 @@ - -import os -import string -import subprocess -import time - -import pytest - -from galaxy_test.driver import integration_util -from .test_datatype_upload import ( - TEST_CASES, - upload_datatype_helper, - UploadTestDatatypeDataTestCase -) - -SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) -# Run test for only the first 10 test files -IRODS_TEST_CASES = dict(list(TEST_CASES.items())[0:10]) -OBJECT_STORE_HOST = os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_HOST', 'localhost') -OBJECT_STORE_PORT = int(os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_PORT', 1247)) -OBJECT_STORE_TIMEOUT = int(os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_TIMEOUT', 30)) -OBJECT_STORE_USERNAME = os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_USERNAME', 'rods') -OBJECT_STORE_PASSWORD = os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_PASSWORD', 'rods') -OBJECT_STORE_RESOURCE = os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_RESOURCE', 'demoResc') -OBJECT_STORE_ZONE = os.environ.get('GALAXY_INTEGRATION_IRODS_OBJECT_STORE_ZONE', 'tempZone') -OBJECT_STORE_CONFIG = string.Template(""" - - - - - - - - - -""") - - -def start_irods(container_name): - irods_start_args = [ - 'docker', - 'run', - '-p', - '1247:1247', - '-d', - '--name', - container_name, - 'kxk302/irods-server:0.1'] - subprocess.check_call(irods_start_args) - - # Sleep so integration test's iRODS instance is up and running - time.sleep(20) - - -def stop_irods(container_name): - subprocess.check_call(['docker', 'rm', '-f', container_name]) - - -class IrodsUploadTestDatatypeDataTestCase(UploadTestDatatypeDataTestCase): - - UploadTestDatatypeDataTestCase.object_store_config = OBJECT_STORE_CONFIG - - @classmethod - def setUpClass(cls): - cls.container_name = "%s_container" % cls.__name__ - start_irods(cls.container_name) - super(UploadTestDatatypeDataTestCase, cls).setUpClass() - - @classmethod - def tearDownClass(cls): - stop_irods(cls.container_name) - super(UploadTestDatatypeDataTestCase, cls).tearDownClass() - - @classmethod - def handle_galaxy_config_kwds(cls, config): - super(IrodsUploadTestDatatypeDataTestCase, cls).handle_galaxy_config_kwds(config) - temp_directory = cls._test_driver.mkdtemp() - cls.object_stores_parent = temp_directory - cls.object_store_config_path = os.path.join(temp_directory, "object_store_conf.xml") - # This doesn't quite work yet, fails with extra_files_path - # config["metadata_strategy"] = "extended" - config["outpus_to_working_dir"] = True - config["retry_metadata_internally"] = False - config["object_store_store_by"] = "uuid" - with open(cls.object_store_config_path, "w") as f: - f.write( - UploadTestDatatypeDataTestCase.object_store_config.safe_substitute( - { - "temp_directory": temp_directory, - "host": OBJECT_STORE_HOST, - "port": OBJECT_STORE_PORT, - "timeout": OBJECT_STORE_TIMEOUT, - "username": OBJECT_STORE_USERNAME, - "password": OBJECT_STORE_PASSWORD, - "resource": OBJECT_STORE_RESOURCE, - "zone": OBJECT_STORE_ZONE, - } - ) - ) - config["object_store_config_file"] = cls.object_store_config_path - - -instance = integration_util.integration_module_instance(IrodsUploadTestDatatypeDataTestCase) - - -@pytest.mark.parametrize('test_data', IRODS_TEST_CASES.values(), ids=list(IRODS_TEST_CASES.keys())) -def test_upload_datatype_irods(instance, test_data, temp_file): - upload_datatype_helper(instance, test_data, temp_file)