From 52596d186e8a606ad42dc94df17ad51954711df6 Mon Sep 17 00:00:00 2001 From: vahid Date: Sat, 25 May 2019 16:36:50 -0700 Subject: [PATCH 01/11] Organize ObjectStore integration tests under a dedicated package. --- test/integration/objectstore/__init__.py | 0 .../{ => objectstore}/objectstore_selection_job_conf.xml | 0 .../objectstore_selection_job_resource_parameters_conf.xml | 0 test/integration/{ => objectstore}/test_objectstore_jobs.py | 0 test/integration/{ => objectstore}/test_objectstore_selection.py | 1 - 5 files changed, 1 deletion(-) create mode 100644 test/integration/objectstore/__init__.py rename test/integration/{ => objectstore}/objectstore_selection_job_conf.xml (100%) rename test/integration/{ => objectstore}/objectstore_selection_job_resource_parameters_conf.xml (100%) rename test/integration/{ => objectstore}/test_objectstore_jobs.py (100%) rename test/integration/{ => objectstore}/test_objectstore_selection.py (99%) diff --git a/test/integration/objectstore/__init__.py b/test/integration/objectstore/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/integration/objectstore_selection_job_conf.xml b/test/integration/objectstore/objectstore_selection_job_conf.xml similarity index 100% rename from test/integration/objectstore_selection_job_conf.xml rename to test/integration/objectstore/objectstore_selection_job_conf.xml diff --git a/test/integration/objectstore_selection_job_resource_parameters_conf.xml b/test/integration/objectstore/objectstore_selection_job_resource_parameters_conf.xml similarity index 100% rename from test/integration/objectstore_selection_job_resource_parameters_conf.xml rename to test/integration/objectstore/objectstore_selection_job_resource_parameters_conf.xml diff --git a/test/integration/test_objectstore_jobs.py b/test/integration/objectstore/test_objectstore_jobs.py similarity index 100% rename from test/integration/test_objectstore_jobs.py rename to test/integration/objectstore/test_objectstore_jobs.py diff --git a/test/integration/test_objectstore_selection.py b/test/integration/objectstore/test_objectstore_selection.py similarity index 99% rename from test/integration/test_objectstore_selection.py rename to test/integration/objectstore/test_objectstore_selection.py index a6f25eb8cf6..fcbe0aedcd2 100644 --- a/test/integration/test_objectstore_selection.py +++ b/test/integration/objectstore/test_objectstore_selection.py @@ -80,7 +80,6 @@ class ObjectStoreJobsIntegrationTestCase(integration_util.IntegrationTestCase): assert dynamic_s3 == files_dynamic_s3_count def test_tool_simple_constructs(self): - with self.dataset_populator.test_history() as history_id: def _run_tool(tool_id, inputs): From 6acb9e50ebb687b105e249296950efa5694bc0c0 Mon Sep 17 00:00:00 2001 From: vahid Date: Sat, 25 May 2019 17:20:00 -0700 Subject: [PATCH 02/11] Remove `objectstore` from its integration tests filenames. --- ...ectstore_selection_job_conf.xml => selection_job_conf.xml} | 0 ...rs_conf.xml => selection_job_resource_parameters_conf.xml} | 0 .../objectstore/{test_objectstore_jobs.py => test_jobs.py} | 0 .../{test_objectstore_selection.py => test_selection.py} | 4 ++-- 4 files changed, 2 insertions(+), 2 deletions(-) rename test/integration/objectstore/{objectstore_selection_job_conf.xml => selection_job_conf.xml} (100%) rename test/integration/objectstore/{objectstore_selection_job_resource_parameters_conf.xml => selection_job_resource_parameters_conf.xml} (100%) rename test/integration/objectstore/{test_objectstore_jobs.py => test_jobs.py} (100%) rename test/integration/objectstore/{test_objectstore_selection.py => test_selection.py} (97%) diff --git a/test/integration/objectstore/objectstore_selection_job_conf.xml b/test/integration/objectstore/selection_job_conf.xml similarity index 100% rename from test/integration/objectstore/objectstore_selection_job_conf.xml rename to test/integration/objectstore/selection_job_conf.xml diff --git a/test/integration/objectstore/objectstore_selection_job_resource_parameters_conf.xml b/test/integration/objectstore/selection_job_resource_parameters_conf.xml similarity index 100% rename from test/integration/objectstore/objectstore_selection_job_resource_parameters_conf.xml rename to test/integration/objectstore/selection_job_resource_parameters_conf.xml diff --git a/test/integration/objectstore/test_objectstore_jobs.py b/test/integration/objectstore/test_jobs.py similarity index 100% rename from test/integration/objectstore/test_objectstore_jobs.py rename to test/integration/objectstore/test_jobs.py diff --git a/test/integration/objectstore/test_objectstore_selection.py b/test/integration/objectstore/test_selection.py similarity index 97% rename from test/integration/objectstore/test_objectstore_selection.py rename to test/integration/objectstore/test_selection.py index fcbe0aedcd2..da2576ecf97 100644 --- a/test/integration/objectstore/test_objectstore_selection.py +++ b/test/integration/objectstore/test_selection.py @@ -9,8 +9,8 @@ from base.populators import ( ) SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) -JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "objectstore_selection_job_conf.xml") -JOB_RESOURCE_PARAMETERS_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "objectstore_selection_job_resource_parameters_conf.xml") +JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "selection_job_conf.xml") +JOB_RESOURCE_PARAMETERS_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "selection_job_resource_parameters_conf.xml") DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE = string.Template(""" From 275cc365a549364f982e80db04673d2b284222bb Mon Sep 17 00:00:00 2001 From: vahid Date: Sat, 25 May 2019 17:26:14 -0700 Subject: [PATCH 03/11] Move some code from test method to setUp method in OS integration tests. --- test/integration/objectstore/test_jobs.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/test/integration/objectstore/test_jobs.py b/test/integration/objectstore/test_jobs.py index c1db9c522e7..755c96cd4b0 100644 --- a/test/integration/objectstore/test_jobs.py +++ b/test/integration/objectstore/test_jobs.py @@ -55,8 +55,6 @@ class ObjectStoreJobsIntegrationTestCase(integration_util.IntegrationTestCase): def setUp(self): super(ObjectStoreJobsIntegrationTestCase, self).setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) - - def test_tool_simple_constructs(self): with self.dataset_populator.test_history() as history_id: hda1 = self.dataset_populator.new_dataset(history_id, content="1 2 3") create_10_inputs = { @@ -71,6 +69,25 @@ class ObjectStoreJobsIntegrationTestCase(integration_util.IntegrationTestCase): ) self.dataset_populator.wait_for_history(history_id) + def test_tool_simple_constructs(self): + """ + According to the ObjectStore configuration given in the + `DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE` variable, datasets + can be stored on three backends, named: + - primary/files1; + - primary/files2; + - secondary/files3. + + Objectstore _randomly_ distributes tools outputs on + `primary/files1` and `primary/files2`, and will use + `secondary/files3` and both `primary` backends fail. + + This test runs a tools that creates ten dummy datasets, + and asserts if ObjectStore correctly creates ten files + in `primary/files1` and `primary/files2`, and none in + `secondary/files3`, assuming it will not fail persisting + data in `primary` backend. + """ files_1_count = _files_count(self.files1_path) files_2_count = _files_count(self.files2_path) files_3_count = _files_count(self.files3_path) From 28c894391911085e7e40a2539aab07a9cbe3ae23 Mon Sep 17 00:00:00 2001 From: vahid Date: Sat, 25 May 2019 17:30:14 -0700 Subject: [PATCH 04/11] Assert the exact number of files in the OS jobs output than a range. --- test/integration/objectstore/test_jobs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/integration/objectstore/test_jobs.py b/test/integration/objectstore/test_jobs.py index 755c96cd4b0..f85aa4112c5 100644 --- a/test/integration/objectstore/test_jobs.py +++ b/test/integration/objectstore/test_jobs.py @@ -99,8 +99,7 @@ class ObjectStoreJobsIntegrationTestCase(integration_util.IntegrationTestCase): # stores (it will have either 10 or 11 depeending on whether the input was also # written there. The other disk store may or may not have the input file so should # have at most one file. - assert (files_1_count >= 10) or (files_2_count >= 10) - assert (files_1_count <= 1) or (files_2_count <= 1) + assert (files_1_count + files_2_count == 10) or (files_1_count + files_2_count == 11) # Other sanity checks on the test - just make sure the test was setup as intended # and not actually testing object store behavior. From 216f31a2f1df185c987b3db1068b3152322e22c8 Mon Sep 17 00:00:00 2001 From: vahid Date: Sat, 25 May 2019 17:31:01 -0700 Subject: [PATCH 05/11] Refactor a test name. --- test/integration/objectstore/test_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/objectstore/test_jobs.py b/test/integration/objectstore/test_jobs.py index f85aa4112c5..73ff924b978 100644 --- a/test/integration/objectstore/test_jobs.py +++ b/test/integration/objectstore/test_jobs.py @@ -69,7 +69,7 @@ class ObjectStoreJobsIntegrationTestCase(integration_util.IntegrationTestCase): ) self.dataset_populator.wait_for_history(history_id) - def test_tool_simple_constructs(self): + def test_files_count_in_each_objectstore_backend(self): """ According to the ObjectStore configuration given in the `DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE` variable, datasets From ffc8b87ee93305efdaf8289a3a8a69a10b73311b Mon Sep 17 00:00:00 2001 From: vahid Date: Sat, 25 May 2019 19:47:13 -0700 Subject: [PATCH 06/11] Extend ObjectStore integration test to assert the content of datasets. --- test/integration/objectstore/test_jobs.py | 30 ++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/test/integration/objectstore/test_jobs.py b/test/integration/objectstore/test_jobs.py index 73ff924b978..ae5cfecc0fe 100644 --- a/test/integration/objectstore/test_jobs.py +++ b/test/integration/objectstore/test_jobs.py @@ -34,6 +34,8 @@ DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE = string.Template(""" """) +TEST_INPUT_FILES_CONTENT = "1 2 3" + class ObjectStoreJobsIntegrationTestCase(integration_util.IntegrationTestCase): @@ -56,7 +58,7 @@ class ObjectStoreJobsIntegrationTestCase(integration_util.IntegrationTestCase): super(ObjectStoreJobsIntegrationTestCase, self).setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) with self.dataset_populator.test_history() as history_id: - hda1 = self.dataset_populator.new_dataset(history_id, content="1 2 3") + hda1 = self.dataset_populator.new_dataset(history_id, content=TEST_INPUT_FILES_CONTENT) create_10_inputs = { "input1": {"src": "hda", "id": hda1["id"]}, "input2": {"src": "hda", "id": hda1["id"]}, @@ -69,7 +71,7 @@ class ObjectStoreJobsIntegrationTestCase(integration_util.IntegrationTestCase): ) self.dataset_populator.wait_for_history(history_id) - def test_files_count_in_each_objectstore_backend(self): + def test_files_count_and_content_in_each_objectstore_backend(self): """ According to the ObjectStore configuration given in the `DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE` variable, datasets @@ -96,7 +98,7 @@ class ObjectStoreJobsIntegrationTestCase(integration_util.IntegrationTestCase): assert files_3_count == 0 # Ensure the 10 inputs were written to one of the distributed object store's disk - # stores (it will have either 10 or 11 depeending on whether the input was also + # stores (it will have either 10 or 11 depending on whether the input was also # written there. The other disk store may or may not have the input file so should # have at most one file. assert (files_1_count + files_2_count == 10) or (files_1_count + files_2_count == 11) @@ -106,6 +108,28 @@ class ObjectStoreJobsIntegrationTestCase(integration_util.IntegrationTestCase): assert (files_1_count <= 11) and (files_2_count <= 11) assert (files_1_count >= 0) and (files_2_count >= 0) + contents = [] + path1_files = _get_datasets_files_in_path(self.files1_path) + path2_files = _get_datasets_files_in_path(self.files2_path) + path3_files = _get_datasets_files_in_path(self.files3_path) + for filename in set().union(path1_files, path2_files, path3_files): + with open(filename) as f: + content = f.read().strip('\n').strip('\t') + if content != TEST_INPUT_FILES_CONTENT: + contents.append(content) + + for expected_content in range(1, 10): + assert str(expected_content) in contents + def _files_count(directory): return sum(len(files) for _, _, files in os.walk(directory)) + + +def _get_datasets_files_in_path(directory): + files = [] + for path, _, filename in os.walk(directory): + for f in filename: + if f.endswith(".dat"): + files.append(os.path.join(path, f)) + return files From 4b3dbc6bcb6ddf567b42e69359df41b9a78b0703 Mon Sep 17 00:00:00 2001 From: vahid Date: Mon, 27 May 2019 11:15:10 -0700 Subject: [PATCH 07/11] Add a todo comment. --- test/integration/objectstore/test_jobs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/integration/objectstore/test_jobs.py b/test/integration/objectstore/test_jobs.py index ae5cfecc0fe..7b08c78d349 100644 --- a/test/integration/objectstore/test_jobs.py +++ b/test/integration/objectstore/test_jobs.py @@ -108,6 +108,7 @@ class ObjectStoreJobsIntegrationTestCase(integration_util.IntegrationTestCase): assert (files_1_count <= 11) and (files_2_count <= 11) assert (files_1_count >= 0) and (files_2_count >= 0) + # TODO: ideally the following assertion should be separated in a different test method. contents = [] path1_files = _get_datasets_files_in_path(self.files1_path) path2_files = _get_datasets_files_in_path(self.files2_path) From 1dbca872d5719d25752b7563b5e24a3674bd7ae7 Mon Sep 17 00:00:00 2001 From: Vahid Date: Mon, 17 Jun 2019 09:07:22 -0700 Subject: [PATCH 08/11] Update test/integration/objectstore/test_jobs.py Co-Authored-By: Marius van den Beek --- test/integration/objectstore/test_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/objectstore/test_jobs.py b/test/integration/objectstore/test_jobs.py index 7b08c78d349..31a0010d3e3 100644 --- a/test/integration/objectstore/test_jobs.py +++ b/test/integration/objectstore/test_jobs.py @@ -82,7 +82,7 @@ class ObjectStoreJobsIntegrationTestCase(integration_util.IntegrationTestCase): Objectstore _randomly_ distributes tools outputs on `primary/files1` and `primary/files2`, and will use - `secondary/files3` and both `primary` backends fail. + `secondary/files3` if both `primary` backends fail. This test runs a tools that creates ten dummy datasets, and asserts if ObjectStore correctly creates ten files From a2bfba12a10b7d665bc4d0998ad98e732b23a61b Mon Sep 17 00:00:00 2001 From: Vahid Date: Mon, 17 Jun 2019 09:08:05 -0700 Subject: [PATCH 09/11] Update test/integration/objectstore/test_jobs.py Co-Authored-By: Marius van den Beek --- test/integration/objectstore/test_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/objectstore/test_jobs.py b/test/integration/objectstore/test_jobs.py index 31a0010d3e3..d386d3de0fe 100644 --- a/test/integration/objectstore/test_jobs.py +++ b/test/integration/objectstore/test_jobs.py @@ -113,7 +113,7 @@ class ObjectStoreJobsIntegrationTestCase(integration_util.IntegrationTestCase): path1_files = _get_datasets_files_in_path(self.files1_path) path2_files = _get_datasets_files_in_path(self.files2_path) path3_files = _get_datasets_files_in_path(self.files3_path) - for filename in set().union(path1_files, path2_files, path3_files): + for filename in path1_files + path2_files + path3_files: with open(filename) as f: content = f.read().strip('\n').strip('\t') if content != TEST_INPUT_FILES_CONTENT: From 428a29dac4fc8733e94d62015640aa32b77fe851 Mon Sep 17 00:00:00 2001 From: vahid Date: Mon, 17 Jun 2019 09:29:09 -0700 Subject: [PATCH 10/11] Remove an unnecessary tab strip. --- test/integration/objectstore/test_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/objectstore/test_jobs.py b/test/integration/objectstore/test_jobs.py index d386d3de0fe..c5f2770409d 100644 --- a/test/integration/objectstore/test_jobs.py +++ b/test/integration/objectstore/test_jobs.py @@ -115,7 +115,7 @@ class ObjectStoreJobsIntegrationTestCase(integration_util.IntegrationTestCase): path3_files = _get_datasets_files_in_path(self.files3_path) for filename in path1_files + path2_files + path3_files: with open(filename) as f: - content = f.read().strip('\n').strip('\t') + content = f.read().strip('\n') if content != TEST_INPUT_FILES_CONTENT: contents.append(content) From a4656c26548eb2218862a90fecf2f1541fb5c3ce Mon Sep 17 00:00:00 2001 From: vahid Date: Mon, 17 Jun 2019 09:46:22 -0700 Subject: [PATCH 11/11] Remove `\n` from test content strip. --- test/integration/objectstore/test_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/objectstore/test_jobs.py b/test/integration/objectstore/test_jobs.py index c5f2770409d..e81960a12bc 100644 --- a/test/integration/objectstore/test_jobs.py +++ b/test/integration/objectstore/test_jobs.py @@ -115,7 +115,7 @@ class ObjectStoreJobsIntegrationTestCase(integration_util.IntegrationTestCase): path3_files = _get_datasets_files_in_path(self.files3_path) for filename in path1_files + path2_files + path3_files: with open(filename) as f: - content = f.read().strip('\n') + content = f.read().strip() if content != TEST_INPUT_FILES_CONTENT: contents.append(content)