diff --git a/config/job_conf.xml.sample_advanced b/config/job_conf.xml.sample_advanced index 921cc063aec..361d3693526 100644 --- a/config/job_conf.xml.sample_advanced +++ b/config/job_conf.xml.sample_advanced @@ -652,6 +652,30 @@ 50 + + + burst + local + burst_if_size + 2 + queued + + + python + to_destination_if_size + + galaxycloudrunner.rules + 1g + pulsar + local + + integration.chained_dyndest_rules.module2 + from1 + + + + diff --git a/test/integration/chained_dyndest_rules/__init__.py b/test/integration/chained_dyndest_rules/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/integration/chained_dyndest_rules/module1/__init__.py b/test/integration/chained_dyndest_rules/module1/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/integration/chained_dyndest_rules/module1/rules.py b/test/integration/chained_dyndest_rules/module1/rules.py new file mode 100644 index 00000000000..8a70a8069e5 --- /dev/null +++ b/test/integration/chained_dyndest_rules/module1/rules.py @@ -0,0 +1,15 @@ +def dyndest_chain_1(): + # Check whether chaining dynamic job destinations work + return "dyn_dest2" + + +def dyndest_chain_2(): + # Return an invalid destination as this module's function + # should never be called + return "invalid_destination" + + +def dyndest_chain_3(): + # Return an invalid destination as this module's function + # should never be called + return "invalid_destination" diff --git a/test/integration/chained_dyndest_rules/module2/__init__.py b/test/integration/chained_dyndest_rules/module2/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/integration/chained_dyndest_rules/module2/rules.py b/test/integration/chained_dyndest_rules/module2/rules.py new file mode 100644 index 00000000000..afe0d6dcc0c --- /dev/null +++ b/test/integration/chained_dyndest_rules/module2/rules.py @@ -0,0 +1,23 @@ +from galaxy.jobs import JobDestination + + +def dyndest_chain_1(): + # Return an invalid destination as this module's function + # should never be called + return "invalid_destination" + + +def dyndest_chain_2(tmp_dir_prefix): + # Chain to yet a third + return JobDestination( + runner="dynamic", + params={'type': 'python', + 'function': 'dyndest_chain_3', + 'rules_module': 'integration.chained_dyndest_rules.module3', + 'tmp_dir_prefix_two': '%sand2' % tmp_dir_prefix}) + + +def dyndest_chain_3(): + # Return an invalid destination as this module's function + # should never be called + return "invalid_destination" diff --git a/test/integration/chained_dyndest_rules/module3/__init__.py b/test/integration/chained_dyndest_rules/module3/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/integration/chained_dyndest_rules/module3/rules.py b/test/integration/chained_dyndest_rules/module3/rules.py new file mode 100644 index 00000000000..72b3f9b68f5 --- /dev/null +++ b/test/integration/chained_dyndest_rules/module3/rules.py @@ -0,0 +1,19 @@ +from galaxy.jobs import JobDestination + + +def dyndest_chain_1(): + # Return an invalid destination as this module's function + # should never be called + return "invalid_destination" + + +def dyndest_chain_2(): + # Return an invalid destination as this module's function + # should never be called + return "invalid_destination" + + +def dyndest_chain_3(tmp_dir_prefix_two): + tmp_dir = '$(mktemp %sand3XXXXXXXXXXXX)' % tmp_dir_prefix_two + return JobDestination(runner="local", + params={'tmp_dir': tmp_dir}) diff --git a/test/integration/test_chained_dynamic_destinations.py b/test/integration/test_chained_dynamic_destinations.py new file mode 100644 index 00000000000..2e3b8c8ef23 --- /dev/null +++ b/test/integration/test_chained_dynamic_destinations.py @@ -0,0 +1,32 @@ +"""Integration tests for chained dynamic job destinations.""" + +import os +import tempfile + +from base.populators import ( + skip_without_tool, +) + +from .test_job_environments import BaseJobEnvironmentIntegrationTestCase + +SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) +CHAINED_DYNDESTS_JOB_CONFIG = os.path.join(SCRIPT_DIRECTORY, "chained_dyndest_job_conf.xml") + + +class ChainedDynamicDestinationIntegrationTestCase(BaseJobEnvironmentIntegrationTestCase): + + @classmethod + def handle_galaxy_config_kwds(cls, config): + cls.jobs_directory = tempfile.mkdtemp() + config["jobs_directory"] = cls.jobs_directory + config["job_config_file"] = CHAINED_DYNDESTS_JOB_CONFIG + + @skip_without_tool("job_environment_default") + def test_default_environment_1801(self): + job_env = self._run_and_get_environment_properties() + + # Since dynamic destinations compute final tmp_dir parameter to be + # $(mktemp from1and2and3XXXXXXXXXXXX), tmpdir should start + # with from1and2and3. + basename = os.path.basename(job_env.tmp) + assert basename.startswith("from1and2and3"), job_env.tmp diff --git a/test/unit/jobs/test_mapper.py b/test/unit/jobs/test_mapper.py index 30e02a77dc4..9f0e70d6bbc 100644 --- a/test/unit/jobs/test_mapper.py +++ b/test/unit/jobs/test_mapper.py @@ -36,6 +36,12 @@ def test_dynamic_mapping(): assert mapper.job_config.rule_response == "local_runner" +def test_chained_dynamic_mapping(): + mapper = __mapper(__dynamic_destination(dict(function="dynamic_chain_1"))) + assert mapper.get_job_destination({}) is DYNAMICALLY_GENERATED_DESTINATION + assert mapper.job_config.rule_response == "final_destination" + + def test_dynamic_mapping_priorities(): mapper = __mapper(__dynamic_destination(dict(function="tophat"))) assert mapper.get_job_destination({}) is DYNAMICALLY_GENERATED_DESTINATION @@ -97,6 +103,23 @@ def test_dynamic_mapping_rule_module_override(): assert mapper.job_config.rule_response == "new_rules_package" +def test_dynamic_mapping_externally_set_job_destination(): + mapper = __mapper(__dynamic_destination(dict(function="upload"))) + # Initially, the mapper should not have a cached destination + assert not hasattr(mapper, 'cached_job_destination') + # Overwrite with an externally set job destination + manually_set_destination = JobDestination(runner="dynamic") + mapper.cached_job_destination = manually_set_destination + destination = mapper.get_job_destination({}) + assert destination == manually_set_destination + assert mapper.cached_job_destination == manually_set_destination + # Force overwrite with mapper determined destination + mapper.cache_job_destination(None) + assert mapper.cached_job_destination is not None + assert mapper.cached_job_destination != manually_set_destination + assert mapper.job_config.rule_response == "local_runner" + + def __assert_mapper_errors_with_message(mapper, message): exception = None try: diff --git a/test/unit/jobs/test_rules/10_site.py b/test/unit/jobs/test_rules/10_site.py index a3578dac359..0712df11d87 100644 --- a/test/unit/jobs/test_rules/10_site.py +++ b/test/unit/jobs/test_rules/10_site.py @@ -1,3 +1,4 @@ +from galaxy.jobs import JobDestination def upload(): @@ -15,6 +16,20 @@ def tool1(): return 'tool1_dest_id' +def dynamic_chain_1(): + # Check whether chaining dynamic job destinations work + return JobDestination(runner="dynamic", + params={'type': 'python', + 'function': 'dynamic_chain_2', + 'test_param': 'my_test_param'}) + + +def dynamic_chain_2(test_param): + # Check whether chaining dynamic job destinations work + assert test_param == "my_test_param" + return "final_destination" + + def check_rule_params( job_id, tool,