diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 7524e216adb..ba4740dbf56 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -939,6 +939,7 @@ class JobWrapper(HasResourceParameters): self.environment_variables: List[Dict[str, str]] = [] self.interactivetools: List[Dict[str, Any]] = [] self.command_line = None + self.version_command_line = None self._dependency_shell_commands = None # Tool versioning variables self.version_string = "" @@ -1192,7 +1193,7 @@ class JobWrapper(HasResourceParameters): tool_evaluator = self._get_tool_evaluator(job) compute_environment = compute_environment or self.default_compute_environment(job) tool_evaluator.set_compute_environment(compute_environment, get_special=get_special) - self.command_line, self.extra_filenames, self.environment_variables = tool_evaluator.build() + self.command_line, self.version_command_line, self.extra_filenames, self.environment_variables = tool_evaluator.build() job.command_line = self.command_line self.interactivetools = tool_evaluator.populate_interactivetools() self.app.interactivetool_manager.create_interactivetool(job, self.tool, self.interactivetools) @@ -1949,7 +1950,10 @@ class JobWrapper(HasResourceParameters): return has_output_limit or has_walltime_limit def get_command_line(self): - return self.command_line + """Return complete command line, including possible version command.""" + if self.remote_command_line: + return None + return f'{self.version_command_line or ""}{self.command_line}' def get_session_id(self): return self.session_id @@ -2305,7 +2309,7 @@ class TaskWrapper(JobWrapper): self.sa_session.flush() if not self.remote_command_line: - self.command_line, extra_filenames, self.environment_variables = tool_evaluator.build() + self.command_line, self.version_command_line, extra_filenames, self.environment_variables = tool_evaluator.build() self.extra_filenames.extend(extra_filenames) # Ensure galaxy_lib_dir is set in case there are any later chdirs diff --git a/lib/galaxy/metadata/set_metadata.py b/lib/galaxy/metadata/set_metadata.py index 90755ce774c..6012fdc45c0 100644 --- a/lib/galaxy/metadata/set_metadata.py +++ b/lib/galaxy/metadata/set_metadata.py @@ -281,7 +281,13 @@ def set_metadata_portable(): job.state = final_job_state if os.path.exists(tool_script_file): with open(tool_script_file) as command_fh: - job.command_line = command_fh.read().strip() + command_line_lines = [] + for i, line in enumerate(command_fh): + if i == 0 and line.endswith('COMMAND_VERSION 2>&1;'): + # Don't record version command as part of command line + continue + command_line_lines.append(line) + job.command_line = "".join(command_line_lines).strip() export_store.export_job(job, include_job_data=False) unnamed_id_to_path = {} diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py index 6f4eed1fa99..29d8c6b008a 100644 --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -91,6 +91,7 @@ class ToolEvaluator: self.param_dict: Dict[str, Any] = {} self.extra_filenames: List[str] = [] self.environment_variables: List[Dict[str, str]] = [] + self.version_command_line: Optional[str] = None self.command_line: Optional[str] = None def set_compute_environment(self, compute_environment: ComputeEnvironment, get_special: Optional[Callable] = None): @@ -445,8 +446,9 @@ class ToolEvaluator: global_tool_logs(self._build_config_files, config_file, "Building Config Files") global_tool_logs(self._build_param_file, config_file, 'Building Param File') global_tool_logs(self._build_command_line, config_file, "Building Command Line") + global_tool_logs(self._build_version_command, config_file, "Building Version Command Line") global_tool_logs(self._build_environment_variables, config_file, "Building Environment Variables") - return self.command_line, self.extra_filenames, self.environment_variables + return self.command_line, self.version_command_line, self.extra_filenames, self.environment_variables def _build_command_line(self): """ @@ -455,7 +457,6 @@ class ToolEvaluator: command = self.tool.command or '' param_dict = self.param_dict interpreter = self.tool.interpreter - version_string_cmd_raw = self.tool.version_string_cmd command_line = None if not command: return @@ -479,11 +480,14 @@ class ToolEvaluator: tool_dir = os.path.abspath(self.tool.tool_dir) abs_executable = os.path.join(tool_dir, executable) command_line = command_line.replace(executable, f"{interpreter} {shlex.quote(abs_executable)}", 1) + self.command_line = command_line + + def _build_version_command(self): + version_string_cmd_raw = self.tool.version_string_cmd if version_string_cmd_raw: version_command_template = string.Template(version_string_cmd_raw) - version_string_cmd = version_command_template.safe_substitute({"__tool_directory__": self.compute_environment.tool_directory()}) - command_line = f"{version_string_cmd} > {self.compute_environment.version_path()} 2>&1;\n{command_line}" - self.command_line = command_line + version_command = version_command_template.safe_substitute({"__tool_directory__": self.compute_environment.tool_directory()}) + self.version_command_line = f"{version_command} > {self.compute_environment.version_path()} 2>&1;\n" def _build_config_files(self): """ @@ -641,7 +645,7 @@ class PartialToolEvaluator(ToolEvaluator): def build(self): config_file = self.tool.config_file global_tool_logs(self._build_environment_variables, config_file, "Building Environment Variables") - return self.command_line, self.extra_filenames, self.environment_variables + return self.command_line, self.version_command_line, self.extra_filenames, self.environment_variables class RemoteToolEvaluator(ToolEvaluator): @@ -656,4 +660,5 @@ class RemoteToolEvaluator(ToolEvaluator): global_tool_logs(self._build_config_files, config_file, "Building Config Files") global_tool_logs(self._build_param_file, config_file, 'Building Param File') global_tool_logs(self._build_command_line, config_file, "Building Command Line") - return self.command_line, self.extra_filenames, self.environment_variables + global_tool_logs(self._build_version_command, config_file, "Building Version Command Line") + return self.command_line, self.version_command_line, self.extra_filenames, self.environment_variables diff --git a/lib/galaxy/tools/remote_tool_eval.py b/lib/galaxy/tools/remote_tool_eval.py index 8ffef968398..0235311023a 100644 --- a/lib/galaxy/tools/remote_tool_eval.py +++ b/lib/galaxy/tools/remote_tool_eval.py @@ -101,8 +101,8 @@ def main(TMPDIR, WORKING_DIRECTORY, IMPORT_STORE_DIRECTORY): tool_evaluator = evaluation.RemoteToolEvaluator(app=app, tool=tool, job=job_io.job, local_working_directory=WORKING_DIRECTORY) tool_evaluator.set_compute_environment(compute_environment=SharedComputeEnvironment(job_io=job_io, job=job_io.job)) with open(os.path.join(WORKING_DIRECTORY, 'tool_script.sh'), 'a') as out: - command_line, extra_filenames, environment_variables = tool_evaluator.build() - out.write(command_line) + command_line, version_command_line, extra_filenames, environment_variables = tool_evaluator.build() + out.write(f'{version_command_line or ""}{command_line}') if __name__ == "__main__": diff --git a/test/functional/tools/version_command_plain.xml b/test/functional/tools/version_command_plain.xml index b2ffc3e07ef..4d3b925624f 100644 --- a/test/functional/tools/version_command_plain.xml +++ b/test/functional/tools/version_command_plain.xml @@ -15,6 +15,9 @@ cp '$input' '$output' + + + diff --git a/test/unit/app/jobs/test_job_wrapper.py b/test/unit/app/jobs/test_job_wrapper.py index ba9cb55ca74..4566c8f8ba0 100644 --- a/test/unit/app/jobs/test_job_wrapper.py +++ b/test/unit/app/jobs/test_job_wrapper.py @@ -105,7 +105,7 @@ class MockEvaluator: pass def build(self): - return TEST_COMMAND, [], [] + return TEST_COMMAND, "", [], [] class MockJobQueue: diff --git a/test/unit/app/tools/test_evaluation.py b/test/unit/app/tools/test_evaluation.py index b6b0ce484c2..6f2a709cc32 100644 --- a/test/unit/app/tools/test_evaluation.py +++ b/test/unit/app/tools/test_evaluation.py @@ -53,7 +53,7 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): def test_simple_evaluation(self): self._setup_test_bwa_job() self._set_compute_environment() - command_line, extra_filenames, _ = self.evaluator.build() + command_line = self.evaluator.build()[0] self.assertEqual(command_line, "bwa --thresh=4 --in=/galaxy/files/dataset_1.dat --out=/galaxy/files/dataset_2.dat") def test_repeat_evaluation(self): @@ -64,19 +64,19 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): self.job.parameters = [JobParameter(name="r", value='''[{"thresh": 4, "__index__": 0},{"thresh": 5, "__index__": 1}]''')] self.tool._command_line = "prog1 #for $r_i in $r # $r_i.thresh#end for#" self._set_compute_environment() - command_line, extra_filenames, _ = self.evaluator.build() + command_line = self.evaluator.build()[0] self.assertEqual(command_line, "prog1 4 5") def test_eval_galaxy_url(self): self.tool._command_line = "prog1 $__galaxy_url__" self._set_compute_environment() - command_line, extra_filenames, _ = self.evaluator.build() + command_line = self.evaluator.build()[0] self.assertEqual(command_line, "prog1 %s" % TEST_GALAXY_URL) def test_eval_history_id(self): self.tool._command_line = "prog1 '$__history_id__'" self._set_compute_environment() - command_line, extra_filenames, _ = self.evaluator.build() + command_line = self.evaluator.build()[0] self.assertEqual(command_line, "prog1 '%s'" % self.app.security.encode_id(42)) def test_conditional_evaluation(self): @@ -94,7 +94,7 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): self.job.parameters = [JobParameter(name="c", value='''{"thresh": 4, "always_true": "true", "__current_case__": 0}''')] self.tool._command_line = "prog1 --thresh=${c.thresh} --test_param=${c.always_true}" self._set_compute_environment() - command_line, extra_filenames, _ = self.evaluator.build() + command_line = self.evaluator.build()[0] self.assertEqual(command_line, "prog1 --thresh=4 --test_param=true") def test_evaluation_of_optional_datasets(self): @@ -106,7 +106,7 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): self.tool.set_params({"input1": parameter}) self.tool._command_line = "prog1 --opt_input='${input1}'" self._set_compute_environment() - command_line, extra_filenames, _ = self.evaluator.build() + command_line = self.evaluator.build()[0] self.assertEqual(command_line, "prog1 --opt_input='None'") def test_evaluation_with_path_rewrites_wrapped(self): @@ -128,14 +128,14 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): input_paths=[DatasetPath(1, '/galaxy/files/dataset_1.dat', false_path=job_path_1)], output_paths=[DatasetPath(2, '/galaxy/files/dataset_2.dat', false_path=job_path_2)], ) - command_line, extra_filenames, _ = self.evaluator.build() + command_line = self.evaluator.build()[0] self.assertEqual(command_line, f"bwa --thresh=4 --in={job_path_1} --out={job_path_2}") def test_configfiles_evaluation(self): self.tool.config_files.append(("conf1", None, "$thresh")) self.tool._command_line = "prog1 $conf1" self._set_compute_environment() - command_line, extra_filenames, _ = self.evaluator.build() + command_line, _, extra_filenames, _ = self.evaluator.build() self.assertEqual(len(extra_filenames), 1) config_filename = extra_filenames[0] config_basename = os.path.basename(config_filename) @@ -175,15 +175,15 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): }) self.tool._command_line = "prog1 $index_path.fields.path" self._set_compute_environment(unstructured_path_rewrites={"/old": "/new"}) - command_line, extra_filenames, _ = self.evaluator.build() + command_line = self.evaluator.build()[0] self.assertEqual(command_line, "prog1 /new/path/human") def test_version_command(self): self.tool.version_string_cmd = "echo v.1.1" self._setup_test_bwa_job() self._set_compute_environment() - command_line, extra_filenames, _ = self.evaluator.build() - assert self.tool.version_string_cmd in command_line + version_command_line = self.evaluator.build()[1] + assert self.tool.version_string_cmd in version_command_line def test_template_property_app(self): self._assert_template_property_is("$__app__.config.new_file_path", self.app.config.new_file_path) @@ -201,7 +201,7 @@ class ToolEvaluatorTestCase(TestCase, UsesApp): self.tool._command_line = "test.exe" self.tool.config_files.append(("conf1", None, """%s""" % expression)) self._set_compute_environment() - _, extra_filenames, _ = self.evaluator.build() + extra_filenames = self.evaluator.build()[2] config_filename = extra_filenames[0] self.assertEqual(open(config_filename).read(), value)