opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #13317 from mvdbeek/fix_command_recording

Browse Source
This commit is contained in:
Nicola Soranzo
2022-02-09 10:21:52 +00:00
committed by GitHub
parent e9270c5299 7c4905251c
commit bf3a5420ef
7 changed files with 44 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/jobs/__init__.py
+7 -3
View File
@@ -939,6 +939,7 @@ class JobWrapper(HasResourceParameters):
self.environment_variables: List[Dict[str, str]] = []
self.interactivetools: List[Dict[str, Any]] = []
self.command_line = None
self.version_command_line = None
self._dependency_shell_commands = None
# Tool versioning variables
self.version_string = ""
@@ -1192,7 +1193,7 @@ class JobWrapper(HasResourceParameters):
tool_evaluator = self._get_tool_evaluator(job)
compute_environment = compute_environment or self.default_compute_environment(job)
tool_evaluator.set_compute_environment(compute_environment, get_special=get_special)
self.command_line, self.extra_filenames, self.environment_variables = tool_evaluator.build()
self.command_line, self.version_command_line, self.extra_filenames, self.environment_variables = tool_evaluator.build()
job.command_line = self.command_line
self.interactivetools = tool_evaluator.populate_interactivetools()
self.app.interactivetool_manager.create_interactivetool(job, self.tool, self.interactivetools)
@@ -1949,7 +1950,10 @@ class JobWrapper(HasResourceParameters):
return has_output_limit or has_walltime_limit
def get_command_line(self):
return self.command_line
"""Return complete command line, including possible version command."""
if self.remote_command_line:
return None
return f'{self.version_command_line or ""}{self.command_line}'
def get_session_id(self):
return self.session_id
@@ -2305,7 +2309,7 @@ class TaskWrapper(JobWrapper):
self.sa_session.flush()
if not self.remote_command_line:
self.command_line, extra_filenames, self.environment_variables = tool_evaluator.build()
self.command_line, self.version_command_line, extra_filenames, self.environment_variables = tool_evaluator.build()
self.extra_filenames.extend(extra_filenames)
# Ensure galaxy_lib_dir is set in case there are any later chdirs
lib/galaxy/metadata/set_metadata.py
+7 -1
View File
@@ -281,7 +281,13 @@ def set_metadata_portable():
job.state = final_job_state
if os.path.exists(tool_script_file):
with open(tool_script_file) as command_fh:
job.command_line = command_fh.read().strip()
command_line_lines = []
for i, line in enumerate(command_fh):
if i == 0 and line.endswith('COMMAND_VERSION 2>&1;'):
# Don't record version command as part of command line
continue
command_line_lines.append(line)
job.command_line = "".join(command_line_lines).strip()
export_store.export_job(job, include_job_data=False)
unnamed_id_to_path = {}
lib/galaxy/tools/evaluation.py
+12 -7
View File
@@ -91,6 +91,7 @@ class ToolEvaluator:
self.param_dict: Dict[str, Any] = {}
self.extra_filenames: List[str] = []
self.environment_variables: List[Dict[str, str]] = []
self.version_command_line: Optional[str] = None
self.command_line: Optional[str] = None
def set_compute_environment(self, compute_environment: ComputeEnvironment, get_special: Optional[Callable] = None):
@@ -445,8 +446,9 @@ class ToolEvaluator:
global_tool_logs(self._build_config_files, config_file, "Building Config Files")
global_tool_logs(self._build_param_file, config_file, 'Building Param File')
global_tool_logs(self._build_command_line, config_file, "Building Command Line")
global_tool_logs(self._build_version_command, config_file, "Building Version Command Line")
global_tool_logs(self._build_environment_variables, config_file, "Building Environment Variables")
return self.command_line, self.extra_filenames, self.environment_variables
return self.command_line, self.version_command_line, self.extra_filenames, self.environment_variables
def _build_command_line(self):
"""
@@ -455,7 +457,6 @@ class ToolEvaluator:
command = self.tool.command or ''
param_dict = self.param_dict
interpreter = self.tool.interpreter
version_string_cmd_raw = self.tool.version_string_cmd
command_line = None
if not command:
return
@@ -479,11 +480,14 @@ class ToolEvaluator:
tool_dir = os.path.abspath(self.tool.tool_dir)
abs_executable = os.path.join(tool_dir, executable)
command_line = command_line.replace(executable, f"{interpreter} {shlex.quote(abs_executable)}", 1)
self.command_line = command_line
def _build_version_command(self):
version_string_cmd_raw = self.tool.version_string_cmd
if version_string_cmd_raw:
version_command_template = string.Template(version_string_cmd_raw)
version_string_cmd = version_command_template.safe_substitute({"__tool_directory__": self.compute_environment.tool_directory()})
command_line = f"{version_string_cmd} > {self.compute_environment.version_path()} 2>&1;\n{command_line}"
self.command_line = command_line
version_command = version_command_template.safe_substitute({"__tool_directory__": self.compute_environment.tool_directory()})
self.version_command_line = f"{version_command} > {self.compute_environment.version_path()} 2>&1;\n"
def _build_config_files(self):
"""
@@ -641,7 +645,7 @@ class PartialToolEvaluator(ToolEvaluator):
def build(self):
config_file = self.tool.config_file
global_tool_logs(self._build_environment_variables, config_file, "Building Environment Variables")
return self.command_line, self.extra_filenames, self.environment_variables
return self.command_line, self.version_command_line, self.extra_filenames, self.environment_variables
class RemoteToolEvaluator(ToolEvaluator):
@@ -656,4 +660,5 @@ class RemoteToolEvaluator(ToolEvaluator):
global_tool_logs(self._build_config_files, config_file, "Building Config Files")
global_tool_logs(self._build_param_file, config_file, 'Building Param File')
global_tool_logs(self._build_command_line, config_file, "Building Command Line")
return self.command_line, self.extra_filenames, self.environment_variables
global_tool_logs(self._build_version_command, config_file, "Building Version Command Line")
return self.command_line, self.version_command_line, self.extra_filenames, self.environment_variables
lib/galaxy/tools/remote_tool_eval.py
+2 -2
View File
@@ -101,8 +101,8 @@ def main(TMPDIR, WORKING_DIRECTORY, IMPORT_STORE_DIRECTORY):
tool_evaluator = evaluation.RemoteToolEvaluator(app=app, tool=tool, job=job_io.job, local_working_directory=WORKING_DIRECTORY)
tool_evaluator.set_compute_environment(compute_environment=SharedComputeEnvironment(job_io=job_io, job=job_io.job))
with open(os.path.join(WORKING_DIRECTORY, 'tool_script.sh'), 'a') as out:
command_line, extra_filenames, environment_variables = tool_evaluator.build()
out.write(command_line)
command_line, version_command_line, extra_filenames, environment_variables = tool_evaluator.build()
out.write(f'{version_command_line or ""}{command_line}')
if __name__ == "__main__":
test/functional/tools/version_command_plain.xml
+3
View File
@@ -15,6 +15,9 @@ cp '$input' '$output'
<test>
<param name="input" value="simple_line.txt" />
<output name="out_file1" checksum="sha1$8156d7ca0f46ed7abac98f82e36cfaddb2aca041" />
<assert_command>
<not_has_text text="VERSION"/>
</assert_command>
</test>
</tests>
</tool>
test/unit/app/jobs/test_job_wrapper.py
+1 -1
View File
@@ -105,7 +105,7 @@ class MockEvaluator:
pass
def build(self):
return TEST_COMMAND, [], []
return TEST_COMMAND, "", [], []
class MockJobQueue:
test/unit/app/tools/test_evaluation.py
+12 -12
View File
@@ -53,7 +53,7 @@ class ToolEvaluatorTestCase(TestCase, UsesApp):
def test_simple_evaluation(self):
self._setup_test_bwa_job()
self._set_compute_environment()
command_line, extra_filenames, _ = self.evaluator.build()
command_line = self.evaluator.build()[0]
self.assertEqual(command_line, "bwa --thresh=4 --in=/galaxy/files/dataset_1.dat --out=/galaxy/files/dataset_2.dat")
def test_repeat_evaluation(self):
@@ -64,19 +64,19 @@ class ToolEvaluatorTestCase(TestCase, UsesApp):
self.job.parameters = [JobParameter(name="r", value='''[{"thresh": 4, "__index__": 0},{"thresh": 5, "__index__": 1}]''')]
self.tool._command_line = "prog1 #for $r_i in $r # $r_i.thresh#end for#"
self._set_compute_environment()
command_line, extra_filenames, _ = self.evaluator.build()
command_line = self.evaluator.build()[0]
self.assertEqual(command_line, "prog1 4 5")
def test_eval_galaxy_url(self):
self.tool._command_line = "prog1 $__galaxy_url__"
self._set_compute_environment()
command_line, extra_filenames, _ = self.evaluator.build()
command_line = self.evaluator.build()[0]
self.assertEqual(command_line, "prog1 %s" % TEST_GALAXY_URL)
def test_eval_history_id(self):
self.tool._command_line = "prog1 '$__history_id__'"
self._set_compute_environment()
command_line, extra_filenames, _ = self.evaluator.build()
command_line = self.evaluator.build()[0]
self.assertEqual(command_line, "prog1 '%s'" % self.app.security.encode_id(42))
def test_conditional_evaluation(self):
@@ -94,7 +94,7 @@ class ToolEvaluatorTestCase(TestCase, UsesApp):
self.job.parameters = [JobParameter(name="c", value='''{"thresh": 4, "always_true": "true", "__current_case__": 0}''')]
self.tool._command_line = "prog1 --thresh=${c.thresh} --test_param=${c.always_true}"
self._set_compute_environment()
command_line, extra_filenames, _ = self.evaluator.build()
command_line = self.evaluator.build()[0]
self.assertEqual(command_line, "prog1 --thresh=4 --test_param=true")
def test_evaluation_of_optional_datasets(self):
@@ -106,7 +106,7 @@ class ToolEvaluatorTestCase(TestCase, UsesApp):
self.tool.set_params({"input1": parameter})
self.tool._command_line = "prog1 --opt_input='${input1}'"
self._set_compute_environment()
command_line, extra_filenames, _ = self.evaluator.build()
command_line = self.evaluator.build()[0]
self.assertEqual(command_line, "prog1 --opt_input='None'")
def test_evaluation_with_path_rewrites_wrapped(self):
@@ -128,14 +128,14 @@ class ToolEvaluatorTestCase(TestCase, UsesApp):
input_paths=[DatasetPath(1, '/galaxy/files/dataset_1.dat', false_path=job_path_1)],
output_paths=[DatasetPath(2, '/galaxy/files/dataset_2.dat', false_path=job_path_2)],
)
command_line, extra_filenames, _ = self.evaluator.build()
command_line = self.evaluator.build()[0]
self.assertEqual(command_line, f"bwa --thresh=4 --in={job_path_1} --out={job_path_2}")
def test_configfiles_evaluation(self):
self.tool.config_files.append(("conf1", None, "$thresh"))
self.tool._command_line = "prog1 $conf1"
self._set_compute_environment()
command_line, extra_filenames, _ = self.evaluator.build()
command_line, _, extra_filenames, _ = self.evaluator.build()
self.assertEqual(len(extra_filenames), 1)
config_filename = extra_filenames[0]
config_basename = os.path.basename(config_filename)
@@ -175,15 +175,15 @@ class ToolEvaluatorTestCase(TestCase, UsesApp):
})
self.tool._command_line = "prog1 $index_path.fields.path"
self._set_compute_environment(unstructured_path_rewrites={"/old": "/new"})
command_line, extra_filenames, _ = self.evaluator.build()
command_line = self.evaluator.build()[0]
self.assertEqual(command_line, "prog1 /new/path/human")
def test_version_command(self):
self.tool.version_string_cmd = "echo v.1.1"
self._setup_test_bwa_job()
self._set_compute_environment()
command_line, extra_filenames, _ = self.evaluator.build()
assert self.tool.version_string_cmd in command_line
version_command_line = self.evaluator.build()[1]
assert self.tool.version_string_cmd in version_command_line
def test_template_property_app(self):
self._assert_template_property_is("$__app__.config.new_file_path", self.app.config.new_file_path)
@@ -201,7 +201,7 @@ class ToolEvaluatorTestCase(TestCase, UsesApp):
self.tool._command_line = "test.exe"
self.tool.config_files.append(("conf1", None, """%s""" % expression))
self._set_compute_environment()
_, extra_filenames, _ = self.evaluator.build()
extra_filenames = self.evaluator.build()[2]
config_filename = extra_filenames[0]
self.assertEqual(open(config_filename).read(), value)