Merge branch 'release_23.2' into dev

This commit is contained in:
mvdbeek
2024-02-14 18:44:36 +01:00
14 changed files with 236 additions and 117 deletions
+7 -2
View File
@@ -1,6 +1,10 @@
import abc
import os
import time
from dataclasses import (
dataclass,
field,
)
from enum import Enum
from typing import (
Any,
@@ -88,19 +92,20 @@ class FilesSourceProperties(TypedDict):
browsable: NotRequired[bool]
@dataclass
class FilesSourceOptions:
"""Options to control behavior of file source operations, such as realize_to, write_from and list."""
# Indicates access to the FS operation with intent to write.
# Even if a file source is "writeable" some directories (or elements) may be restricted or read-only
# so those should be skipped while browsing with writeable=True.
writeable: Optional[bool]
writeable: Optional[bool] = False
# Property overrides for values initially configured through the constructor. For example
# the HTTPFilesSource passes in additional http_headers through these properties, which
# are merged with constructor defined http_headers. The interpretation of these properties
# are filesystem specific.
extra_props: Optional[FilesSourceProperties]
extra_props: Optional[FilesSourceProperties] = field(default_factory=lambda: FilesSourceProperties())
class EntryData(TypedDict):
+2 -3
View File
@@ -1244,15 +1244,13 @@ class MinimalJobWrapper(HasResourceParameters):
tool_evaluator = self._get_tool_evaluator(job)
compute_environment = compute_environment or self.default_compute_environment(job)
if hasattr(self.app, "interactivetool_manager"):
self.interactivetools = tool_evaluator.populate_interactivetools()
self.app.interactivetool_manager.create_interactivetool(job, self.tool, self.interactivetools)
tool_evaluator.set_compute_environment(compute_environment, get_special=get_special)
(
self.command_line,
self.version_command_line,
self.extra_filenames,
self.environment_variables,
self.interactivetools,
) = tool_evaluator.build()
job.command_line = self.command_line
@@ -2616,6 +2614,7 @@ class TaskWrapper(JobWrapper):
self.version_command_line,
extra_filenames,
self.environment_variables,
*_,
) = tool_evaluator.build()
self.extra_filenames.extend(extra_filenames)
+31 -4
View File
@@ -121,6 +121,7 @@ class ToolEvaluator:
self.environment_variables: List[Dict[str, str]] = []
self.version_command_line: Optional[str] = None
self.command_line: Optional[str] = None
self.interactivetools: List[Dict[str, Any]] = []
def set_compute_environment(self, compute_environment: ComputeEnvironment, get_special: Optional[Callable] = None):
"""
@@ -514,7 +515,12 @@ class ToolEvaluator:
# the paths rewritten.
self.__walk_inputs(self.tool.inputs, param_dict, rewrite_unstructured_paths)
def populate_interactivetools(self):
def _create_interactivetools_entry_points(self):
if hasattr(self.app, "interactivetool_manager"):
self.interactivetools = self._populate_interactivetools_template()
self.app.interactivetool_manager.create_interactivetool(self.job, self.tool, self.interactivetools)
def _populate_interactivetools_template(self):
"""
Populate InteractiveTools templated values.
"""
@@ -571,12 +577,21 @@ class ToolEvaluator:
compute environment.
"""
config_file = self.tool.config_file
global_tool_logs(
self._create_interactivetools_entry_points, config_file, "Building Interactive Tool Entry Points"
)
global_tool_logs(self._build_config_files, config_file, "Building Config Files")
global_tool_logs(self._build_param_file, config_file, "Building Param File")
global_tool_logs(self._build_command_line, config_file, "Building Command Line")
global_tool_logs(self._build_version_command, config_file, "Building Version Command Line")
global_tool_logs(self._build_environment_variables, config_file, "Building Environment Variables")
return self.command_line, self.version_command_line, self.extra_filenames, self.environment_variables
return (
self.command_line,
self.version_command_line,
self.extra_filenames,
self.environment_variables,
self.interactivetools,
)
def _build_command_line(self):
"""
@@ -819,7 +834,13 @@ class PartialToolEvaluator(ToolEvaluator):
def build(self):
config_file = self.tool.config_file
global_tool_logs(self._build_environment_variables, config_file, "Building Environment Variables")
return self.command_line, self.version_command_line, self.extra_filenames, self.environment_variables
return (
self.command_line,
self.version_command_line,
self.extra_filenames,
self.environment_variables,
self.interactivetools,
)
class RemoteToolEvaluator(ToolEvaluator):
@@ -837,4 +858,10 @@ class RemoteToolEvaluator(ToolEvaluator):
global_tool_logs(self._build_param_file, config_file, "Building Param File")
global_tool_logs(self._build_command_line, config_file, "Building Command Line")
global_tool_logs(self._build_version_command, config_file, "Building Version Command Line")
return self.command_line, self.version_command_line, self.extra_filenames, self.environment_variables
return (
self.command_line,
self.version_command_line,
self.extra_filenames,
self.environment_variables,
self.interactivetools,
)
+1 -1
View File
@@ -114,7 +114,7 @@ def main(TMPDIR, WORKING_DIRECTORY, IMPORT_STORE_DIRECTORY):
)
tool_evaluator.set_compute_environment(compute_environment=SharedComputeEnvironment(job_io=job_io, job=job_io.job))
with open(os.path.join(WORKING_DIRECTORY, "tool_script.sh"), "a") as out:
command_line, version_command_line, extra_filenames, environment_variables = tool_evaluator.build()
command_line, version_command_line, extra_filenames, environment_variables, *_ = tool_evaluator.build()
out.write(f'{version_command_line or ""}{command_line}')
@@ -247,7 +247,7 @@ class DatasetCollectionsService(ServiceBase, UsesLibraryMixinItems):
raise exceptions.RequestParameterInvalidException(
"Parameter instance_type not being 'history' is not yet implemented."
)
hdca: HistoryDatasetCollectionAssociation = self.collection_manager.get_dataset_collection_instance(
hdca: "HistoryDatasetCollectionAssociation" = self.collection_manager.get_dataset_collection_instance(
trans, "history", hdca_id
)