From 3c4e1195f7d4572cc12183b86c0fb1bc8740492c Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 25 Aug 2023 15:24:33 +0200 Subject: [PATCH 1/8] Implement downloading directories from S3 --- lib/galaxy/objectstore/s3.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/objectstore/s3.py b/lib/galaxy/objectstore/s3.py index 6fa1b6e7e4f..a719ba3ec19 100644 --- a/lib/galaxy/objectstore/s3.py +++ b/lib/galaxy/objectstore/s3.py @@ -47,6 +47,21 @@ log = logging.getLogger(__name__) logging.getLogger("boto").setLevel(logging.INFO) # Otherwise boto is quite noisy +def download_directory(bucket, remote_folder, local_path): + # List objects in the specified S3 folder + objects = bucket.list(prefix=remote_folder) + + for obj in objects: + remote_file_path = obj.key + local_file_path = os.path.join(local_path, os.path.relpath(remote_file_path, remote_folder)) + + # Create directories if they don't exist + os.makedirs(os.path.dirname(local_file_path), exist_ok=True) + + # Download the file + obj.get_contents_to_filename(local_file_path) + + def parse_config_xml(config_xml): try: a_xml = config_xml.findall("auth")[0] @@ -720,7 +735,8 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin): return cache_path # Check if the file exists in persistent storage and, if it does, pull it into cache elif self._exists(obj, **kwargs): - if dir_only: # Directories do not get pulled into cache + if dir_only: + download_directory(self._bucket, rel_path, cache_path) return cache_path else: if self._pull_into_cache(rel_path): From 27991b950d51d9101e0455c3c53ad94977f64762 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 25 Aug 2023 15:25:06 +0200 Subject: [PATCH 2/8] Fix writing data bundle manifest to object store --- lib/galaxy/tool_util/data/__init__.py | 6 +++++- lib/galaxy/tools/__init__.py | 12 +++++++++++- lib/galaxy/tools/data_manager/manager.py | 4 ++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/tool_util/data/__init__.py b/lib/galaxy/tool_util/data/__init__.py index ea2e1a77c47..5e2fdfb7908 100644 --- a/lib/galaxy/tool_util/data/__init__.py +++ b/lib/galaxy/tool_util/data/__init__.py @@ -1174,8 +1174,10 @@ class ToolDataTableManager(Dictifiable): out_data: Dict[str, OutputDataset], bundle_description: DataTableBundleProcessorDescription, repo_info: Optional[RepoInfo], - ) -> None: + ) -> Dict[str, OutputDataset]: + """Writes bundle and returns bundle path.""" data_manager_dict = _data_manager_dict(out_data, ensure_single_output=True) + bundle_datasets: Dict[str, OutputDataset] = {} for output_name, dataset in out_data.items(): if dataset.ext != "data_manager_json": continue @@ -1190,6 +1192,8 @@ class ToolDataTableManager(Dictifiable): bundle_path = os.path.join(extra_files_path, BUNDLE_INDEX_FILE_NAME) with open(bundle_path, "w") as fw: json.dump(bundle.dict(), fw) + bundle_datasets[bundle_path] = dataset + return bundle_datasets SUPPORTED_DATA_TABLE_TYPES = TabularToolDataTable diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index a8ccec0b37a..58ff950c7c4 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -3064,7 +3064,17 @@ class DataManagerTool(OutputParameterJSONTool): elif data_manager_mode == "dry_run": pass elif data_manager_mode == "bundle": - data_manager.write_bundle(out_data) + for bundle_path, dataset in data_manager.write_bundle(out_data).items(): + dataset = cast(model.HistoryDatasetAssociation, dataset) + dataset.dataset.object_store.update_from_file( + dataset.dataset, + extra_dir=dataset.dataset.extra_files_path_name, + file_name=bundle_path, + alt_name=os.path.basename(bundle_path), + create=True, + preserve_symlinks=True, + ) + else: raise Exception("Unknown data manager mode encountered type...") diff --git a/lib/galaxy/tools/data_manager/manager.py b/lib/galaxy/tools/data_manager/manager.py index 89894db1f30..aaa7e4d23ee 100644 --- a/lib/galaxy/tools/data_manager/manager.py +++ b/lib/galaxy/tools/data_manager/manager.py @@ -236,9 +236,9 @@ class DataManager: def write_bundle( self, out_data: Dict[str, OutputDataset], - ) -> None: + ): tool_data_tables = self.data_managers.app.tool_data_tables - tool_data_tables.write_bundle( + return tool_data_tables.write_bundle( out_data, self.processor_description, self.repo_info, From ff504d9d3d9503a889e411e01804f64ef5ec997f Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 25 Aug 2023 15:25:47 +0200 Subject: [PATCH 3/8] Extend data bundle integration test to use S3 object store, check that cache can be repopulated --- test/integration/test_tool_data_bundles.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/integration/test_tool_data_bundles.py b/test/integration/test_tool_data_bundles.py index 67eca7e3a04..d4ede08b768 100644 --- a/test/integration/test_tool_data_bundles.py +++ b/test/integration/test_tool_data_bundles.py @@ -1,10 +1,12 @@ import os +import shutil from galaxy.util.compression_utils import decompress_bytes_to_directory +from .objectstore._base import BaseSwiftObjectStoreIntegrationTestCase from .test_tool_data_delete import DataManagerIntegrationTestCase -class TestDataBundlesIntegration(DataManagerIntegrationTestCase): +class TestDataBundlesIntegration(BaseSwiftObjectStoreIntegrationTestCase, DataManagerIntegrationTestCase): def test_admin_build_data_bundle_by_uri(self): original_count = self._testbeta_field_count() @@ -24,6 +26,9 @@ class TestDataBundlesIntegration(DataManagerIntegrationTestCase): post_job_count = self._testbeta_field_count() assert original_count == post_job_count + shutil.rmtree(self.object_store_cache_path) + os.makedirs(self.object_store_cache_path) + content = self.dataset_populator.get_history_dataset_content( history_id, to_ext="data_manager_json", type="bytes" ) From 0477581217d014cff25edebda4b3c4f1777c3cff Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 25 Aug 2023 14:57:20 +0200 Subject: [PATCH 4/8] Push extra files from object store cache to object store --- lib/galaxy/job_execution/output_collect.py | 7 ++++--- test/integration/test_tool_data_bundles.py | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/job_execution/output_collect.py b/lib/galaxy/job_execution/output_collect.py index a8cd4369c12..9464de383e9 100644 --- a/lib/galaxy/job_execution/output_collect.py +++ b/lib/galaxy/job_execution/output_collect.py @@ -727,18 +727,19 @@ def collect_extra_files(object_store, dataset, job_working_directory): # Fall back to working dir, remove in 23.2 output_location = "working" temp_file_path = os.path.join(job_working_directory, output_location, file_name) - extra_dir = None + if not os.path.exists(temp_file_path): + # no outputs to working directory, but may still need to push form cache to backend + temp_file_path = dataset.extra_files_path try: # This skips creation of directories - object store # automatically creates them. However, empty directories will # not be created in the object store at all, which might be a # problem. for root, _dirs, files in os.walk(temp_file_path): - extra_dir = root.replace(os.path.join(job_working_directory, output_location), "", 1).lstrip(os.path.sep) for f in files: object_store.update_from_file( dataset.dataset, - extra_dir=extra_dir, + extra_dir=os.path.normpath(os.path.join(file_name, os.path.relpath(root, temp_file_path))), alt_name=f, file_name=os.path.join(root, f), create=True, diff --git a/test/integration/test_tool_data_bundles.py b/test/integration/test_tool_data_bundles.py index d4ede08b768..54f88633473 100644 --- a/test/integration/test_tool_data_bundles.py +++ b/test/integration/test_tool_data_bundles.py @@ -33,6 +33,7 @@ class TestDataBundlesIntegration(BaseSwiftObjectStoreIntegrationTestCase, DataMa history_id, to_ext="data_manager_json", type="bytes" ) temp_directory = decompress_bytes_to_directory(content) + assert os.path.exists(os.path.join(temp_directory, "newvalue.txt")) uri = f"file://{os.path.normpath(temp_directory)}" data = { "source": { From e93e986ad6dffea6b0e5f7278c2472783c77acbe Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 25 Aug 2023 15:53:39 +0200 Subject: [PATCH 5/8] Fix new_dataset._extra_files_path just in case something still uses it --- lib/galaxy/metadata/set_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/metadata/set_metadata.py b/lib/galaxy/metadata/set_metadata.py index 3bfd55a3615..47e7eb3dcf4 100644 --- a/lib/galaxy/metadata/set_metadata.py +++ b/lib/galaxy/metadata/set_metadata.py @@ -528,7 +528,7 @@ def write_job_metadata(tool_job_working_directory, job_metadata, set_meta, tool_ new_dataset = Dataset(id=-i, external_filename=new_dataset_filename) extra_files = file_dict.get("extra_files", None) if extra_files is not None: - new_dataset._extra_files_path = os.path.join(tool_job_working_directory, "working", extra_files) + new_dataset._extra_files_path = os.path.join(tool_job_working_directory, "outputs", extra_files) new_dataset.state = new_dataset.states.OK new_dataset_instance = HistoryDatasetAssociation( id=-i, dataset=new_dataset, extension=file_dict.get("ext", "data") From efef30d6ab3bccb4f177792b982b08d055959d3d Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 25 Aug 2023 11:00:06 +0200 Subject: [PATCH 6/8] make sure that TMP, TEMP, and TMPDIR are set seems needed for docker volume strings (an empty variable would lead to `-v "::rw"` which causes `docker: invalid spec: ::rw: empty section between colons.`) --- .../job_script/DEFAULT_JOB_FILE_TEMPLATE.sh | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh index 08d698ab779..e89cc001e45 100644 --- a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh +++ b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh @@ -35,6 +35,29 @@ GALAXY_LIB="$galaxy_lib" _galaxy_setup_environment "$PRESERVE_GALAXY_ENVIRONMENT" export _GALAXY_JOB_HOME_DIR export _GALAXY_JOB_TMP_DIR + +TEMP="${TEMP:-$TMP}" +TMPDIR="${TMPDIR:-$TMP}" + +TMP="${TMP:-$TEMP}" +TMPDIR="${TMPDIR:-$TEMP}" + +TMP="${TMP:-$TMPDIR}" +TEMP="${TEMP:-$TMPDIR}" + +TMP="${TMP:-_GALAXY_JOB_TMP_DIR}" +TEMP="${TEMP:-_GALAXY_JOB_TMP_DIR}" +TMPDIR="${TMPDIR:-_GALAXY_JOB_TMP_DIR}" + +export TMP +export TEMP +export TMPDIR + +echo "_GALAXY_JOB_TMP_DIR $_GALAXY_JOB_TMP_DIR" >> outputs/tmp +echo "TMP $TMP" >> outputs/tmp +echo "TEMP $TEMP" >> outputs/tmp +echo "TMPDIR $TMPDIR" >> outputs/tmp + GALAXY_PYTHON=`command -v python` cd $working_directory $memory_statement From 913e890c6acab1d25123a2dec4484fababff2569 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 25 Aug 2023 16:02:22 +0200 Subject: [PATCH 7/8] Fix _GALAXY_JOB_TMP_DIR Co-authored-by: Marius van den Beek --- .../runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh index e89cc001e45..0b39cafb159 100644 --- a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh +++ b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh @@ -45,9 +45,9 @@ TMPDIR="${TMPDIR:-$TEMP}" TMP="${TMP:-$TMPDIR}" TEMP="${TEMP:-$TMPDIR}" -TMP="${TMP:-_GALAXY_JOB_TMP_DIR}" -TEMP="${TEMP:-_GALAXY_JOB_TMP_DIR}" -TMPDIR="${TMPDIR:-_GALAXY_JOB_TMP_DIR}" +TMP="${TMP:-$_GALAXY_JOB_TMP_DIR}" +TEMP="${TEMP:-$_GALAXY_JOB_TMP_DIR}" +TMPDIR="${TMPDIR:-$_GALAXY_JOB_TMP_DIR}" export TMP export TEMP From 99ffa0e5ce7dd346d8a68be2b6a308eb3a93e5db Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 25 Aug 2023 16:03:11 +0200 Subject: [PATCH 8/8] remove debug code --- .../runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh index 0b39cafb159..514552419dd 100644 --- a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh +++ b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh @@ -53,11 +53,6 @@ export TMP export TEMP export TMPDIR -echo "_GALAXY_JOB_TMP_DIR $_GALAXY_JOB_TMP_DIR" >> outputs/tmp -echo "TMP $TMP" >> outputs/tmp -echo "TEMP $TEMP" >> outputs/tmp -echo "TMPDIR $TMPDIR" >> outputs/tmp - GALAXY_PYTHON=`command -v python` cd $working_directory $memory_statement