From cbbe316e100d82e9d040bece53fd472e5d1b2720 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Tue, 29 Sep 2020 19:21:47 +0200 Subject: [PATCH 1/3] allow copying datasets rather than uploading in StagingInterface --- lib/galaxy/tool_util/client/staging.py | 10 ++++++++++ lib/galaxy/tool_util/cwl/util.py | 14 ++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_util/client/staging.py b/lib/galaxy/tool_util/client/staging.py index 895ad810b43..edbd9dc142b 100644 --- a/lib/galaxy/tool_util/client/staging.py +++ b/lib/galaxy/tool_util/client/staging.py @@ -200,6 +200,15 @@ class StagingInterace(object): payload["files_0|url_paste"] = content return self._tools_post(payload) + # To be used when datasets are already available and can be copied instead of uploaded + def copy_func(dataset_id): + copy_payload = { + 'content': dataset_id, + 'source': 'hda', + 'type': 'dataset' + } + return self._post('histories/{}/contents'.format(history_id), payload=copy_payload) + def create_collection_func(element_identifiers, collection_type): payload = { "name": "dataset collection", @@ -230,6 +239,7 @@ class StagingInterace(object): job, job_dir, upload, + copy_func, create_collection_func, tool_or_workflow, ) diff --git a/lib/galaxy/tool_util/cwl/util.py b/lib/galaxy/tool_util/cwl/util.py index 990636db2c4..827eceef3d7 100644 --- a/lib/galaxy/tool_util/cwl/util.py +++ b/lib/galaxy/tool_util/cwl/util.py @@ -84,7 +84,7 @@ def path_or_uri_to_uri(path_or_uri): def galactic_job_json( - job, test_data_directory, upload_func, collection_create_func, tool_or_workflow="workflow" + job, test_data_directory, upload_func, copy_func, collection_create_func, tool_or_workflow="workflow" ): """Adapt a CWL job object to the Galaxy API. @@ -145,6 +145,7 @@ def galactic_job_json( is_file = item_class == "File" is_directory = item_class == "Directory" is_collection = item_class == "Collection" # Galaxy extension. + is_galaxy_id = item_class == "GalaxyID" # Galaxy dataset/collection ID. if force_to_file: if is_file: @@ -167,6 +168,8 @@ def galactic_job_json( return replacement_directory(value) elif is_collection: return replacement_collection(value) + elif is_galaxy_id: + return replacement_galaxy_id(value) else: return replacement_record(value) @@ -287,6 +290,12 @@ def galactic_job_json( hdca_id = collection["id"] return {"src": "hdca", "id": hdca_id} + def replacement_galaxy_id(value): + response = copy_func(value['location']) + target = FileLiteralTarget(contents=None) + datasets.append((response, target)) + return {"src": "hda", "id": value['location']} + def replacement_record(value): collection_element_identifiers = [] for record_key, record_value in value.items(): @@ -328,9 +337,10 @@ def _ensure_file_exists(file_path): class FileLiteralTarget: - def __init__(self, contents, **kwargs): + def __init__(self, contents, path=None, **kwargs): self.contents = contents self.properties = kwargs + self.path = path def __str__(self): return "FileLiteralTarget[path={}] with {}".format(self.path, self.properties) From 2652f58314861a19060875acdea8102d9070b650 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Wed, 30 Sep 2020 10:47:58 +0200 Subject: [PATCH 2/3] remove copy_func, appears unnecessary --- lib/galaxy/tool_util/client/staging.py | 10 ---------- lib/galaxy/tool_util/cwl/util.py | 17 +++++++++-------- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/tool_util/client/staging.py b/lib/galaxy/tool_util/client/staging.py index edbd9dc142b..895ad810b43 100644 --- a/lib/galaxy/tool_util/client/staging.py +++ b/lib/galaxy/tool_util/client/staging.py @@ -200,15 +200,6 @@ class StagingInterace(object): payload["files_0|url_paste"] = content return self._tools_post(payload) - # To be used when datasets are already available and can be copied instead of uploaded - def copy_func(dataset_id): - copy_payload = { - 'content': dataset_id, - 'source': 'hda', - 'type': 'dataset' - } - return self._post('histories/{}/contents'.format(history_id), payload=copy_payload) - def create_collection_func(element_identifiers, collection_type): payload = { "name": "dataset collection", @@ -239,7 +230,6 @@ class StagingInterace(object): job, job_dir, upload, - copy_func, create_collection_func, tool_or_workflow, ) diff --git a/lib/galaxy/tool_util/cwl/util.py b/lib/galaxy/tool_util/cwl/util.py index 827eceef3d7..427deb0f074 100644 --- a/lib/galaxy/tool_util/cwl/util.py +++ b/lib/galaxy/tool_util/cwl/util.py @@ -84,7 +84,7 @@ def path_or_uri_to_uri(path_or_uri): def galactic_job_json( - job, test_data_directory, upload_func, copy_func, collection_create_func, tool_or_workflow="workflow" + job, test_data_directory, upload_func, collection_create_func, tool_or_workflow="workflow" ): """Adapt a CWL job object to the Galaxy API. @@ -145,7 +145,7 @@ def galactic_job_json( is_file = item_class == "File" is_directory = item_class == "Directory" is_collection = item_class == "Collection" # Galaxy extension. - is_galaxy_id = item_class == "GalaxyID" # Galaxy dataset/collection ID. + # is_galaxy_id = item_class == "GalaxyID" # Galaxy dataset/collection ID. if force_to_file: if is_file: @@ -168,12 +168,14 @@ def galactic_job_json( return replacement_directory(value) elif is_collection: return replacement_collection(value) - elif is_galaxy_id: - return replacement_galaxy_id(value) + # elif is_galaxy_id: + # return replacement_galaxy_id(value) else: return replacement_record(value) def replacement_file(value): + if value.get('galaxy_id'): + return replacement_galaxy_id(value) file_path = value.get("location", None) or value.get("path", None) # format to match output definitions in tool, where did filetype come from? filetype = value.get("filetype", None) or value.get("format", None) @@ -281,6 +283,8 @@ def galactic_job_json( return collection_element_identifiers def replacement_collection(value): + if value.get('galaxy_id'): + return replacement_galaxy_id(value) assert "collection_type" in value collection_type = value["collection_type"] elements = to_elements(value, collection_type) @@ -291,10 +295,7 @@ def galactic_job_json( return {"src": "hdca", "id": hdca_id} def replacement_galaxy_id(value): - response = copy_func(value['location']) - target = FileLiteralTarget(contents=None) - datasets.append((response, target)) - return {"src": "hda", "id": value['location']} + return {"src": "hda", "id": value['galaxy_id']} def replacement_record(value): collection_element_identifiers = [] From f653eb69c4926a2dd1a77fbdd6fd6b2e1eaad549 Mon Sep 17 00:00:00 2001 From: Simon Bray Date: Wed, 30 Sep 2020 14:25:22 +0200 Subject: [PATCH 3/3] tidy --- lib/galaxy/tool_util/cwl/util.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/tool_util/cwl/util.py b/lib/galaxy/tool_util/cwl/util.py index 427deb0f074..a4605653f18 100644 --- a/lib/galaxy/tool_util/cwl/util.py +++ b/lib/galaxy/tool_util/cwl/util.py @@ -145,7 +145,6 @@ def galactic_job_json( is_file = item_class == "File" is_directory = item_class == "Directory" is_collection = item_class == "Collection" # Galaxy extension. - # is_galaxy_id = item_class == "GalaxyID" # Galaxy dataset/collection ID. if force_to_file: if is_file: @@ -168,14 +167,12 @@ def galactic_job_json( return replacement_directory(value) elif is_collection: return replacement_collection(value) - # elif is_galaxy_id: - # return replacement_galaxy_id(value) else: return replacement_record(value) def replacement_file(value): if value.get('galaxy_id'): - return replacement_galaxy_id(value) + return {"src": "hda", "id": value['galaxy_id']} file_path = value.get("location", None) or value.get("path", None) # format to match output definitions in tool, where did filetype come from? filetype = value.get("filetype", None) or value.get("format", None) @@ -284,7 +281,7 @@ def galactic_job_json( def replacement_collection(value): if value.get('galaxy_id'): - return replacement_galaxy_id(value) + return {"src": "hdca", "id": value['galaxy_id']} assert "collection_type" in value collection_type = value["collection_type"] elements = to_elements(value, collection_type) @@ -294,9 +291,6 @@ def galactic_job_json( hdca_id = collection["id"] return {"src": "hdca", "id": hdca_id} - def replacement_galaxy_id(value): - return {"src": "hda", "id": value['galaxy_id']} - def replacement_record(value): collection_element_identifiers = [] for record_key, record_value in value.items():