opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Switch most tests to use `__DATA_FETCH__ instead of upload1`

Browse Source
This commit is contained in:
mvdbeek
2022-04-23 12:05:54 +02:00
parent 4cc3f33f30
commit 9abbac0180
23 changed files with 475 additions and 301 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/tool_util/verify/interactor.py
+46 -32
View File
@@ -16,6 +16,7 @@ from typing import Optional
import requests
from packaging.version import parse as parse_version
from packaging.version import Version
from requests.cookies import RequestsCookieJar
try:
from nose.tools import nottest
@@ -829,42 +830,55 @@ class GalaxyInteractorApi:
allow files to be specified with the json parameter - so rewrite the parameters
to handle that if as_json is True with specified files.
"""
params = params or {}
data = data or {}
return prepare_request_params(
data=data, files=files, as_json=as_json, params=params, headers=headers, cookies=self.cookies
)
# handle encoded files
if files is None:
# if not explicitly passed, check __files... convention used in tool testing
# and API testing code
files = data.get("__files", None)
if files is not None:
del data["__files"]
# files doesn't really work with json, so dump the parameters
# and do a normal POST with request's data parameter.
if bool(files) and as_json:
as_json = False
new_items = {}
for key, val in data.items():
if isinstance(val, dict) or isinstance(val, list):
new_items[key] = dumps(val)
data.update(new_items)
def prepare_request_params(
data=None,
files=None,
as_json: bool = False,
params: Optional[dict] = None,
headers: Optional[dict] = None,
cookies: Optional[RequestsCookieJar] = None,
):
params = params or {}
data = data or {}
kwd = {
"files": files,
}
if headers:
kwd["headers"] = headers
if as_json:
kwd["json"] = data or None
kwd["params"] = params
else:
data.update(params)
kwd["data"] = data
if self.cookies:
kwd["cookies"] = self.cookies
# handle encoded files
if files is None:
# if not explicitly passed, check __files... convention used in tool testing
# and API testing code
files = data.get("__files", None)
if files is not None:
del data["__files"]
return kwd
# files doesn't really work with json, so dump the parameters
# and do a normal POST with request's data parameter.
if bool(files) and as_json:
as_json = False
new_items = {}
for key, val in data.items():
if isinstance(val, dict) or isinstance(val, list):
new_items[key] = dumps(val)
data.update(new_items)
kwd = {
"files": files,
}
if headers:
kwd["headers"] = headers
if as_json:
kwd["json"] = data or None
kwd["params"] = params
else:
data.update(params)
kwd["data"] = data
if cookies:
kwd["cookies"] = cookies
return kwd
def ensure_tool_run_response_okay(submit_response_object, request_desc, inputs=None):
lib/galaxy_test/api/test_dataset_collections.py
+21 -13
View File
@@ -1,4 +1,3 @@
import json
import zipfile
from io import BytesIO
from typing import List
@@ -26,7 +25,7 @@ class DatasetCollectionApiTestCase(ApiTestCase):
self.history_id,
instance_type="history",
)
create_response = self._post("dataset_collections", payload, json=True)
create_response = self.dataset_populator.fetch(payload, wait=True)
dataset_collection = self._check_create_response(create_response)
returned_datasets = dataset_collection["elements"]
assert len(returned_datasets) == 2, dataset_collection
@@ -51,7 +50,7 @@ class DatasetCollectionApiTestCase(ApiTestCase):
self.history_id,
instance_type="history",
)
pair_create_response = self._post("dataset_collections", pair_payload, json=True)
pair_create_response = self._post("tools/fetch", pair_payload, json=True)
dataset_collection = self._check_create_response(pair_create_response)
hdca_id = dataset_collection["id"]
@@ -153,8 +152,9 @@ class DatasetCollectionApiTestCase(ApiTestCase):
)
def test_list_list_download(self):
dataset_collection = self.dataset_collection_populator.create_list_of_list_in_history(self.history_id).json()
self.dataset_collection_populator.wait_for_dataset_collection(dataset_collection, assert_ok=True)
dataset_collection = self.dataset_collection_populator.create_list_of_list_in_history(
self.history_id, wait=True
).json()
returned_dce = dataset_collection["elements"]
assert len(returned_dce) == 1, dataset_collection
create_response = self._download_dataset_collection(
@@ -167,9 +167,10 @@ class DatasetCollectionApiTestCase(ApiTestCase):
def test_list_list_list_download(self):
dataset_collection = self.dataset_collection_populator.create_list_of_list_in_history(
self.history_id, collection_type="list:list:list"
self.history_id,
collection_type="list:list:list",
wait=True,
).json()
self.dataset_collection_populator.wait_for_dataset_collection(dataset_collection, assert_ok=True)
returned_dce = dataset_collection["elements"]
assert len(returned_dce) == 1, dataset_collection
create_response = self._download_dataset_collection(
@@ -227,7 +228,7 @@ class DatasetCollectionApiTestCase(ApiTestCase):
]
payload = {
"history_id": self.history_id,
"targets": json.dumps(targets),
"targets": targets,
"__files": {"files_0|file_data": open(self.test_data_resolver.get_filename("4.bed"))},
}
self.dataset_populator.fetch(payload)
@@ -256,7 +257,7 @@ class DatasetCollectionApiTestCase(ApiTestCase):
]
payload = {
"history_id": self.history_id,
"targets": json.dumps(targets),
"targets": targets,
"__files": {"files_0|file_data": open(self.test_data_resolver.get_filename("4.bed"))},
}
self.dataset_populator.fetch(payload)
@@ -284,7 +285,7 @@ class DatasetCollectionApiTestCase(ApiTestCase):
]
payload = {
"history_id": self.history_id,
"targets": json.dumps(targets),
"targets": targets,
}
self.dataset_populator.fetch(payload)
hdca = self._assert_one_collection_created_in_history()
@@ -306,7 +307,7 @@ class DatasetCollectionApiTestCase(ApiTestCase):
]
payload = {
"history_id": self.history_id,
"targets": json.dumps(targets),
"targets": targets,
}
self.dataset_populator.fetch(payload, assert_ok=False, wait=True)
hdca = self._assert_one_collection_created_in_history()
@@ -328,6 +329,10 @@ class DatasetCollectionApiTestCase(ApiTestCase):
def _check_create_response(self, create_response):
self._assert_status_code_is(create_response, 200)
dataset_collection = create_response.json()
if "output_collections" in dataset_collection:
# fetch data response, we'll have to check the final response
dataset_collection = dataset_collection["output_collections"][0]
dataset_collection = self._get(f"dataset_collections/{dataset_collection['id']}").json()
self._assert_has_keys(dataset_collection, "elements", "url", "name", "collection_type", "element_count")
return dataset_collection
@@ -418,6 +423,7 @@ class DatasetCollectionApiTestCase(ApiTestCase):
],
},
],
wait=True,
)
self._assert_status_code_is(response, 200)
hdca_list_id = response.json()["outputs"][0]["id"]
@@ -461,6 +467,7 @@ class DatasetCollectionApiTestCase(ApiTestCase):
],
},
],
wait=True,
)
self._assert_status_code_is(response, 200)
hdca_list_id = response.json()["outputs"][0]["id"]
@@ -491,6 +498,7 @@ class DatasetCollectionApiTestCase(ApiTestCase):
],
},
],
wait=True,
)
self._assert_status_code_is(response, 200)
hdca_list_id = response.json()["outputs"][0]["id"]
@@ -513,7 +521,7 @@ class DatasetCollectionApiTestCase(ApiTestCase):
]
payload = {
"history_id": self.history_id,
"targets": json.dumps(targets),
"targets": targets,
"__files": {"files_0|file_data": open(self.test_data_resolver.get_filename("4.bed"))},
}
hdca_id = self.dataset_populator.fetch(payload).json()["output_collections"][0]["id"]
@@ -545,7 +553,7 @@ class DatasetCollectionApiTestCase(ApiTestCase):
def _create_collection_contents_pair(self):
# Create a simple collection, return hdca and contents_url
payload = self.dataset_collection_populator.create_pair_payload(self.history_id, instance_type="history")
create_response = self._post("dataset_collections", payload, json=True)
create_response = self.dataset_populator.fetch(payload=payload, wait=True)
hdca = self._check_create_response(create_response)
root_contents_url = self._get_contents_url_for_hdca(hdca)
return hdca, root_contents_url
lib/galaxy_test/api/test_datasets.py
+48 -19
View File
@@ -63,20 +63,24 @@ class DatasetsApiTestCase(ApiTestCase):
def test_search_datasets(self):
hda_id = self.dataset_populator.new_dataset(self.history_id)["id"]
payload = {"limit": 1, "offset": 0}
payload = {"limit": 1, "offset": 0, "history_id": self.history_id}
index_response = self._get("datasets", payload).json()
assert len(index_response) == 1
assert index_response[0]["id"] == hda_id
hdca_id = self.dataset_collection_populator.create_list_in_history(
fetch_response = self.dataset_collection_populator.create_list_in_history(
self.history_id, contents=["1\n2\n3"]
).json()["id"]
index_payload_1 = {"limit": 3, "offset": 0}
).json()
hdca_id = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response)["id"]
index_payload_1 = {"limit": 3, "offset": 0, "order": "hid", "history_id": self.history_id}
index_response = self._get("datasets", index_payload_1).json()
assert len(index_response) == 3
assert index_response[0]["id"] == hdca_id
assert index_response[0]["history_content_type"] == "dataset_collection"
assert index_response[2]["id"] == hda_id
assert index_response[0]["hid"] == 3
assert index_response[1]["hid"] == 2
assert index_response[2]["hid"] == 1
assert index_response[2]["history_content_type"] == "dataset"
assert index_response[2]["id"] == hda_id
assert index_response[1]["history_content_type"] == "dataset_collection"
assert index_response[1]["id"] == hdca_id
index_payload_2 = {"limit": 2, "offset": 0, "q": ["history_content_type"], "qv": ["dataset"]}
index_response = self._get("datasets", index_payload_2).json()
assert index_response[1]["id"] == hda_id
@@ -89,7 +93,13 @@ class DatasetsApiTestCase(ApiTestCase):
updated_hda = self._put(f"histories/{self.history_id}/contents/{hda_id}", update_payload, json=True).json()
assert "cool:new_tag" in updated_hda["tags"]
assert "cool:another_tag" in updated_hda["tags"]
payload = {"limit": 10, "offset": 0, "q": ["history_content_type", "tag"], "qv": ["dataset", "cool:new_tag"]}
payload = {
"limit": 10,
"offset": 0,
"q": ["history_content_type", "tag"],
"qv": ["dataset", "cool:new_tag"],
"history_id": self.history_id,
}
index_response = self._get("datasets", payload).json()
assert len(index_response) == 1
payload = {
@@ -97,36 +107,55 @@ class DatasetsApiTestCase(ApiTestCase):
"offset": 0,
"q": ["history_content_type", "tag-contains"],
"qv": ["dataset", "new_tag"],
"history_id": self.history_id,
}
index_response = self._get("datasets", payload).json()
assert len(index_response) == 1
payload = {"limit": 10, "offset": 0, "q": ["history_content_type", "tag-contains"], "qv": ["dataset", "notag"]}
payload = {
"limit": 10,
"offset": 0,
"q": ["history_content_type", "tag-contains"],
"qv": ["dataset", "notag"],
"history_id": self.history_id,
}
index_response = self._get("datasets", payload).json()
assert len(index_response) == 0
def test_search_by_tool_id(self):
self.dataset_populator.new_dataset(self.history_id)
payload = {"limit": 1, "offset": 0, "q": ["history_content_type", "tool_id"], "qv": ["dataset", "upload1"]}
payload = {
"limit": 1,
"offset": 0,
"q": ["history_content_type", "tool_id"],
"qv": ["dataset", "__DATA_FETCH__"],
"history_id": self.history_id,
}
assert len(self._get("datasets", payload).json()) == 1
payload = {"limit": 1, "offset": 0, "q": ["history_content_type", "tool_id"], "qv": ["dataset", "uploadX"]}
payload = {
"limit": 1,
"offset": 0,
"q": ["history_content_type", "tool_id"],
"qv": ["dataset", "__DATA_FETCH__X"],
"history_id": self.history_id,
}
assert len(self._get("datasets", payload).json()) == 0
payload = {
"limit": 1,
"offset": 0,
"q": ["history_content_type", "tool_id-contains"],
"qv": ["dataset", "pload1"],
"qv": ["dataset", "ATA_FETCH"],
"history_id": self.history_id,
}
assert len(self._get("datasets", payload).json()) == 1
self.dataset_collection_populator.create_list_in_history(
self.history_id, name="search by tool id", contents=["1\n2\n3"]
).json()
self.dataset_populator.wait_for_history(self.history_id)
self.history_id, name="search by tool id", contents=["1\n2\n3"], wait=True
)
payload = {
"limit": 10,
"offset": 0,
"history_id": self.history_id,
"q": ["name", "tool_id"],
"qv": ["search by tool id", "upload1"],
"qv": ["search by tool id", "__DATA_FETCH__"],
"history_id": self.history_id,
}
result = self._get("datasets", payload).json()
assert result[0]["name"] == "search by tool id", result
@@ -135,6 +164,7 @@ class DatasetsApiTestCase(ApiTestCase):
"offset": 0,
"q": ["history_content_type", "tool_id"],
"qv": ["dataset_collection", "uploadX"],
"history_id": self.history_id,
}
result = self._get("datasets", payload).json()
assert len(result) == 0
@@ -289,8 +319,7 @@ class DatasetsApiTestCase(ApiTestCase):
assert deleted_result["success_count"] == len(expected_purged_source_ids)
for purged_source_id in expected_purged_source_ids:
dataset = self._get(f"histories/{history_id}/contents/{purged_source_id['id']}").json()
assert dataset["purged"] is True
self.dataset_populator.wait_for_purge(history_id, purged_source_id["id"])
def test_delete_batch_error(self):
num_datasets = 4
lib/galaxy_test/api/test_folder_contents.py
+1 -1
View File
@@ -234,7 +234,7 @@ class FolderContentsApiTestCase(ApiTestCase):
def _create_hdca_with_contents(self, contents: List[str]) -> str:
hdca = self.dataset_collection_populator.create_list_in_history(
self.history_id, contents=contents, direct_upload=True
self.history_id, contents=contents, direct_upload=True, wait=True
).json()["outputs"][0]
hdca_id = hdca["id"]
return hdca_id
lib/galaxy_test/api/test_histories.py
+9 -5
View File
@@ -279,7 +279,7 @@ class ImportExportTests(BaseHistories):
imported_history_id = self._reimport_history(history_id, history_name, wait_on_history_length=2)
def upload_job_check(job):
assert job["tool_id"] == "upload1"
assert job["tool_id"] == "__DATA_FETCH__"
def check_discarded(hda):
assert hda["deleted"]
@@ -319,7 +319,7 @@ class ImportExportTests(BaseHistories):
self._assert_history_length(imported_history_id, 2)
def upload_job_check(job):
assert job["tool_id"] == "upload1"
assert job["tool_id"] == "__DATA_FETCH__"
def check_deleted_not_purged(hda):
assert hda["state"] == "ok", hda
@@ -392,7 +392,7 @@ class ImportExportTests(BaseHistories):
history_name = f"for_export_with_collections_{uuid4()}"
history_id = self.dataset_populator.new_history(name=history_name)
self.dataset_collection_populator.create_list_in_history(
history_id, contents=["Hello", "World"], direct_upload=True
history_id, contents=["Hello", "World"], direct_upload=True, wait=True
)
imported_history_id = self._reimport_history(history_id, history_name, wait_on_history_length=3)
@@ -417,7 +417,8 @@ class ImportExportTests(BaseHistories):
def test_import_export_nested_collection(self):
history_name = f"for_export_with_nested_collections_{uuid4()}"
history_id = self.dataset_populator.new_history(name=history_name)
self.dataset_collection_populator.create_list_of_pairs_in_history(history_id)
fetch_response = self.dataset_collection_populator.create_list_of_pairs_in_history(history_id, wait=True).json()
dataset_collection = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response)
imported_history_id = self._reimport_history(history_id, history_name, wait_on_history_length=3)
self._assert_history_length(imported_history_id, 3)
@@ -433,7 +434,10 @@ class ImportExportTests(BaseHistories):
assert element0["collection_type"] == "paired"
self._check_imported_collection(
imported_history_id, hid=1, collection_type="list:paired", elements_checker=check_elements
imported_history_id,
hid=dataset_collection["hid"],
collection_type="list:paired",
elements_checker=check_elements,
)
def _reimport_history(
lib/galaxy_test/api/test_history_contents.py
+49 -37
View File
@@ -352,19 +352,24 @@ class HistoryContentsApiTestCase(ApiTestCase):
data = {"purge": True}
delete_response = self._delete(f"histories/{self.history_id}/contents/{hda1['id']}", data=data, json=True)
assert delete_response.status_code < 300 # Something in the 200s :).
assert str(self.__show(hda1).json()["deleted"]).lower() == "true"
assert str(self.__show(hda1).json()["purged"]).lower() == "true"
# Purging and deleting the dataset may or may not happen asynchronously.
# On 202 the request was accepted and purging will happen later.
if delete_response.status_code == 202:
self.dataset_populator.wait_for_purge(self.history_id, hda1["id"])
else:
assert self.__show(hda1).json()["deleted"]
assert self.__show(hda1).json()["purged"]
def test_dataset_collection_creation_on_contents(self):
payload = self.dataset_collection_populator.create_pair_payload(self.history_id, type="dataset_collection")
endpoint = f"histories/{self.history_id}/contents"
payload = self.dataset_collection_populator.create_pair_payload(
self.history_id, type="dataset_collection", wait=True
)
endpoint = "tools/fetch"
self._check_pair_creation(endpoint, payload)
def test_dataset_collection_creation_on_typed_contents(self):
payload = self.dataset_collection_populator.create_pair_payload(
self.history_id,
)
endpoint = f"histories/{self.history_id}/contents/dataset_collections"
payload = self.dataset_collection_populator.create_pair_payload(self.history_id, wait=True)
endpoint = "tools/fetch"
self._check_pair_creation(endpoint, payload)
def test_dataset_collection_create_from_exisiting_datasets_with_new_tags(self):
@@ -429,10 +434,10 @@ class HistoryContentsApiTestCase(ApiTestCase):
@skip_without_tool("collection_creates_list")
def test_jobs_summary_simple_hdca(self):
create_response = self.dataset_collection_populator.create_list_in_history(
fetch_response = self.dataset_collection_populator.create_list_in_history(
self.history_id, contents=["a\nb\nc\nd", "e\nf\ng\nh"]
)
hdca_id = create_response.json()["id"]
).json()
hdca_id = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response)["id"]
run = self.dataset_populator.run_collection_creates_list(self.history_id, hdca_id)
collections = run["output_collections"]
collection = collections[0]
@@ -445,9 +450,9 @@ class HistoryContentsApiTestCase(ApiTestCase):
@skip_without_tool("cat1")
def test_jobs_summary_implicit_hdca(self):
create_response = self.dataset_collection_populator.create_pair_in_history(
self.history_id, contents=["123", "456"]
self.history_id, contents=["123", "456"], wait=True
)
hdca_id = create_response.json()["id"]
hdca_id = create_response.json()["outputs"][0]["id"]
inputs = {
"input1": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]},
}
@@ -464,7 +469,9 @@ class HistoryContentsApiTestCase(ApiTestCase):
assert states.get("ok") == 2, states
def test_dataset_collection_hide_originals(self):
payload = self.dataset_collection_populator.create_pair_payload(self.history_id, type="dataset_collection")
payload = self.dataset_collection_populator.create_pair_payload(
self.history_id, type="dataset_collection", direct_upload=False
)
payload["hide_source_items"] = True
dataset_collection_response = self._post(f"histories/{self.history_id}/contents", payload, json=True)
@@ -499,14 +506,14 @@ class HistoryContentsApiTestCase(ApiTestCase):
def _create_pair_collection(self):
payload = self.dataset_collection_populator.create_pair_payload(self.history_id, type="dataset_collection")
dataset_collection_response = self._post(f"histories/{self.history_id}/contents", payload, json=True)
dataset_collection_response = self._post("tools/fetch", payload, json=True)
self._assert_status_code_is(dataset_collection_response, 200)
hdca = dataset_collection_response.json()
hdca = dataset_collection_response.json()["output_collections"][0]
return hdca
def test_hdca_copy(self):
hdca = self.dataset_collection_populator.create_pair_in_history(self.history_id).json()
hdca_id = hdca["id"]
hdca = self.dataset_collection_populator.create_pair_in_history(self.history_id, wait=True).json()
hdca_id = hdca["outputs"][0]["id"]
second_history_id = self.dataset_populator.new_history()
create_data = dict(
source="hdca",
@@ -524,7 +531,8 @@ class HistoryContentsApiTestCase(ApiTestCase):
assert new_forward["history_id"] == self.history_id
def test_hdca_copy_with_new_dbkey(self):
hdca = self.dataset_collection_populator.create_pair_in_history(self.history_id).json()
fetch_response = self.dataset_collection_populator.create_pair_in_history(self.history_id, wait=True).json()
hdca = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response)
hdca_id = hdca["id"]
assert hdca["elements"][0]["object"]["metadata_dbkey"] == "?"
assert hdca["elements"][0]["object"]["genome_build"] == "?"
@@ -538,7 +546,7 @@ class HistoryContentsApiTestCase(ApiTestCase):
assert new_forward["genome_build"] == "hg19"
def test_hdca_copy_and_elements(self):
hdca = self.dataset_collection_populator.create_pair_in_history(self.history_id).json()
hdca = self.dataset_collection_populator.create_pair_in_history(self.history_id, wait=True).json()["outputs"][0]
hdca_id = hdca["id"]
second_history_id = self.dataset_populator.new_history()
create_data = dict(
@@ -615,6 +623,8 @@ class HistoryContentsApiTestCase(ApiTestCase):
def __check_create_collection_response(self, response):
self._assert_status_code_is(response, 200)
dataset_collection = response.json()
if "output_collections" in dataset_collection:
dataset_collection = dataset_collection["output_collections"][0]
self._assert_has_keys(dataset_collection, "url", "name", "deleted", "visible", "elements")
return dataset_collection
@@ -648,7 +658,8 @@ class HistoryContentsApiTestCase(ApiTestCase):
def test_job_state_summary_field(self):
create_response = self.dataset_collection_populator.create_pair_in_history(
self.history_id, contents=["123", "456"]
self.history_id,
contents=["123", "456"],
)
self._assert_status_code_is(create_response, 200)
contents_response = self._get(f"histories/{self.history_id}/contents?v=dev&keys=job_state_summary&view=summary")
@@ -667,7 +678,7 @@ class HistoryContentsApiTestCase(ApiTestCase):
with self.dataset_populator.test_history() as history_id:
first_time = datetime.utcnow().isoformat()
assert len(self._get_content(history_id, update_time=first_time)) == 0
self.dataset_collection_populator.create_list_in_history(history_id=history_id)
self.dataset_collection_populator.create_list_in_history(history_id=history_id, wait=True)
assert len(self._get_content(history_id, update_time=first_time)) == 4 # 3 datasets
self.dataset_populator.wait_for_history(history_id)
all_datasets_finished = first_time = datetime.utcnow().isoformat()
@@ -690,7 +701,7 @@ class HistoryContentsApiTestCase(ApiTestCase):
assert history_contents.status_code == 204
# add some stuff
self.dataset_collection_populator.create_list_in_history(history_id=history_id)
self.dataset_collection_populator.create_list_in_history(history_id=history_id, wait=True)
self.dataset_populator.wait_for_history(history_id)
# check to make sure the added stuff is there
@@ -739,7 +750,8 @@ class HistoryContentsApiTestCase(ApiTestCase):
@skip_without_tool("cat_data_and_sleep")
def test_history_contents_near_with_update_time_implicit_collection(self):
with self.dataset_populator.test_history() as history_id:
hdca_id = self.dataset_collection_populator.create_list_in_history(history_id=history_id).json()["id"]
fetch_response = self.dataset_collection_populator.create_list_in_history(history_id=history_id).json()
hdca_id = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response)["id"]
self.dataset_populator.wait_for_history(history_id)
inputs = {
"input1": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]},
@@ -792,7 +804,7 @@ class HistoryContentsApiTestCase(ApiTestCase):
def test_index_filter_by_type(self):
history_id = self.dataset_populator.new_history()
self.dataset_populator.new_dataset(history_id)
self.dataset_collection_populator.create_list_in_history(history_id=history_id)
self.dataset_collection_populator.create_list_in_history(history_id=history_id, wait=True)
contents_response = self._get(f"histories/{history_id}/contents").json()
num_items = len(contents_response)
@@ -830,7 +842,7 @@ class HistoryContentsApiTestCase(ApiTestCase):
def _upload_collection_list_with_elements(self, history_id: str, collection_name: str, elements: List[Any]):
create_homogeneous_response = self.dataset_collection_populator.upload_collection(
history_id, "list", elements=elements, name=collection_name
history_id, "list", elements=elements, name=collection_name, wait=True
)
self._assert_status_code_is_ok(create_homogeneous_response)
@@ -860,7 +872,7 @@ class HistoryContentsApiNearTestCase(ApiTestCase):
def _create_list_in_history(self, history_id, n=2):
# Creates list of size n*4 (n collections with 3 items each)
for _ in range(n):
self.dataset_collection_populator.create_list_in_history(history_id=history_id)
self.dataset_collection_populator.create_list_in_history(history_id=history_id, wait=True)
def _get_content(self, history_id, direction, *, hid, limit=1000):
return self._get(f"/api/histories/{history_id}/contents/{direction}/{hid}/{limit}").json()
@@ -971,10 +983,6 @@ class HistoryContentsApiBulkOperationTestCase(ApiTestCase):
with self.dataset_populator.test_history() as history_id:
datasets_ids, collection_ids, history_contents = self._create_test_history_contents(history_id)
# All items are visible
for item in history_contents:
assert item["visible"]
# Hide 2 collections and 3 datasets, 5 in total
payload = {
"operation": "hide",
@@ -1016,10 +1024,6 @@ class HistoryContentsApiBulkOperationTestCase(ApiTestCase):
with self.dataset_populator.test_history() as history_id:
_, collection_ids, history_contents = self._create_test_history_contents(history_id)
# All items are visible
for item in history_contents:
assert item["visible"]
# Hide all collections using query
payload = {"operation": "hide"}
query = "q=history_content_type-eq&qv=dataset_collection"
@@ -1167,12 +1171,20 @@ class HistoryContentsApiBulkOperationTestCase(ApiTestCase):
datasets_ids = list(map(lambda dataset: dataset["id"], datasets))
assert len(history_contents) == num_expected_datasets + num_expected_collections
assert len(datasets_ids) == num_expected_datasets
for dataset_id in datasets_ids:
self._put(f"histories/{history_id}/contents/{dataset_id}", {"visible": True}, json=True).json()
# All items are visible
history_contents = self._get_history_contents(history_id)
for item in history_contents:
assert item["visible"]
return datasets_ids, collection_ids, history_contents
def _create_collection_in_history(self, history_id, num_collections=1) -> List[str]:
collection_ids = []
for _ in range(num_collections):
collection_id = self.dataset_collection_populator.create_pair_in_history(history_id=history_id).json()["id"]
collection_id = self.dataset_collection_populator.create_pair_in_history(
history_id=history_id, wait=True
).json()["outputs"][0]["id"]
collection_ids.append(collection_id)
return collection_ids
@@ -1180,7 +1192,7 @@ class HistoryContentsApiBulkOperationTestCase(ApiTestCase):
return self._get(f"histories/{history_id}/contents").json()
def _get_hidden_items_from_history_contents(self, history_contents) -> List[Any]:
return list(filter(lambda item: item["visible"] is False, history_contents))
return [content for content in history_contents if not content["visible"]]
def _apply_bulk_operation(self, history_id: str, payload, query: str = ""):
if query:
lib/galaxy_test/api/test_jobs.py
+5 -4
View File
@@ -203,7 +203,7 @@ steps:
# Initial number of ok jobs
original_count = len(self.__uploads_with_state("ok", "new"))
# Run through dataset upload to ensure num uplaods at least greater
# Run through dataset upload to ensure num uploads at least greater
# by 1.
self.__history_with_ok_dataset(history_id)
@@ -280,9 +280,10 @@ steps:
assert dataset["visible"]
def _run_map_over_error(self, history_id):
hdca1 = self.dataset_collection_populator.create_list_in_history(
fetch_response = self.dataset_collection_populator.create_list_in_history(
history_id, contents=[("sample1-1", "1 2 3")]
).json()
hdca1 = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response)
inputs = {
"error_bool": "true",
"dataset": {
@@ -874,10 +875,10 @@ steps:
self._assert_status_code_is(jobs_response, 200)
jobs = jobs_response.json()
assert not [j for j in jobs if not j["state"] in states]
return [j for j in jobs if j["tool_id"] == "upload1"]
return [j for j in jobs if j["tool_id"] == "__DATA_FETCH__"]
def __history_with_new_dataset(self, history_id):
dataset_id = self.dataset_populator.new_dataset(history_id)["id"]
dataset_id = self.dataset_populator.new_dataset(history_id, wait=True)["id"]
return dataset_id
def __history_with_ok_dataset(self, history_id):
lib/galaxy_test/api/test_libraries.py
+12 -13
View File
@@ -1,4 +1,3 @@
import json
import unittest
import pytest
@@ -222,10 +221,10 @@ class LibrariesApiTestCase(ApiTestCase):
targets = [{"destination": destination, "items": items}]
payload = {
"history_id": history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
"targets": targets,
"__files": {"files_0|file_data": open(self.test_data_resolver.get_filename("4.bed"))},
}
self.dataset_populator.fetch(payload)
self.dataset_populator.fetch(payload, wait=True)
dataset = self.library_populator.get_library_contents_with_path(library["id"], "/4.bed")
assert dataset["file_size"] == 61, dataset
assert dataset["genome_build"] == "hg19", dataset
@@ -245,7 +244,7 @@ class LibrariesApiTestCase(ApiTestCase):
]
payload = {
"history_id": history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
"targets": targets,
"__files": {"files_0|file_data": open(bed_test_data_path, "rb")},
}
self.dataset_populator.fetch(payload)
@@ -280,7 +279,7 @@ class LibrariesApiTestCase(ApiTestCase):
]
payload = {
"history_id": history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
"targets": targets,
"validate_hashes": True,
}
return library, self.dataset_populator.fetch(payload, assert_ok=assert_ok)
@@ -322,7 +321,7 @@ class LibrariesApiTestCase(ApiTestCase):
]
payload = {
"history_id": history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
"targets": targets,
"validate_hashes": True,
}
tool_response = self.dataset_populator.fetch(payload, assert_ok=False)
@@ -349,7 +348,7 @@ class LibrariesApiTestCase(ApiTestCase):
]
payload = {
"history_id": history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
"targets": targets,
}
self.dataset_populator.fetch(payload)
dataset = self.library_populator.get_library_contents_with_path(library["id"], "/4.bed")
@@ -368,7 +367,7 @@ class LibrariesApiTestCase(ApiTestCase):
]
payload = {
"history_id": history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
"targets": targets,
"__files": {"files_0|file_data": open(example_bag_path)},
}
self.dataset_populator.fetch(payload)
@@ -520,7 +519,7 @@ class LibrariesApiTestCase(ApiTestCase):
folder_id = folder_response.json()[0]["id"]
history_id = self.dataset_populator.new_history()
hdca_id = self.dataset_collection_populator.create_list_in_history(
history_id, contents=["xxx", "yyy"], direct_upload=True
history_id, contents=["xxx", "yyy"], direct_upload=True, wait=True
).json()["outputs"][0]["id"]
payload = {"from_hdca_id": hdca_id, "create_type": "file", "folder_id": folder_id}
create_response = self._post(f"libraries/{library['id']}/contents", payload)
@@ -530,7 +529,7 @@ class LibrariesApiTestCase(ApiTestCase):
library = self.library_populator.new_private_library("ForCreateDatasetsFromCollection")
history_id = self.dataset_populator.new_history()
hdca_id = self.dataset_collection_populator.create_list_in_history(
history_id, contents=["xxx", "yyy"], direct_upload=True
history_id, contents=["xxx", "yyy"], direct_upload=True, wait=True
).json()["outputs"][0]["id"]
folder_response = self._create_folder(library)
self._assert_status_code_is(folder_response, 200)
@@ -541,9 +540,9 @@ class LibrariesApiTestCase(ApiTestCase):
assert len(create_response.json()) == 2
# Also test that anything different from a flat dataset collection list
# is refused
hdca_pair_id = self.dataset_collection_populator.create_list_of_pairs_in_history(history_id).json()["outputs"][
0
]["id"]
hdca_pair_id = self.dataset_collection_populator.create_list_of_pairs_in_history(history_id, wait=True).json()[
"outputs"
][0]["id"]
payload = {"from_hdca_id": hdca_pair_id}
create_response = self._post(f"folders/{folder_id}/contents", payload)
self._assert_status_code_is(create_response, 501)
lib/galaxy_test/api/test_tools.py
+84 -61
View File
@@ -59,7 +59,10 @@ class TestsTools:
def _build_pair(self, history_id, contents):
create_response = self.dataset_collection_populator.create_pair_in_history(
history_id, contents=contents, direct_upload=True
history_id,
contents=contents,
direct_upload=True,
wait=True,
)
hdca_id = create_response.json()["outputs"][0]["id"]
return hdca_id
@@ -518,7 +521,7 @@ class ToolsTestCase(ApiTestCase, TestsTools):
],
},
]
hdca1 = self.dataset_collection_populator.upload_collection(history_id, "list:paired", elements=data)
hdca1 = self.dataset_collection_populator.upload_collection(history_id, "list:paired", elements=data, wait=True)
self._assert_status_code_is(hdca1, 200)
payload = {
@@ -529,14 +532,12 @@ class ToolsTestCase(ApiTestCase, TestsTools):
"history_id": history_id,
}
create_response = self._post("tools/CONVERTER_fasta_to_tabular/convert", payload)
self.dataset_populator.wait_for_job(create_response.json()["jobs"][0]["id"], assert_ok=True)
create_response.raise_for_status()
assert create_response.json()["implicit_collections"] != []
hdca_id = create_response.json()["implicit_collections"][0]["hid"]
fetchedResponse = self.dataset_populator.get_history_collection_details(history_id, hid=hdca_id)
for element in fetchedResponse["elements"][0]["object"]["elements"]:
hid = create_response.json()["implicit_collections"][0]["hid"]
self.dataset_populator.wait_for_history(history_id, assert_ok=True)
collection_details = self.dataset_populator.get_history_collection_details(history_id, hid=hid)
for element in collection_details["elements"][0]["object"]["elements"]:
assert element["object"]["file_ext"] == "tabular"
def test_unzip_collection(self):
@@ -592,6 +593,7 @@ class ToolsTestCase(ApiTestCase, TestsTools):
],
}
],
wait=True,
)
self._assert_status_code_is(response, 200)
hdca_id = response.json()["outputs"][0]["id"]
@@ -601,7 +603,6 @@ class ToolsTestCase(ApiTestCase, TestsTools):
"values": [{"src": "hdca", "map_over_type": "paired", "id": hdca_id}],
}
}
self.dataset_populator.wait_for_history(history_id, assert_ok=True)
response = self._run("__UNZIP_COLLECTION__", history_id, inputs, assert_ok=True)
implicit_collections = response["implicit_collections"]
self.assertEqual(len(implicit_collections), 2)
@@ -645,7 +646,9 @@ class ToolsTestCase(ApiTestCase, TestsTools):
@skip_without_tool("__UNZIP_COLLECTION__")
@uses_test_history(require_new=False)
def test_collection_operation_collection_input_permissions(self, history_id):
create_response = self.dataset_collection_populator.create_pair_in_history(history_id, direct_upload=True)
create_response = self.dataset_collection_populator.create_pair_in_history(
history_id, direct_upload=True, wait=True
)
self._assert_status_code_is(create_response, 200)
collection = create_response.json()["outputs"][0]
self.dataset_populator.wait_for_history(history_id, assert_ok=True)
@@ -662,11 +665,12 @@ class ToolsTestCase(ApiTestCase, TestsTools):
def test_zip_list_inputs(self):
with self.dataset_populator.test_history() as history_id:
hdca1_id = self.dataset_collection_populator.create_list_in_history(
history_id, contents=["a\nb\nc\nd", "e\nf\ng\nh"]
).json()["id"]
history_id, contents=["a\nb\nc\nd", "e\nf\ng\nh"], wait=True
).json()["outputs"][0]["id"]
hdca2_id = self.dataset_collection_populator.create_list_in_history(
history_id, contents=["1\n2\n3\n4", "5\n6\n7\n8"]
).json()["id"]
history_id, contents=["1\n2\n3\n4", "5\n6\n7\n8"], wait=True
).json()["outputs"][0]["id"]
inputs = {
"input_forward": {"batch": True, "values": [{"src": "hdca", "id": hdca1_id}]},
"input_reverse": {"batch": True, "values": [{"src": "hdca", "id": hdca2_id}]},
@@ -685,8 +689,8 @@ class ToolsTestCase(ApiTestCase, TestsTools):
def test_filter_failed_list(self):
with self.dataset_populator.test_history() as history_id:
ok_hdca_id = self.dataset_collection_populator.create_list_in_history(
history_id, contents=["0", "1", "0", "1"]
).json()["id"]
history_id, contents=["0", "1", "0", "1"], wait=True
).json()["outputs"][0]["id"]
response = self.dataset_populator.run_exit_code_from_file(history_id, ok_hdca_id)
mixed_implicit_collections = response["implicit_collections"]
@@ -709,12 +713,12 @@ class ToolsTestCase(ApiTestCase, TestsTools):
@skip_without_tool("__FILTER_FAILED_DATASETS__")
def test_filter_failed_list_paired(self):
with self.dataset_populator.test_history() as history_id:
pair1 = self.dataset_collection_populator.create_pair_in_history(history_id, contents=["0", "0"]).json()[
"id"
]
pair2 = self.dataset_collection_populator.create_pair_in_history(history_id, contents=["0", "1"]).json()[
"id"
]
pair1 = self.dataset_collection_populator.create_pair_in_history(
history_id, contents=["0", "0"], wait=True
).json()["outputs"][0]["id"]
pair2 = self.dataset_collection_populator.create_pair_in_history(
history_id, contents=["0", "1"], wait=True
).json()["outputs"][0]["id"]
ok_hdca_id = self.dataset_collection_populator.create_list_from_pairs(history_id, [pair1, pair2]).json()[
"id"
]
@@ -1003,7 +1007,7 @@ class ToolsTestCase(ApiTestCase, TestsTools):
return element0["id"]
response = self.dataset_collection_populator.create_list_in_history(
history_id, contents=["a\nb\nc\nd", "e\nf\ng\nh"], direct_upload=True
history_id, contents=["a\nb\nc\nd", "e\nf\ng\nh"], direct_upload=True, wait=True
)
self._assert_status_code_is(response, 200)
hdca = response.json()["output_collections"][0]
@@ -1123,15 +1127,15 @@ class ToolsTestCase(ApiTestCase, TestsTools):
@uses_test_history(require_new=False)
def test_list_collection_output(self, history_id):
create_response = self.dataset_collection_populator.create_list_in_history(
history_id, contents=["a\nb\nc\nd", "e\nf\ng\nh"]
history_id, contents=["a\nb\nc\nd", "e\nf\ng\nh"], wait=True
)
hdca_id = create_response.json()["id"]
hdca_id = create_response.json()["outputs"][0]["id"]
create = self.dataset_populator.run_collection_creates_list(history_id, hdca_id)
output_collection = self._assert_one_job_one_collection_run(create)
element0, element1 = self._assert_elements_are(output_collection, "data1", "data2")
element0, element1 = self._assert_elements_are(output_collection, "data0", "data1")
self.dataset_populator.wait_for_history(history_id, assert_ok=True)
self._verify_element(history_id, element0, contents="identifier is data1\n", file_ext="txt")
self._verify_element(history_id, element1, contents="identifier is data2\n", file_ext="txt")
self._verify_element(history_id, element0, contents="identifier is data0\n", file_ext="txt")
self._verify_element(history_id, element1, contents="identifier is data1\n", file_ext="txt")
@skip_without_tool("collection_creates_list_2")
@uses_test_history(require_new=False)
@@ -1139,9 +1143,9 @@ class ToolsTestCase(ApiTestCase, TestsTools):
# test using format_source with a tool
new_dataset1 = self.dataset_populator.new_dataset(history_id, content="#col1\tcol2")
create_response = self.dataset_collection_populator.create_list_in_history(
history_id, contents=["a\tb\nc\td", "e\tf\ng\th"]
history_id, contents=["a\tb\nc\td", "e\tf\ng\th"], wait=True
)
hdca_id = create_response.json()["id"]
hdca_id = create_response.json()["outputs"][0]["id"]
inputs = {
"header": {"src": "hda", "id": new_dataset1["id"]},
"input_collect": {"src": "hdca", "id": hdca_id},
@@ -1150,7 +1154,7 @@ class ToolsTestCase(ApiTestCase, TestsTools):
self.dataset_populator.wait_for_history(history_id, assert_ok=True)
create = self._run("collection_creates_list_2", history_id, inputs, assert_ok=True)
output_collection = self._assert_one_job_one_collection_run(create)
element0, element1 = self._assert_elements_are(output_collection, "data1", "data2")
element0, element1 = self._assert_elements_are(output_collection, "data0", "data1")
self.dataset_populator.wait_for_history(history_id, assert_ok=True)
self._verify_element(history_id, element0, contents="#col1\tcol2\na\tb\nc\td\n", file_ext="txt")
self._verify_element(history_id, element1, contents="#col1\tcol2\ne\tf\ng\th\n", file_ext="txt")
@@ -1605,7 +1609,9 @@ class ToolsTestCase(ApiTestCase, TestsTools):
@skip_without_tool("cat1")
@uses_test_history(require_new=False)
def test_map_over_empty_collection(self, history_id):
hdca_id = self.dataset_collection_populator.create_list_in_history(history_id, contents=[]).json()["id"]
hdca_id = self.dataset_collection_populator.create_list_in_history(history_id, contents=[]).json()["outputs"][
0
]["id"]
inputs = {
"input1": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]},
}
@@ -1661,7 +1667,9 @@ class ToolsTestCase(ApiTestCase, TestsTools):
@skip_without_tool("output_filter_with_input")
@uses_test_history(require_new=False)
def test_map_over_with_output_filter_no_filtering(self, history_id):
hdca_id = self.dataset_collection_populator.create_list_in_history(history_id).json()["id"]
hdca_id = self.dataset_collection_populator.create_list_in_history(history_id, wait=True).json()["outputs"][0][
"id"
]
inputs = {
"input_1": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]},
"produce_out_1": "true",
@@ -1677,9 +1685,9 @@ class ToolsTestCase(ApiTestCase, TestsTools):
@skip_without_tool("output_filter_with_input_optional")
@uses_test_history(require_new=False)
def test_map_over_with_output_filter_on_optional_input(self, history_id):
hdca_id = self.dataset_collection_populator.create_list_in_history(history_id, contents=["myinputs"]).json()[
"id"
]
hdca_id = self.dataset_collection_populator.create_list_in_history(
history_id, contents=["myinputs"], wait=True
).json()["outputs"][0]["id"]
inputs = {
"input_1": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]},
}
@@ -1694,7 +1702,9 @@ class ToolsTestCase(ApiTestCase, TestsTools):
@skip_without_tool("output_filter_with_input")
@uses_test_history(require_new=False)
def test_map_over_with_output_filter_one_filtered(self, history_id):
hdca_id = self.dataset_collection_populator.create_list_in_history(history_id).json()["id"]
hdca_id = self.dataset_collection_populator.create_list_in_history(history_id, wait=True).json()["outputs"][0][
"id"
]
inputs = {
"input_1": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]},
"produce_out_1": "true",
@@ -1733,7 +1743,9 @@ class ToolsTestCase(ApiTestCase, TestsTools):
@skip_without_tool("collection_creates_dynamic_list_of_pairs")
@uses_test_history(require_new=False)
def test_map_over_with_discovered_output_collection_elements(self, history_id):
hdca_id = self.dataset_collection_populator.create_list_in_history(history_id).json()["id"]
hdca_id = self.dataset_collection_populator.create_list_in_history(history_id, wait=True).json()["outputs"][0][
"id"
]
inputs = {"input": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]}}
create = self._run("collection_creates_dynamic_list_of_pairs", history_id, inputs).json()
implicit_collections = create["implicit_collections"]
@@ -1750,8 +1762,8 @@ class ToolsTestCase(ApiTestCase, TestsTools):
bed1_contents = open(self.get_filename("1.bed")).read()
bed2_contents = open(self.get_filename("2.bed")).read()
contents = [bed1_contents, bed2_contents]
hdca = self.dataset_collection_populator.create_list_in_history(history_id, contents=contents).json()
return hdca["id"]
hdca = self.dataset_collection_populator.create_list_in_history(history_id, contents=contents, wait=True).json()
return hdca["outputs"][0]["id"]
@skip_without_tool("identifier_single")
@uses_test_history(require_new=False)
@@ -1799,7 +1811,7 @@ class ToolsTestCase(ApiTestCase, TestsTools):
@skip_without_tool("identifier_multiple")
@uses_test_history(require_new=False)
def test_list_selectable_in_multidata_input(self, history_id):
self.dataset_collection_populator.create_list_in_history(history_id, contents=["123", "456"])
self.dataset_collection_populator.create_list_in_history(history_id, contents=["123", "456"], wait=True)
build = self.dataset_populator.build_tool_state("identifier_multiple", history_id)
assert len(build["inputs"][0]["options"]["hdca"]) == 1
@@ -2244,11 +2256,11 @@ class ToolsTestCase(ApiTestCase, TestsTools):
def test_default_identifier_source_map_over(self):
with self.dataset_populator.test_history() as history_id:
input_a_hdca_id = self.dataset_collection_populator.create_list_in_history(
history_id, contents=[("A", "A content")]
).json()["id"]
history_id, contents=[("A", "A content")], wait=True
).json()["outputs"][0]["id"]
input_b_hdca_id = self.dataset_collection_populator.create_list_in_history(
history_id, contents=[("B", "B content")]
).json()["id"]
history_id, contents=[("B", "B content")], wait=True
).json()["outputs"][0]["id"]
inputs = {
"inputA": {"batch": True, "values": [dict(src="hdca", id=input_a_hdca_id)]},
"inputB": {"batch": True, "values": [dict(src="hdca", id=input_b_hdca_id)]},
@@ -2265,9 +2277,9 @@ class ToolsTestCase(ApiTestCase, TestsTools):
def test_map_over_collection_output(self):
with self.dataset_populator.test_history() as history_id:
create_response = self.dataset_collection_populator.create_list_in_history(
history_id, contents=["a\nb\nc\nd", "e\nf\ng\nh"]
history_id, contents=["a\nb\nc\nd", "e\nf\ng\nh"], wait=True
)
hdca_id = create_response.json()["id"]
hdca_id = create_response.json()["outputs"][0]["id"]
inputs = {
"input1": {"batch": True, "values": [dict(src="hdca", id=hdca_id)]},
}
@@ -2282,8 +2294,8 @@ class ToolsTestCase(ApiTestCase, TestsTools):
outer_elements = implicit_collection["elements"]
assert len(outer_elements) == 2
element0, element1 = outer_elements
assert element0["element_identifier"] == "data1"
assert element1["element_identifier"] == "data2"
assert element0["element_identifier"] == "data0"
assert element1["element_identifier"] == "data1"
pair0, pair1 = element0["object"], element1["object"]
pair00, pair01 = pair0["elements"]
@@ -2313,7 +2325,7 @@ class ToolsTestCase(ApiTestCase, TestsTools):
def test_cannot_map_over_incompatible_collections(self):
with self.dataset_populator.test_history() as history_id:
hdca1_id = self._build_pair(history_id, ["123\n", "456\n"])
hdca2_id = self.dataset_collection_populator.create_list_in_history(history_id).json()["id"]
hdca2_id = self.dataset_collection_populator.create_list_in_history(history_id).json()["outputs"][0]["id"]
inputs = {
"input1": {
"batch": True,
@@ -2333,8 +2345,8 @@ class ToolsTestCase(ApiTestCase, TestsTools):
def test_map_over_collection_structured_like(self):
with self.dataset_populator.test_history() as history_id:
hdca_id = self.dataset_collection_populator.create_list_in_history(
history_id, contents=[("A", "A"), ("B", "B")]
).json()["id"]
history_id, contents=[("A", "A"), ("B", "B")], wait=True
).json()["outputs"][0]["id"]
self.dataset_populator.wait_for_history(history_id, assert_ok=True)
inputs = {
"input": {"values": [dict(src="hdca", id=hdca_id)]},
@@ -2370,8 +2382,8 @@ class ToolsTestCase(ApiTestCase, TestsTools):
def test_map_over_collection_type_source(self):
with self.dataset_populator.test_history() as history_id:
hdca_id = self.dataset_collection_populator.create_list_in_history(
history_id, contents=[("A", "A"), ("B", "B")]
).json()["id"]
history_id, contents=[("A", "A"), ("B", "B")], wait=True
).json()["outputs"][0]["id"]
self.dataset_populator.wait_for_history(history_id, assert_ok=True)
inputs = {
"input_collect": {"values": [dict(src="hdca", id=hdca_id)]},
@@ -2385,7 +2397,9 @@ class ToolsTestCase(ApiTestCase, TestsTools):
def test_reduce_collections_legacy(self):
with self.dataset_populator.test_history() as history_id:
hdca1_id = self._build_pair(history_id, ["123\n", "456\n"])
hdca2_id = self.dataset_collection_populator.create_list_in_history(history_id).json()["id"]
hdca2_id = self.dataset_collection_populator.create_list_in_history(history_id, wait=True).json()[
"outputs"
][0]["id"]
inputs = {
"f1": f"__collection_reduce__|{hdca1_id}",
"f2": f"__collection_reduce__|{hdca2_id}",
@@ -2396,7 +2410,9 @@ class ToolsTestCase(ApiTestCase, TestsTools):
def test_reduce_collections(self):
with self.dataset_populator.test_history() as history_id:
hdca1_id = self._build_pair(history_id, ["123\n", "456\n"])
hdca2_id = self.dataset_collection_populator.create_list_in_history(history_id).json()["id"]
hdca2_id = self.dataset_collection_populator.create_list_in_history(history_id, wait=True).json()[
"outputs"
][0]["id"]
inputs = {
"f1": {"src": "hdca", "id": hdca1_id},
"f2": {"src": "hdca", "id": hdca2_id},
@@ -2407,7 +2423,9 @@ class ToolsTestCase(ApiTestCase, TestsTools):
def test_implicit_reduce_with_mapping(self):
with self.dataset_populator.test_history() as history_id:
hdca1_id = self._build_pair(history_id, ["123\n", "456\n"])
hdca2_id = self.dataset_collection_populator.create_list_of_list_in_history(history_id).json()["id"]
hdca2_id = self.dataset_collection_populator.create_list_of_list_in_history(history_id, wait=True).json()[
"id"
]
inputs = {
"f1": {"src": "hdca", "id": hdca1_id},
"f2": {
@@ -2454,6 +2472,7 @@ class ToolsTestCase(ApiTestCase, TestsTools):
"ext": "fasta",
}
],
wait=True,
)
self._assert_status_code_is(response, 200)
hdca_id = response.json()["outputs"][0]["id"]
@@ -2503,7 +2522,9 @@ class ToolsTestCase(ApiTestCase, TestsTools):
def test_reduce_multiple_lists_on_multi_data(self):
with self.dataset_populator.test_history() as history_id:
hdca1_id = self._build_pair(history_id, ["123\n", "456\n"])
hdca2_id = self.dataset_collection_populator.create_list_in_history(history_id).json()["id"]
hdca2_id = self.dataset_collection_populator.create_list_in_history(history_id, wait=True).json()[
"outputs"
][0]["id"]
inputs = {
"f1": [{"src": "hdca", "id": hdca1_id}, {"src": "hdca", "id": hdca2_id}],
"f2": [{"src": "hdca", "id": hdca1_id}],
@@ -2561,9 +2582,9 @@ class ToolsTestCase(ApiTestCase, TestsTools):
with self.dataset_populator.test_history() as history_id:
nested_list_id = self.__build_nested_list(history_id)
create_response = self.dataset_collection_populator.create_list_in_history(
history_id, contents=["xxx\n", "yyy\n"]
history_id, contents=["xxx\n", "yyy\n"], wait=True
)
list_id = create_response.json()["id"]
list_id = create_response.json()["outputs"][0]["id"]
inputs = {
"f1": {
"batch": True,
@@ -2724,6 +2745,7 @@ class ToolsTestCase(ApiTestCase, TestsTools):
"tags": ["group:type:single", "group:condition:untreated"],
},
],
wait=True,
)
self._assert_status_code_is(response, 200)
hdca_list_id = response.json()["outputs"][0]["id"]
@@ -2749,6 +2771,7 @@ class ToolsTestCase(ApiTestCase, TestsTools):
],
},
],
wait=True,
)
self._assert_status_code_is(response, 200)
hdca_list_id = response.json()["outputs"][0]["id"]
@@ -2756,7 +2779,7 @@ class ToolsTestCase(ApiTestCase, TestsTools):
def _build_pair(self, history_id, contents, run_cat=False):
create_response = self.dataset_collection_populator.create_pair_in_history(
history_id, contents=contents, direct_upload=True
history_id, contents=contents, direct_upload=True, wait=True
)
hdca_id = create_response.json()["output_collections"][0]["id"]
inputs = {
lib/galaxy_test/api/test_tools_upload.py
+8 -6
View File
@@ -299,7 +299,7 @@ class ToolsUploadTestCase(ApiTestCase):
]
payload = {
"history_id": history_id,
"targets": json.dumps(targets),
"targets": targets,
}
fetch_response = self.dataset_populator.fetch(payload)
self._assert_status_code_is(fetch_response, 200)
@@ -340,7 +340,7 @@ class ToolsUploadTestCase(ApiTestCase):
]
payload = {
"history_id": history_id,
"targets": json.dumps(targets),
"targets": targets,
}
fetch_response = self.dataset_populator.fetch(payload)
self._assert_status_code_is(fetch_response, 200)
@@ -511,7 +511,7 @@ class ToolsUploadTestCase(ApiTestCase):
]
payload = {
"history_id": history_id,
"targets": json.dumps(targets),
"targets": targets,
}
payload["__files"] = {"files_0|file_data": tar_f}
fetch_response = self.dataset_populator.fetch(payload)
@@ -882,15 +882,17 @@ class ToolsUploadTestCase(ApiTestCase):
return self.dataset_populator.get_history_dataset_content(history_id, dataset=new_dataset)
def _upload_and_get_details(self, content, **upload_kwds):
assert_ok = upload_kwds.pop("assert_ok", True)
history_id, new_dataset = self._upload(content, **upload_kwds)
assert_ok = upload_kwds.get("assert_ok", True)
return self.dataset_populator.get_history_dataset_details(history_id, dataset=new_dataset, assert_ok=assert_ok)
def _upload(self, content, api="upload1", history_id=None, **upload_kwds):
assert_ok = upload_kwds.get("assert_ok", True)
history_id = history_id or self.dataset_populator.new_history()
if api == "upload1":
new_dataset = self.dataset_populator.new_dataset(history_id, content=content, **upload_kwds)
new_dataset = self.dataset_populator.new_dataset(
history_id, content=content, fetch_data=False, **upload_kwds
)
else:
assert api == "fetch"
element = dict(src="files", **upload_kwds)
@@ -898,7 +900,7 @@ class ToolsUploadTestCase(ApiTestCase):
"destination": {"type": "hdas"},
"elements": [element],
}
targets = json.dumps([target])
targets = [target]
payload = {"history_id": history_id, "targets": targets, "__files": {"files_0|file_data": content}}
new_dataset = self.dataset_populator.fetch(payload, assert_ok=assert_ok).json()["outputs"][0]
self.dataset_populator.wait_for_history(history_id, assert_ok=assert_ok)
lib/galaxy_test/api/test_workflow_extraction.py
+8 -6
View File
@@ -151,8 +151,8 @@ class WorkflowExtractionApiTestCase(BaseWorkflowsApiTestCase):
@skip_without_tool("multi_data_param")
def test_extract_reduction_from_history(self):
hdca = self.dataset_collection_populator.create_pair_in_history(
self.history_id, contents=["1 2 3\n4 5 6", "7 8 9\n10 11 10"]
).json()
self.history_id, contents=["1 2 3\n4 5 6", "7 8 9\n10 11 10"], wait=True
).json()["outputs"][0]
hdca_id = hdca["id"]
inputs1 = {"input": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]}, "num_lines": 2}
implicit_hdca1, job_id1 = self._run_tool_get_collection_and_job_id(self.history_id, "random_lines1", inputs1)
@@ -406,8 +406,8 @@ test_data:
def __run_random_lines_mapped_over_pair(self, history_id):
hdca = self.dataset_collection_populator.create_pair_in_history(
history_id, contents=["1 2 3\n4 5 6", "7 8 9\n10 11 10"]
).json()
history_id, contents=["1 2 3\n4 5 6", "7 8 9\n10 11 10"], wait=True
).json()["outputs"][0]
hdca_id = hdca["id"]
inputs1 = {"input": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]}, "num_lines": 2}
implicit_hdca1, job_id1 = self._run_tool_get_collection_and_job_id(history_id, "random_lines1", inputs1)
@@ -474,8 +474,10 @@ test_data:
workflow_request, history_id, workflow_id = self._setup_workflow_run(workflow, history_id=history_id)
run_workflow_response = self._post(f"workflows/{workflow_id}/invocations", data=workflow_request)
self._assert_status_code_is(run_workflow_response, 200)
self.dataset_populator.wait_for_history(history_id, assert_ok=True)
invocation_response = run_workflow_response.json()
self.workflow_populator.wait_for_invocation_and_jobs(
history_id=history_id, workflow_id=workflow_id, invocation_id=invocation_response["id"]
)
return self.__cat_job_id(history_id)
def _assert_first_step_is_paired_input(self, downloaded_workflow):
lib/galaxy_test/api/test_workflows.py
+25 -23
View File
@@ -339,7 +339,7 @@ class WorkflowsApiTestCase(BaseWorkflowsApiTestCase, ChangeDatatypeTestCase):
assert not [w for w in workflow_index if w["id"] == workflow_id]
def test_index_hidden(self):
workflow_id = self.workflow_populator.simple_workflow("test_hidden")
workflow_id = self.workflow_populator.simple_workflow("test_delete")
workflow_index = self._get("workflows").json()
workflow = [w for w in workflow_index if w["id"] == workflow_id][0]
workflow["hidden"] = True
@@ -1364,9 +1364,10 @@ steps:
def test_workflow_run_output_collection_mapping(self):
workflow_id = self._upload_yaml_workflow(WORKFLOW_WITH_OUTPUT_COLLECTION_MAPPING)
with self.dataset_populator.test_history() as history_id:
hdca1 = self.dataset_collection_populator.create_list_in_history(
fetch_response = self.dataset_collection_populator.create_list_in_history(
history_id, contents=["a\nb\nc\nd\n", "e\nf\ng\nh\n"]
).json()
hdca1 = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response)
self.dataset_populator.wait_for_history(history_id, assert_ok=True)
inputs = {
"0": self._ds_entry(hdca1),
@@ -2764,9 +2765,10 @@ input1:
workflow = self.workflow_populator.load_workflow_from_resource("test_workflow_map_reduce_pause")
uploaded_workflow_id = self.workflow_populator.create_workflow(workflow)
hda1 = self.dataset_populator.new_dataset(history_id, content="reviewed\nunreviewed")
hdca1 = self.dataset_collection_populator.create_list_in_history(
fetch_response = self.dataset_collection_populator.create_list_in_history(
history_id, contents=["1\n2\n3", "4\n5\n6"]
).json()
hdca1 = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response)
index_map = {
"0": self._ds_entry(hda1),
"1": self._ds_entry(hdca1),
@@ -2844,14 +2846,16 @@ steps:
"""
)
DELETED = 0
PAUSED_1 = 3
PAUSED_2 = 5
hdca1 = self.dataset_collection_populator.create_list_in_history(
history_id, contents=[("sample1-1", "1 2 3")]
PAUSED_1 = 1
PAUSED_2 = 2
fetch_response = self.dataset_collection_populator.create_list_in_history(
history_id, contents=[("sample1-1", "1 2 3")], wait=True
).json()
self.dataset_populator.wait_for_history(history_id, assert_ok=True)
hdca1 = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response)
deleted_id = hdca1["elements"][DELETED]["object"]["id"]
r = self._delete(f"histories/{history_id}/contents/{deleted_id}?purge={purge}")
self.dataset_populator.delete_dataset(
history_id=history_id, content_id=deleted_id, purge=purge, wait_for_purge=True
)
label_map = {"input1": self._ds_entry(hdca1)}
workflow_request = dict(
history=f"hist_id={history_id}",
@@ -2865,14 +2869,12 @@ steps:
self.workflow_populator.wait_for_invocation_and_jobs(
workflow_id, history_id, invocation_id, assert_ok=False
)
# Why is this sleep needed? -John
if not purge:
time.sleep(5)
contents = self.__history_contents(history_id)
assert contents[DELETED]["deleted"]
datasets = [content for content in contents if content["history_content_type"] == "dataset"]
assert datasets[DELETED]["deleted"]
state = "error" if purge else "paused"
assert contents[PAUSED_1]["state"] == state
assert contents[PAUSED_2]["state"] == "paused"
assert datasets[PAUSED_1]["state"] == state
assert datasets[PAUSED_2]["state"] == "paused"
def test_run_with_implicit_connection(self):
with self.dataset_populator.test_history() as history_id:
@@ -3437,6 +3439,8 @@ outer_input:
hdca2 = self.dataset_collection_populator.create_list_in_history(
history_id, contents=[("sample1-2", "4 5 6"), ("sample2-2", "0 a b")]
).json()
hdca1 = self.dataset_collection_populator.wait_for_fetched_collection(hdca1)
hdca2 = self.dataset_collection_populator.wait_for_fetched_collection(hdca2)
self.dataset_populator.wait_for_history(history_id, assert_ok=True)
label_map = {"list1": self._ds_entry(hdca1), "list2": self._ds_entry(hdca2)}
workflow_request = dict(
@@ -4596,10 +4600,10 @@ steps:
workflow = self.workflow_populator.load_workflow_from_resource("test_workflow_batch")
workflow_id = self.workflow_populator.create_workflow(workflow)
with self.dataset_populator.test_history() as history_id:
hda1 = self.dataset_populator.new_dataset(history_id, content="1 2 3")
hda2 = self.dataset_populator.new_dataset(history_id, content="4 5 6")
hda3 = self.dataset_populator.new_dataset(history_id, content="7 8 9")
hda4 = self.dataset_populator.new_dataset(history_id, content="10 11 12")
hda1 = self.dataset_populator.new_dataset(history_id, content="1 2 3", wait=True)
hda2 = self.dataset_populator.new_dataset(history_id, content="4 5 6", wait=True)
hda3 = self.dataset_populator.new_dataset(history_id, content="7 8 9", wait=True)
hda4 = self.dataset_populator.new_dataset(history_id, content="10 11 12", wait=True)
parameters = {
"0": {
"input": {
@@ -5397,10 +5401,8 @@ test_data:
inputs_by="name",
inputs=json.dumps({"input1": self._ds_entry(hda1)}),
)
self.workflow_populator.invoke_workflow_and_assert_ok(
workflow_id, history_id=history_id, request=workflow_request
)
self.workflow_populator.invoke_workflow_and_wait(workflow_id, history_id=history_id, request=workflow_request)
self.assertEqual(
"Hello World Second!\nhello world 2\n",
self.dataset_populator.get_history_dataset_content(history_id, hid=4),
self.dataset_populator.get_history_dataset_content(history_id),
)
lib/galaxy_test/base/populators.py
+120 -29
View File
@@ -295,21 +295,48 @@ class BaseDatasetPopulator(BasePopulator):
Galaxy - implementations must implement _get, _post and _delete.
"""
def new_dataset(self, history_id: str, content=None, wait: bool = False, **kwds) -> dict:
def new_dataset(
self,
history_id: str,
content=None,
wait: bool = False,
fetch_data=True,
to_posix_lines=True,
auto_decompress=True,
**kwds,
) -> dict:
"""Create a new history dataset instance (HDA) and return its ID.
:returns: the HDA id of the new object
"""
run_response = self.new_dataset_request(history_id, content=content, wait=wait, **kwds)
run_response = self.new_dataset_request(
history_id,
content=content,
wait=wait,
fetch_data=fetch_data,
to_posix_lines=to_posix_lines,
auto_decompress=auto_decompress,
**kwds,
)
assert run_response.status_code == 200, f"Failed to create new dataset with response: {run_response.text}"
if fetch_data and wait:
return self.get_history_dataset_details_raw(
history_id=history_id, dataset_id=run_response.json()["outputs"][0]["id"]
).json()
return run_response.json()["outputs"][0]
def new_dataset_request(self, history_id: str, content=None, wait: bool = False, **kwds) -> requests.Response:
def new_dataset_request(
self, history_id: str, content=None, wait: bool = False, fetch_data=True, **kwds
) -> requests.Response:
"""Lower-level dataset creation that returns the upload tool response object."""
if content is None and "ftp_files" not in kwds:
content = "TestData123"
payload = self.upload_payload(history_id, content=content, **kwds)
run_response = self.tools_post(payload)
if not fetch_data:
payload = self.upload_payload(history_id, content=content, **kwds)
run_response = self.tools_post(payload)
else:
payload = self.fetch_payload(history_id, content=content, **kwds)
run_response = self.fetch(payload, wait=wait)
if wait:
self.wait_for_tool_run(history_id, run_response, assert_ok=kwds.get("assert_ok", True))
return run_response
@@ -321,7 +348,7 @@ class BaseDatasetPopulator(BasePopulator):
timeout: timeout_type = DEFAULT_TIMEOUT,
wait: Optional[bool] = None,
):
tool_response = self._post("tools/fetch", data=payload)
tool_response = self._post("tools/fetch", data=payload, json=True)
if wait is None:
wait = assert_ok
if wait:
@@ -344,7 +371,7 @@ class BaseDatasetPopulator(BasePopulator):
]
payload = {
"history_id": history_id,
"targets": json.dumps(targets),
"targets": targets,
}
fetch_response = self.fetch(payload, wait=wait)
api_asserts.assert_status_code_is(fetch_response, 200)
@@ -375,8 +402,14 @@ class BaseDatasetPopulator(BasePopulator):
return run_response
def check_run(self, run_response: requests.Response) -> dict:
run_response.raise_for_status()
run = run_response.json()
run = None
try:
run = run_response.json()
run_response.raise_for_status()
except Exception:
if run and run["err_msg"]:
raise Exception(run["err_msg"])
raise
job = run["jobs"][0]
return job
@@ -440,10 +473,26 @@ class BaseDatasetPopulator(BasePopulator):
delete_response = self._delete(f"histories/{history_id}")
delete_response.raise_for_status()
def delete_dataset(self, history_id: str, content_id: str, purge: bool = False) -> Response:
delete_response = self._delete(f"histories/{history_id}/contents/{content_id}", {"purge": purge}, json=True)
def delete_dataset(
self, history_id: str, content_id: str, purge: bool = False, wait_for_purge: bool = False
) -> Response:
dataset_url = f"histories/{history_id}/contents/{content_id}"
delete_response = self._delete(dataset_url, {"purge": purge}, json=True)
delete_response.raise_for_status()
if wait_for_purge and delete_response.status_code == 202:
return self.wait_for_purge(history_id, content_id)
return delete_response
def wait_for_purge(self, history_id, content_id):
dataset_url = f"histories/{history_id}/contents/{content_id}"
def _wait_for_purge():
dataset = self._get(dataset_url).json()
return dataset["purged"] or None
wait_on(_wait_for_purge, "dataset to become purged", timeout=2)
return self._get(dataset_url)
def create_tool_from_path(self, tool_path: str) -> Dict[str, Any]:
tool_directory = os.path.dirname(os.path.abspath(tool_path))
payload = dict(
@@ -518,6 +567,47 @@ class BaseDatasetPopulator(BasePopulator):
def copy_history(self, history_id, name="API Test Copied History", **kwds) -> Response:
return self._post("histories", data={"name": name, "history_id": history_id, **kwds})
def fetch_payload(
self,
history_id: str,
content: str,
auto_decompress: bool = False,
file_type: str = "txt",
dbkey: str = "?",
name: str = "Test_Dataset",
**kwds,
) -> dict:
__files = {}
element = {
"ext": file_type,
"dbkey": dbkey,
"name": name,
"auto_decompress": auto_decompress,
}
for arg in ["to_posix_lines", "space_to_tab"]:
val = kwds.get(arg)
if val:
element[arg] = val
target = {
"destination": {"type": "hdas"},
"elements": [element],
}
if "ftp_files" in kwds:
element["src"] = "ftp_import"
element["ftp_path"] = kwds["ftp_files"]
elif hasattr(content, "read"):
element["src"] = "files"
__files["files_0|file_data"] = content
elif content and "://" in content:
element["src"] = "url"
element["url"] = content
else:
element["src"] = "pasted"
element["paste_content"] = content
targets = [target]
payload = {"history_id": history_id, "targets": targets, "__files": __files}
return payload
def upload_payload(self, history_id: str, content: Optional[str] = None, **kwds) -> dict:
name = kwds.get("name", "Test_Dataset")
dbkey = kwds.get("dbkey", "?")
@@ -1400,8 +1490,8 @@ class BaseWorkflowPopulator(BasePopulator):
workflow_id = self.create_workflow(workflow)
if not history_id:
history_id = self.dataset_populator.new_history()
hda1 = self.dataset_populator.new_dataset(history_id, content="1 2 3")
hda2 = self.dataset_populator.new_dataset(history_id, content="4 5 6")
hda1 = self.dataset_populator.new_dataset(history_id, content="1 2 3", wait=True)
hda2 = self.dataset_populator.new_dataset(history_id, content="4 5 6", wait=True)
workflow_request = dict(
history=f"hist_id={history_id}",
)
@@ -2010,7 +2100,7 @@ class BaseDatasetCollectionPopulator:
# this function uses recursive generation of history hdcas.
collection_type = kwds.pop("collection_type", "list:list")
collection_types = collection_type.split(":")
list = self.create_list_in_history(history_id, **kwds).json()["id"]
list = self.create_list_in_history(history_id, **kwds).json()["output_collections"][0]
current_collection_type = "list"
for collection_type in collection_types[1:]:
current_collection_type = f"{current_collection_type}:{collection_type}"
@@ -2018,22 +2108,22 @@ class BaseDatasetCollectionPopulator:
history_id=history_id,
collection_type=current_collection_type,
name=current_collection_type,
collection=[list],
collection=[list["id"]],
)
list = response.json()["id"]
list = response.json()
return response
def create_pair_in_history(self, history_id, **kwds):
def create_pair_in_history(self, history_id, wait=False, **kwds):
payload = self.create_pair_payload(history_id, instance_type="history", **kwds)
return self.__create(payload)
return self.__create(payload, wait=wait)
def create_list_in_history(self, history_id, **kwds):
def create_list_in_history(self, history_id, wait=False, **kwds):
payload = self.create_list_payload(history_id, instance_type="history", **kwds)
return self.__create(payload)
return self.__create(payload, wait=wait)
def upload_collection(self, history_id, collection_type, elements, **kwds):
def upload_collection(self, history_id, collection_type, elements, wait=False, **kwds):
payload = self.__create_payload_fetch(history_id, collection_type, contents=elements, **kwds)
return self.__create(payload)
return self.__create(payload, wait=wait)
def create_list_payload(self, history_id, **kwds):
return self.__create_payload(history_id, identifiers_func=self.list_identifiers, collection_type="list", **kwds)
@@ -2089,6 +2179,7 @@ class BaseDatasetCollectionPopulator:
element_identifier = "reverse"
element["name"] = element_identifier
element["paste_content"] = dataset_contents
element["to_posix_lines"] = kwds.get("to_posix_lines", True)
elements.append(element)
name = kwds.get("name", "Test Dataset Collection")
@@ -2103,13 +2194,13 @@ class BaseDatasetCollectionPopulator:
]
payload = dict(
history_id=history_id,
targets=json.dumps(targets),
targets=targets,
)
return payload
def wait_for_fetched_collection(self, fetch_response):
self.dataset_populator.wait_for_job(fetch_response["jobs"][0]["id"], assert_ok=True)
initial_dataset_collection = fetch_response["outputs"][0]
initial_dataset_collection = fetch_response["output_collections"][0]
dataset_collection = self.dataset_populator.get_history_collection_details(
initial_dataset_collection["history_id"], hid=initial_dataset_collection["hid"]
)
@@ -2158,14 +2249,14 @@ class BaseDatasetCollectionPopulator:
element_identifiers = [hda_to_identifier(i, hda) for (i, hda) in enumerate(hdas)]
return element_identifiers
def __create(self, payload):
def __create(self, payload, wait=False):
# Create a colleciton - either from existing datasets using collection creation API
# or from direct uploads with the fetch API. Dispatch on "targets" keyword in payload
# to decide which to use.
if "targets" not in payload:
return self._create_collection(payload)
else:
return self.dataset_populator.fetch(payload)
return self.dataset_populator.fetch(payload, wait=wait)
def __datasets(self, history_id, count, contents=None):
datasets = []
@@ -2243,7 +2334,7 @@ def load_data_dict(
if input_type == "File":
content = read_test_data(element_data)
else:
content = element_data["content"]
content = element_data.pop("content")
if content is not None:
element_data["src"] = "pasted"
element_data["paste_content"] = content
@@ -2287,7 +2378,7 @@ def load_data_dict(
new_dataset_kwds["name"] = value["name"]
if "file_type" in value:
new_dataset_kwds["file_type"] = value["file_type"]
hda = dataset_populator.new_dataset(history_id, **new_dataset_kwds)
hda = dataset_populator.new_dataset(history_id, wait=True, **new_dataset_kwds)
label_map[key] = dataset_populator.ds_entry(hda)
has_uploads = True
elif input_type == "raw":
@@ -2357,7 +2448,7 @@ def wait_on_state(
return state
if skip_states is None:
skip_states = ["running", "queued", "new", "ready", "stop", "stopped", "setting_metadata"]
skip_states = ["running", "queued", "new", "ready", "stop", "stopped", "setting_metadata", "waiting"]
if ok_states is None:
ok_states = ["ok", "scheduled"]
try:
lib/galaxy_test/selenium/framework.py
+6 -28
View File
@@ -31,6 +31,7 @@ from galaxy.selenium.navigates_galaxy import (
NavigatesGalaxy,
retry_during_transitions,
)
from galaxy.tool_util.verify.interactor import prepare_request_params
from galaxy.util import (
asbool,
classproperty,
@@ -672,60 +673,37 @@ class SeleniumSessionGetPostMixin:
def _post(self, route, data=None, files=None, headers=None, admin=False, json: bool = False) -> Response:
full_url = self.selenium_context.build_url(f"api/{route}", for_selenium=False)
if data is None:
data = {}
if files is None:
files = data.get("__files", None)
if files is not None:
del data["__files"]
cookies = None
if admin:
full_url = f"{full_url}?key={self._mixin_admin_api_key}"
else:
cookies = self.selenium_context.selenium_to_requests_cookies()
request_kwd = self._prepare_request_data(
dict(cookies=cookies, headers=headers, timeout=DEFAULT_SOCKET_TIMEOUT, files=files), data, as_json=json
)
response = requests.post(full_url, **request_kwd)
request_kwd = prepare_request_params(data=data, files=files, as_json=json, headers=headers, cookies=cookies)
response = requests.post(full_url, timeout=DEFAULT_SOCKET_TIMEOUT, **request_kwd)
return response
def _delete(self, route, data=None, headers=None, admin=False, json: bool = False) -> Response:
data = data or {}
full_url = self.selenium_context.build_url(f"api/{route}", for_selenium=False)
cookies = None
if admin:
full_url = f"{full_url}?key={self._mixin_admin_api_key}"
else:
cookies = self.selenium_context.selenium_to_requests_cookies()
request_kwd = self._prepare_request_data(
dict(cookies=cookies, headers=headers, timeout=DEFAULT_SOCKET_TIMEOUT), data, as_json=json
)
response = requests.delete(full_url, **request_kwd)
request_kwd = prepare_request_params(data=data, as_json=json, headers=headers, cookies=cookies)
response = requests.delete(full_url, timeout=DEFAULT_SOCKET_TIMEOUT, **request_kwd)
return response
def _put(self, route, data=None, headers=None, admin=False, json: bool = False) -> Response:
data = data or {}
full_url = self.selenium_context.build_url(f"api/{route}", for_selenium=False)
cookies = None
if admin:
full_url = f"{full_url}?key={self._mixin_admin_api_key}"
else:
cookies = self.selenium_context.selenium_to_requests_cookies()
request_kwd = self._prepare_request_data(
dict(cookies=cookies, headers=headers, timeout=DEFAULT_SOCKET_TIMEOUT), data, as_json=json
)
request_kwd = prepare_request_params(data=data, as_json=json, headers=headers, cookies=cookies)
response = requests.put(full_url, **request_kwd)
return response
def _prepare_request_data(self, request_kwd: Dict[str, Any], data: Dict[str, Any], as_json: bool = False):
if as_json:
request_kwd["json"] = data
else:
request_kwd["data"] = data
return request_kwd
class SeleniumSessionDatasetPopulator(SeleniumSessionGetPostMixin, populators.BaseDatasetPopulator):
lib/galaxy_test/selenium/test_history_copy_elements.py
+2 -2
View File
@@ -12,8 +12,8 @@ class HistoryCopyElementsTestCase(SeleniumTestCase):
def test_copy_hdca(self):
history_id = self.current_history_id()
input_collection = self.dataset_collection_populator.create_list_in_history(
history_id, contents=["0", "1", "0", "1"]
).json()
history_id, contents=["0", "1", "0", "1"], wait=True
).json()["outputs"][0]
input_hid = input_collection["hid"]
failed_response = self.dataset_populator.run_exit_code_from_file(history_id, input_collection["id"])
lib/galaxy_test/selenium/test_history_multi_view.py
+4 -2
View File
@@ -14,7 +14,7 @@ class HistoryMultiViewTestCase(SeleniumTestCase):
input_collection = self.dataset_collection_populator.create_list_in_history(
history_id, contents=["0", "1", "0", "1"]
).json()
input_hid = input_collection["hid"]
input_hid = input_collection["outputs"][0]["hid"]
self.home()
@@ -28,7 +28,7 @@ class HistoryMultiViewTestCase(SeleniumTestCase):
@selenium_test
def test_list_list_display(self):
history_id = self.current_history_id()
method = self.dataset_collection_populator.create_list_of_list_in_history(history_id).json
method = self.dataset_collection_populator.create_list_of_list_in_history(history_id, wait=True).json
selector = self.prepare_multi_history_view(method)
first_level_element_selector = selector.descendant(".dataset-collection-element")
self.wait_for_and_click(first_level_element_selector)
@@ -150,6 +150,8 @@ class HistoryMultiViewTestCase(SeleniumTestCase):
def prepare_multi_history_view(self, collection_populator_method):
collection = collection_populator_method()
if "outputs" in collection:
collection = self.dataset_collection_populator.wait_for_fetched_collection(collection)
collection_hid = collection["hid"]
self.home()
lib/galaxy_test/selenium/test_history_panel_collections.py
+6 -4
View File
@@ -76,7 +76,7 @@ class HistoryPanelCollectionsTestCase(SeleniumTestCase):
history_id = self.current_history_id()
input_collection = self.dataset_collection_populator.create_list_in_history(
history_id, contents=["0", "1", "0", "1"]
).json()
).json()["outputs"][0]
ok_inputs = {"input1": {"src": "hdca", "id": input_collection["id"]}}
ok_response = self.dataset_populator.run_tool("collection_creates_list", ok_inputs, history_id)
@@ -285,15 +285,17 @@ class HistoryPanelCollectionsTestCase(SeleniumTestCase):
def _generate_partially_failed_collection_with_input(self):
history_id = self.current_history_id()
input_collection = self.dataset_collection_populator.create_list_in_history(
history_id, contents=["0", "1", "0", "1"]
).json()
history_id, contents=["0", "1", "0", "1"], wait=True
).json()["outputs"][0]
failed_response = self.dataset_populator.run_exit_code_from_file(history_id, input_collection["id"])
failed_collection = failed_response["implicit_collections"][0]
return input_collection, failed_collection
def _populated_paired_and_wait_for_it(self):
history_id = self.current_history_id()
input_collection = self.dataset_collection_populator.create_pair_in_history(history_id).json()
input_collection = self.dataset_collection_populator.create_pair_in_history(history_id, wait=True).json()[
"outputs"
][0]
collection_hid = input_collection["hid"]
if not self.is_beta_history():
self.home()
test/integration/test_datatype_upload.py
+5 -1
View File
@@ -72,7 +72,11 @@ def upload_datatype_helper(instance, test_data, temp_file, delete_cache_dir=Fals
else:
file_type = test_data.datatype.file_ext
dataset = instance.dataset_populator.new_dataset(
instance.history_id, content=content, wait=False, file_type=file_type
instance.history_id,
content=content,
wait=False,
file_type=file_type,
auto_decompress=True,
)
dataset = instance.dataset_populator.get_history_dataset_details(
instance.history_id, dataset=dataset, assert_ok=False
test/integration/test_kubernetes_runner.py
+1 -1
View File
@@ -143,7 +143,7 @@ def job_config(jobs_directory):
</destination>
</destinations>
<tools>
<tool id="upload1" destination="local_dest"/>
<tool id="__DATA_FETCH__" destination="local_dest"/>
<tool id="create_2" destination="k8s_destination_walltime_short"/>
<tool id="galaxy_slots_and_memory" destination="k8s_destination_no_cleanup"/>
</tools>
test/integration/test_kubernetes_staging.py
+2 -2
View File
@@ -68,7 +68,7 @@ execution:
local_environment:
runner: local
tools:
- id: upload1
- id: __DATA_FETCH__
environment: local_environment
"""
@@ -98,7 +98,7 @@ execution:
local_environment:
runner: local
tools:
- id: upload1
- id: __DATA_FETCH__
environment: local_environment
"""
test/integration/test_remote_files.py
+3 -3
View File
@@ -216,11 +216,11 @@ class RemoteFilesIntegrationTestCase(ConfiguresRemoteFilesIntegrationTestCase):
assert "test0" not in os.listdir(ftp_dir)
_write_file_fixtures(self.root, ftp_dir)
with dataset_populator.test_history() as history_id:
hdca = self.dataset_collection_populator.create_list_of_list_in_history(history_id).json()
hdca = self.dataset_collection_populator.create_list_of_list_in_history(history_id, wait=True).json()
outer_elements = hdca["elements"][0]
assert outer_elements["element_identifier"] == "test0"
for i in range(2):
assert outer_elements["object"]["elements"][i]["element_identifier"] == f"data{i + 1}"
assert outer_elements["object"]["elements"][i]["element_identifier"] == f"data{i}"
assert outer_elements["object"]["elements"][i]["object"]["file_ext"] == "txt"
incollection = {"src": "hdca", "id": hdca["id"]}
inputs = {
@@ -232,7 +232,7 @@ class RemoteFilesIntegrationTestCase(ConfiguresRemoteFilesIntegrationTestCase):
dataset_populator.wait_for_job(response["jobs"][0]["id"], assert_ok=True)
assert "test0" in os.listdir(ftp_dir)
subdir_content = os.listdir(os.path.join(ftp_dir, "test0"))
assert sorted(subdir_content) == ["data1.txt", "data2.txt", "data3.txt"]
assert sorted(subdir_content) == ["data0.txt", "data1.txt", "data2.txt"]
def _assert_index_empty(self, index):
assert len(index) == 0
test/integration/test_upload_configuration_options.py
+8 -8
View File
@@ -419,10 +419,10 @@ class SimpleFtpUploadConfigurationTestCase(BaseFtpUploadConfigurationTestCase):
assert len(ftp_files) == 1, ftp_files
assert ftp_files[0]["path"] == "test"
assert os.path.exists(ftp_path)
# set to_posix_lines to None to exercise purging - by default this file type wouldn't
# set to_posix_lines to False to exercise purging - by default this file type wouldn't
# be purged.
dataset = self.dataset_populator.new_dataset(
self.history_id, ftp_files="test", file_type="txt", to_posix_lines=None, wait=True
self.history_id, ftp_files="test", file_type="txt", to_posix_lines=False, wait=True
)
self._check_content(dataset, content)
@@ -794,7 +794,7 @@ class FetchByPathTestCase(BaseUploadContentConfigurationTestCase):
targets = [{"destination": destination, "items": items}]
payload = {
"history_id": history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
"targets": targets,
}
self.dataset_populator.fetch(payload)
dataset = self.library_populator.get_library_contents_with_path(library["id"], "/4.bed")
@@ -809,7 +809,7 @@ class FetchByPathTestCase(BaseUploadContentConfigurationTestCase):
targets = [{"destination": destination, "items": items}]
payload = {
"history_id": history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
"targets": targets,
}
self.dataset_populator.fetch(payload)
dataset = self.library_populator.get_library_contents_with_path(library["id"], "/4.bed")
@@ -830,7 +830,7 @@ class FetchByPathTestCase(BaseUploadContentConfigurationTestCase):
]
payload = {
"history_id": history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
"targets": targets,
}
self.dataset_populator.fetch(payload)
dataset = self.library_populator.get_library_contents_with_path(library["id"], "/file1")
@@ -853,7 +853,7 @@ class FetchByPathTestCase(BaseUploadContentConfigurationTestCase):
]
payload = {
"history_id": self.history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
"targets": targets,
}
fetch_response = self.dataset_populator.fetch(payload)
self._assert_status_code_is(fetch_response, 200)
@@ -882,7 +882,7 @@ class FetchByPathTestCase(BaseUploadContentConfigurationTestCase):
]
payload = {
"history_id": self.history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
"targets": targets,
}
self.dataset_populator.fetch(payload)
contents_response = self.dataset_populator._get_contents_request(self.history_id)
@@ -902,7 +902,7 @@ class FetchByPathTestCase(BaseUploadContentConfigurationTestCase):
]
payload = {
"history_id": self.history_id, # TODO: Shouldn't be needed :(
"targets": json.dumps(targets),
"targets": targets,
}
self.dataset_populator.fetch(payload)
libraries = self.library_populator.get_libraries()
test/integration/test_workflow_scheduling_options.py
+2 -1
View File
@@ -80,9 +80,10 @@ steps:
"""
)
with self.dataset_populator.test_history() as history_id:
hdca1 = self.dataset_collection_populator.create_list_in_history(
fetch_response = self.dataset_collection_populator.create_list_in_history(
history_id, contents=["a\nb\nc\nd\n", "e\nf\ng\nh\n"]
).json()
hdca1 = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response)
self.dataset_populator.wait_for_history(history_id, assert_ok=True)
inputs = {
"0": {"src": "hdca", "id": hdca1["id"]},