From 9abbac0180419fbfde018ae1cb6f74cf18789c6b Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Fri, 22 Apr 2022 18:11:22 +0200
Subject: [PATCH] Switch most tests to use ``__DATA_FETCH__`` instead of
 ``upload1``

---
 lib/galaxy/tool_util/verify/interactor.py     |  78 +++++----
 .../api/test_dataset_collections.py           |  34 ++--
 lib/galaxy_test/api/test_datasets.py          |  67 +++++---
 lib/galaxy_test/api/test_folder_contents.py   |   2 +-
 lib/galaxy_test/api/test_histories.py         |  14 +-
 lib/galaxy_test/api/test_history_contents.py  |  86 +++++-----
 lib/galaxy_test/api/test_jobs.py              |   9 +-
 lib/galaxy_test/api/test_libraries.py         |  25 ++-
 lib/galaxy_test/api/test_tools.py             | 145 ++++++++++-------
 lib/galaxy_test/api/test_tools_upload.py      |  14 +-
 .../api/test_workflow_extraction.py           |  14 +-
 lib/galaxy_test/api/test_workflows.py         |  48 +++---
 lib/galaxy_test/base/populators.py            | 149 ++++++++++++++----
 lib/galaxy_test/selenium/framework.py         |  34 +---
 .../selenium/test_history_copy_elements.py    |   4 +-
 .../selenium/test_history_multi_view.py       |   6 +-
 .../test_history_panel_collections.py         |  10 +-
 test/integration/test_datatype_upload.py      |   6 +-
 test/integration/test_kubernetes_runner.py    |   2 +-
 test/integration/test_kubernetes_staging.py   |   4 +-
 test/integration/test_remote_files.py         |   6 +-
 .../test_upload_configuration_options.py      |  16 +-
 .../test_workflow_scheduling_options.py       |   3 +-
 23 files changed, 475 insertions(+), 301 deletions(-)

diff --git a/lib/galaxy/tool_util/verify/interactor.py b/lib/galaxy/tool_util/verify/interactor.py
index b2af1441e6e..07530e82651 100644
--- a/lib/galaxy/tool_util/verify/interactor.py
+++ b/lib/galaxy/tool_util/verify/interactor.py
@@ -16,6 +16,7 @@ from typing import Optional
 import requests
 from packaging.version import parse as parse_version
 from packaging.version import Version
+from requests.cookies import RequestsCookieJar
 
 try:
     from nose.tools import nottest
@@ -829,42 +830,55 @@ class GalaxyInteractorApi:
         allow files to be specified with the json parameter - so rewrite the parameters
         to handle that if as_json is True with specified files.
         """
-        params = params or {}
-        data = data or {}
+        return prepare_request_params(
+            data=data, files=files, as_json=as_json, params=params, headers=headers, cookies=self.cookies
+        )
 
-        # handle encoded files
-        if files is None:
-            # if not explicitly passed, check __files... convention used in tool testing
-            # and API testing code
-            files = data.get("__files", None)
-            if files is not None:
-                del data["__files"]
 
-        # files doesn't really work with json, so dump the parameters
-        # and do a normal POST with request's data parameter.
-        if bool(files) and as_json:
-            as_json = False
-            new_items = {}
-            for key, val in data.items():
-                if isinstance(val, dict) or isinstance(val, list):
-                    new_items[key] = dumps(val)
-            data.update(new_items)
+def prepare_request_params(
+    data=None,
+    files=None,
+    as_json: bool = False,
+    params: Optional[dict] = None,
+    headers: Optional[dict] = None,
+    cookies: Optional[RequestsCookieJar] = None,
+):
+    params = params or {}
+    data = data or {}
 
-        kwd = {
-            "files": files,
-        }
-        if headers:
-            kwd["headers"] = headers
-        if as_json:
-            kwd["json"] = data or None
-            kwd["params"] = params
-        else:
-            data.update(params)
-            kwd["data"] = data
-        if self.cookies:
-            kwd["cookies"] = self.cookies
+    # handle encoded files
+    if files is None:
+        # if not explicitly passed, check __files... convention used in tool testing
+        # and API testing code
+        files = data.get("__files", None)
+        if files is not None:
+            del data["__files"]
 
-        return kwd
+    # files doesn't really work with json, so dump the parameters
+    # and do a normal POST with request's data parameter.
+    if bool(files) and as_json:
+        as_json = False
+        new_items = {}
+        for key, val in data.items():
+            if isinstance(val, dict) or isinstance(val, list):
+                new_items[key] = dumps(val)
+        data.update(new_items)
+
+    kwd = {
+        "files": files,
+    }
+    if headers:
+        kwd["headers"] = headers
+    if as_json:
+        kwd["json"] = data or None
+        kwd["params"] = params
+    else:
+        data.update(params)
+        kwd["data"] = data
+    if cookies:
+        kwd["cookies"] = cookies
+
+    return kwd
 
 
 def ensure_tool_run_response_okay(submit_response_object, request_desc, inputs=None):
diff --git a/lib/galaxy_test/api/test_dataset_collections.py b/lib/galaxy_test/api/test_dataset_collections.py
index 008837fcc49..dbdca4576f3 100644
--- a/lib/galaxy_test/api/test_dataset_collections.py
+++ b/lib/galaxy_test/api/test_dataset_collections.py
@@ -1,4 +1,3 @@
-import json
 import zipfile
 from io import BytesIO
 from typing import List
@@ -26,7 +25,7 @@ class DatasetCollectionApiTestCase(ApiTestCase):
             self.history_id,
             instance_type="history",
         )
-        create_response = self._post("dataset_collections", payload, json=True)
+        create_response = self.dataset_populator.fetch(payload, wait=True)
         dataset_collection = self._check_create_response(create_response)
         returned_datasets = dataset_collection["elements"]
         assert len(returned_datasets) == 2, dataset_collection
@@ -51,7 +50,7 @@ class DatasetCollectionApiTestCase(ApiTestCase):
             self.history_id,
             instance_type="history",
         )
-        pair_create_response = self._post("dataset_collections", pair_payload, json=True)
+        pair_create_response = self._post("tools/fetch", pair_payload, json=True)
         dataset_collection = self._check_create_response(pair_create_response)
         hdca_id = dataset_collection["id"]
 
@@ -153,8 +152,9 @@ class DatasetCollectionApiTestCase(ApiTestCase):
             )
 
     def test_list_list_download(self):
-        dataset_collection = self.dataset_collection_populator.create_list_of_list_in_history(self.history_id).json()
-        self.dataset_collection_populator.wait_for_dataset_collection(dataset_collection, assert_ok=True)
+        dataset_collection = self.dataset_collection_populator.create_list_of_list_in_history(
+            self.history_id, wait=True
+        ).json()
         returned_dce = dataset_collection["elements"]
         assert len(returned_dce) == 1, dataset_collection
         create_response = self._download_dataset_collection(
@@ -167,9 +167,10 @@ class DatasetCollectionApiTestCase(ApiTestCase):
 
     def test_list_list_list_download(self):
         dataset_collection = self.dataset_collection_populator.create_list_of_list_in_history(
-            self.history_id, collection_type="list:list:list"
+            self.history_id,
+            collection_type="list:list:list",
+            wait=True,
         ).json()
-        self.dataset_collection_populator.wait_for_dataset_collection(dataset_collection, assert_ok=True)
         returned_dce = dataset_collection["elements"]
         assert len(returned_dce) == 1, dataset_collection
         create_response = self._download_dataset_collection(
@@ -227,7 +228,7 @@ class DatasetCollectionApiTestCase(ApiTestCase):
         ]
         payload = {
             "history_id": self.history_id,
-            "targets": json.dumps(targets),
+            "targets": targets,
             "__files": {"files_0|file_data": open(self.test_data_resolver.get_filename("4.bed"))},
         }
         self.dataset_populator.fetch(payload)
@@ -256,7 +257,7 @@ class DatasetCollectionApiTestCase(ApiTestCase):
         ]
         payload = {
             "history_id": self.history_id,
-            "targets": json.dumps(targets),
+            "targets": targets,
             "__files": {"files_0|file_data": open(self.test_data_resolver.get_filename("4.bed"))},
         }
         self.dataset_populator.fetch(payload)
@@ -284,7 +285,7 @@ class DatasetCollectionApiTestCase(ApiTestCase):
         ]
         payload = {
             "history_id": self.history_id,
-            "targets": json.dumps(targets),
+            "targets": targets,
         }
         self.dataset_populator.fetch(payload)
         hdca = self._assert_one_collection_created_in_history()
@@ -306,7 +307,7 @@ class DatasetCollectionApiTestCase(ApiTestCase):
         ]
         payload = {
             "history_id": self.history_id,
-            "targets": json.dumps(targets),
+            "targets": targets,
         }
         self.dataset_populator.fetch(payload, assert_ok=False, wait=True)
         hdca = self._assert_one_collection_created_in_history()
@@ -328,6 +329,10 @@ class DatasetCollectionApiTestCase(ApiTestCase):
     def _check_create_response(self, create_response):
         self._assert_status_code_is(create_response, 200)
         dataset_collection = create_response.json()
+        if "output_collections" in dataset_collection:
+            # fetch data response, we'll have to check the final response
+            dataset_collection = dataset_collection["output_collections"][0]
+            dataset_collection = self._get(f"dataset_collections/{dataset_collection['id']}").json()
         self._assert_has_keys(dataset_collection, "elements", "url", "name", "collection_type", "element_count")
         return dataset_collection
 
@@ -418,6 +423,7 @@ class DatasetCollectionApiTestCase(ApiTestCase):
                     ],
                 },
             ],
+            wait=True,
         )
         self._assert_status_code_is(response, 200)
         hdca_list_id = response.json()["outputs"][0]["id"]
@@ -461,6 +467,7 @@ class DatasetCollectionApiTestCase(ApiTestCase):
                     ],
                 },
             ],
+            wait=True,
         )
         self._assert_status_code_is(response, 200)
         hdca_list_id = response.json()["outputs"][0]["id"]
@@ -491,6 +498,7 @@ class DatasetCollectionApiTestCase(ApiTestCase):
                     ],
                 },
             ],
+            wait=True,
         )
         self._assert_status_code_is(response, 200)
         hdca_list_id = response.json()["outputs"][0]["id"]
@@ -513,7 +521,7 @@ class DatasetCollectionApiTestCase(ApiTestCase):
         ]
         payload = {
             "history_id": self.history_id,
-            "targets": json.dumps(targets),
+            "targets": targets,
             "__files": {"files_0|file_data": open(self.test_data_resolver.get_filename("4.bed"))},
         }
         hdca_id = self.dataset_populator.fetch(payload).json()["output_collections"][0]["id"]
@@ -545,7 +553,7 @@ class DatasetCollectionApiTestCase(ApiTestCase):
     def _create_collection_contents_pair(self):
         # Create a simple collection, return hdca and contents_url
         payload = self.dataset_collection_populator.create_pair_payload(self.history_id, instance_type="history")
-        create_response = self._post("dataset_collections", payload, json=True)
+        create_response = self.dataset_populator.fetch(payload=payload, wait=True)
         hdca = self._check_create_response(create_response)
         root_contents_url = self._get_contents_url_for_hdca(hdca)
         return hdca, root_contents_url
diff --git a/lib/galaxy_test/api/test_datasets.py b/lib/galaxy_test/api/test_datasets.py
index 5037cb15937..81b811a6d98 100644
--- a/lib/galaxy_test/api/test_datasets.py
+++ b/lib/galaxy_test/api/test_datasets.py
@@ -63,20 +63,24 @@ class DatasetsApiTestCase(ApiTestCase):
 
     def test_search_datasets(self):
         hda_id = self.dataset_populator.new_dataset(self.history_id)["id"]
-        payload = {"limit": 1, "offset": 0}
+        payload = {"limit": 1, "offset": 0, "history_id": self.history_id}
         index_response = self._get("datasets", payload).json()
         assert len(index_response) == 1
         assert index_response[0]["id"] == hda_id
-        hdca_id = self.dataset_collection_populator.create_list_in_history(
+        fetch_response = self.dataset_collection_populator.create_list_in_history(
             self.history_id, contents=["1\n2\n3"]
-        ).json()["id"]
-        index_payload_1 = {"limit": 3, "offset": 0}
+        ).json()
+        hdca_id = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response)["id"]
+        index_payload_1 = {"limit": 3, "offset": 0, "order": "hid", "history_id": self.history_id}
         index_response = self._get("datasets", index_payload_1).json()
         assert len(index_response) == 3
-        assert index_response[0]["id"] == hdca_id
-        assert index_response[0]["history_content_type"] == "dataset_collection"
-        assert index_response[2]["id"] == hda_id
+        assert index_response[0]["hid"] == 3
+        assert index_response[1]["hid"] == 2
+        assert index_response[2]["hid"] == 1
         assert index_response[2]["history_content_type"] == "dataset"
+        assert index_response[2]["id"] == hda_id
+        assert index_response[1]["history_content_type"] == "dataset_collection"
+        assert index_response[1]["id"] == hdca_id
         index_payload_2 = {"limit": 2, "offset": 0, "q": ["history_content_type"], "qv": ["dataset"]}
         index_response = self._get("datasets", index_payload_2).json()
         assert index_response[1]["id"] == hda_id
@@ -89,7 +93,13 @@ class DatasetsApiTestCase(ApiTestCase):
         updated_hda = self._put(f"histories/{self.history_id}/contents/{hda_id}", update_payload, json=True).json()
         assert "cool:new_tag" in updated_hda["tags"]
         assert "cool:another_tag" in updated_hda["tags"]
-        payload = {"limit": 10, "offset": 0, "q": ["history_content_type", "tag"], "qv": ["dataset", "cool:new_tag"]}
+        payload = {
+            "limit": 10,
+            "offset": 0,
+            "q": ["history_content_type", "tag"],
+            "qv": ["dataset", "cool:new_tag"],
+            "history_id": self.history_id,
+        }
         index_response = self._get("datasets", payload).json()
         assert len(index_response) == 1
         payload = {
@@ -97,36 +107,55 @@ class DatasetsApiTestCase(ApiTestCase):
             "offset": 0,
             "q": ["history_content_type", "tag-contains"],
             "qv": ["dataset", "new_tag"],
+            "history_id": self.history_id,
         }
         index_response = self._get("datasets", payload).json()
         assert len(index_response) == 1
-        payload = {"limit": 10, "offset": 0, "q": ["history_content_type", "tag-contains"], "qv": ["dataset", "notag"]}
+        payload = {
+            "limit": 10,
+            "offset": 0,
+            "q": ["history_content_type", "tag-contains"],
+            "qv": ["dataset", "notag"],
+            "history_id": self.history_id,
+        }
         index_response = self._get("datasets", payload).json()
         assert len(index_response) == 0
 
     def test_search_by_tool_id(self):
         self.dataset_populator.new_dataset(self.history_id)
-        payload = {"limit": 1, "offset": 0, "q": ["history_content_type", "tool_id"], "qv": ["dataset", "upload1"]}
+        payload = {
+            "limit": 1,
+            "offset": 0,
+            "q": ["history_content_type", "tool_id"],
+            "qv": ["dataset", "__DATA_FETCH__"],
+            "history_id": self.history_id,
+        }
         assert len(self._get("datasets", payload).json()) == 1
-        payload = {"limit": 1, "offset": 0, "q": ["history_content_type", "tool_id"], "qv": ["dataset", "uploadX"]}
+        payload = {
+            "limit": 1,
+            "offset": 0,
+            "q": ["history_content_type", "tool_id"],
+            "qv": ["dataset", "__DATA_FETCH__X"],
+            "history_id": self.history_id,
+        }
         assert len(self._get("datasets", payload).json()) == 0
         payload = {
             "limit": 1,
             "offset": 0,
             "q": ["history_content_type", "tool_id-contains"],
-            "qv": ["dataset", "pload1"],
+            "qv": ["dataset", "ATA_FETCH"],
+            "history_id": self.history_id,
         }
         assert len(self._get("datasets", payload).json()) == 1
         self.dataset_collection_populator.create_list_in_history(
-            self.history_id, name="search by tool id", contents=["1\n2\n3"]
-        ).json()
-        self.dataset_populator.wait_for_history(self.history_id)
+            self.history_id, name="search by tool id", contents=["1\n2\n3"], wait=True
+        )
         payload = {
             "limit": 10,
             "offset": 0,
-            "history_id": self.history_id,
             "q": ["name", "tool_id"],
-            "qv": ["search by tool id", "upload1"],
+            "qv": ["search by tool id", "__DATA_FETCH__"],
+            "history_id": self.history_id,
         }
         result = self._get("datasets", payload).json()
         assert result[0]["name"] == "search by tool id", result
@@ -135,6 +164,7 @@ class DatasetsApiTestCase(ApiTestCase):
             "offset": 0,
             "q": ["history_content_type", "tool_id"],
             "qv": ["dataset_collection", "uploadX"],
+            "history_id": self.history_id,
         }
         result = self._get("datasets", payload).json()
         assert len(result) == 0
@@ -289,8 +319,7 @@ class DatasetsApiTestCase(ApiTestCase):
         assert deleted_result["success_count"] == len(expected_purged_source_ids)
 
         for purged_source_id in expected_purged_source_ids:
-            dataset = self._get(f"histories/{history_id}/contents/{purged_source_id['id']}").json()
-            assert dataset["purged"] is True
+            self.dataset_populator.wait_for_purge(history_id, purged_source_id["id"])
 
     def test_delete_batch_error(self):
         num_datasets = 4
diff --git a/lib/galaxy_test/api/test_folder_contents.py b/lib/galaxy_test/api/test_folder_contents.py
index 272280d8ed0..e4401ea6e94 100644
--- a/lib/galaxy_test/api/test_folder_contents.py
+++ b/lib/galaxy_test/api/test_folder_contents.py
@@ -234,7 +234,7 @@ class FolderContentsApiTestCase(ApiTestCase):
 
     def _create_hdca_with_contents(self, contents: List[str]) -> str:
         hdca = self.dataset_collection_populator.create_list_in_history(
-            self.history_id, contents=contents, direct_upload=True
+            self.history_id, contents=contents, direct_upload=True, wait=True
         ).json()["outputs"][0]
         hdca_id = hdca["id"]
         return hdca_id
diff --git a/lib/galaxy_test/api/test_histories.py b/lib/galaxy_test/api/test_histories.py
index f26b418b0a2..8eaf9ae4bb6 100644
--- a/lib/galaxy_test/api/test_histories.py
+++ b/lib/galaxy_test/api/test_histories.py
@@ -279,7 +279,7 @@ class ImportExportTests(BaseHistories):
         imported_history_id = self._reimport_history(history_id, history_name, wait_on_history_length=2)
 
         def upload_job_check(job):
-            assert job["tool_id"] == "upload1"
+            assert job["tool_id"] == "__DATA_FETCH__"
 
         def check_discarded(hda):
             assert hda["deleted"]
@@ -319,7 +319,7 @@ class ImportExportTests(BaseHistories):
         self._assert_history_length(imported_history_id, 2)
 
         def upload_job_check(job):
-            assert job["tool_id"] == "upload1"
+            assert job["tool_id"] == "__DATA_FETCH__"
 
         def check_deleted_not_purged(hda):
             assert hda["state"] == "ok", hda
@@ -392,7 +392,7 @@ class ImportExportTests(BaseHistories):
         history_name = f"for_export_with_collections_{uuid4()}"
         history_id = self.dataset_populator.new_history(name=history_name)
         self.dataset_collection_populator.create_list_in_history(
-            history_id, contents=["Hello", "World"], direct_upload=True
+            history_id, contents=["Hello", "World"], direct_upload=True, wait=True
         )
 
         imported_history_id = self._reimport_history(history_id, history_name, wait_on_history_length=3)
@@ -417,7 +417,8 @@ class ImportExportTests(BaseHistories):
     def test_import_export_nested_collection(self):
         history_name = f"for_export_with_nested_collections_{uuid4()}"
         history_id = self.dataset_populator.new_history(name=history_name)
-        self.dataset_collection_populator.create_list_of_pairs_in_history(history_id)
+        fetch_response = self.dataset_collection_populator.create_list_of_pairs_in_history(history_id, wait=True).json()
+        dataset_collection = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response)
 
         imported_history_id = self._reimport_history(history_id, history_name, wait_on_history_length=3)
         self._assert_history_length(imported_history_id, 3)
@@ -433,7 +434,10 @@ class ImportExportTests(BaseHistories):
             assert element0["collection_type"] == "paired"
 
         self._check_imported_collection(
-            imported_history_id, hid=1, collection_type="list:paired", elements_checker=check_elements
+            imported_history_id,
+            hid=dataset_collection["hid"],
+            collection_type="list:paired",
+            elements_checker=check_elements,
         )
 
     def _reimport_history(
diff --git a/lib/galaxy_test/api/test_history_contents.py b/lib/galaxy_test/api/test_history_contents.py
index 511aa61acfa..33ce8e72340 100644
--- a/lib/galaxy_test/api/test_history_contents.py
+++ b/lib/galaxy_test/api/test_history_contents.py
@@ -352,19 +352,24 @@ class HistoryContentsApiTestCase(ApiTestCase):
         data = {"purge": True}
         delete_response = self._delete(f"histories/{self.history_id}/contents/{hda1['id']}", data=data, json=True)
         assert delete_response.status_code < 300  # Something in the 200s :).
-        assert str(self.__show(hda1).json()["deleted"]).lower() == "true"
-        assert str(self.__show(hda1).json()["purged"]).lower() == "true"
+        # Purging and deleting the dataset may or may not happen asynchronously.
+        # On 202 the request was accepted and purging will happen later.
+        if delete_response.status_code == 202:
+            self.dataset_populator.wait_for_purge(self.history_id, hda1["id"])
+        else:
+            assert self.__show(hda1).json()["deleted"]
+            assert self.__show(hda1).json()["purged"]
 
     def test_dataset_collection_creation_on_contents(self):
-        payload = self.dataset_collection_populator.create_pair_payload(self.history_id, type="dataset_collection")
-        endpoint = f"histories/{self.history_id}/contents"
+        payload = self.dataset_collection_populator.create_pair_payload(
+            self.history_id, type="dataset_collection", wait=True
+        )
+        endpoint = "tools/fetch"
         self._check_pair_creation(endpoint, payload)
 
     def test_dataset_collection_creation_on_typed_contents(self):
-        payload = self.dataset_collection_populator.create_pair_payload(
-            self.history_id,
-        )
-        endpoint = f"histories/{self.history_id}/contents/dataset_collections"
+        payload = self.dataset_collection_populator.create_pair_payload(self.history_id, wait=True)
+        endpoint = "tools/fetch"
         self._check_pair_creation(endpoint, payload)
 
     def test_dataset_collection_create_from_exisiting_datasets_with_new_tags(self):
@@ -429,10 +434,10 @@ class HistoryContentsApiTestCase(ApiTestCase):
 
     @skip_without_tool("collection_creates_list")
     def test_jobs_summary_simple_hdca(self):
-        create_response = self.dataset_collection_populator.create_list_in_history(
+        fetch_response = self.dataset_collection_populator.create_list_in_history(
             self.history_id, contents=["a\nb\nc\nd", "e\nf\ng\nh"]
-        )
-        hdca_id = create_response.json()["id"]
+        ).json()
+        hdca_id = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response)["id"]
         run = self.dataset_populator.run_collection_creates_list(self.history_id, hdca_id)
         collections = run["output_collections"]
         collection = collections[0]
@@ -445,9 +450,9 @@ class HistoryContentsApiTestCase(ApiTestCase):
     @skip_without_tool("cat1")
     def test_jobs_summary_implicit_hdca(self):
         create_response = self.dataset_collection_populator.create_pair_in_history(
-            self.history_id, contents=["123", "456"]
+            self.history_id, contents=["123", "456"], wait=True
         )
-        hdca_id = create_response.json()["id"]
+        hdca_id = create_response.json()["outputs"][0]["id"]
         inputs = {
             "input1": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]},
         }
@@ -464,7 +469,9 @@ class HistoryContentsApiTestCase(ApiTestCase):
         assert states.get("ok") == 2, states
 
     def test_dataset_collection_hide_originals(self):
-        payload = self.dataset_collection_populator.create_pair_payload(self.history_id, type="dataset_collection")
+        payload = self.dataset_collection_populator.create_pair_payload(
+            self.history_id, type="dataset_collection", direct_upload=False
+        )
 
         payload["hide_source_items"] = True
         dataset_collection_response = self._post(f"histories/{self.history_id}/contents", payload, json=True)
@@ -499,14 +506,14 @@ class HistoryContentsApiTestCase(ApiTestCase):
 
     def _create_pair_collection(self):
         payload = self.dataset_collection_populator.create_pair_payload(self.history_id, type="dataset_collection")
-        dataset_collection_response = self._post(f"histories/{self.history_id}/contents", payload, json=True)
+        dataset_collection_response = self._post("tools/fetch", payload, json=True)
         self._assert_status_code_is(dataset_collection_response, 200)
-        hdca = dataset_collection_response.json()
+        hdca = dataset_collection_response.json()["output_collections"][0]
         return hdca
 
     def test_hdca_copy(self):
-        hdca = self.dataset_collection_populator.create_pair_in_history(self.history_id).json()
-        hdca_id = hdca["id"]
+        hdca = self.dataset_collection_populator.create_pair_in_history(self.history_id, wait=True).json()
+        hdca_id = hdca["outputs"][0]["id"]
         second_history_id = self.dataset_populator.new_history()
         create_data = dict(
             source="hdca",
@@ -524,7 +531,8 @@ class HistoryContentsApiTestCase(ApiTestCase):
         assert new_forward["history_id"] == self.history_id
 
     def test_hdca_copy_with_new_dbkey(self):
-        hdca = self.dataset_collection_populator.create_pair_in_history(self.history_id).json()
+        fetch_response = self.dataset_collection_populator.create_pair_in_history(self.history_id, wait=True).json()
+        hdca = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response)
         hdca_id = hdca["id"]
         assert hdca["elements"][0]["object"]["metadata_dbkey"] == "?"
         assert hdca["elements"][0]["object"]["genome_build"] == "?"
@@ -538,7 +546,7 @@ class HistoryContentsApiTestCase(ApiTestCase):
         assert new_forward["genome_build"] == "hg19"
 
     def test_hdca_copy_and_elements(self):
-        hdca = self.dataset_collection_populator.create_pair_in_history(self.history_id).json()
+        hdca = self.dataset_collection_populator.create_pair_in_history(self.history_id, wait=True).json()["outputs"][0]
         hdca_id = hdca["id"]
         second_history_id = self.dataset_populator.new_history()
         create_data = dict(
@@ -615,6 +623,8 @@ class HistoryContentsApiTestCase(ApiTestCase):
     def __check_create_collection_response(self, response):
         self._assert_status_code_is(response, 200)
         dataset_collection = response.json()
+        if "output_collections" in dataset_collection:
+            dataset_collection = dataset_collection["output_collections"][0]
         self._assert_has_keys(dataset_collection, "url", "name", "deleted", "visible", "elements")
         return dataset_collection
 
@@ -648,7 +658,8 @@ class HistoryContentsApiTestCase(ApiTestCase):
 
     def test_job_state_summary_field(self):
         create_response = self.dataset_collection_populator.create_pair_in_history(
-            self.history_id, contents=["123", "456"]
+            self.history_id,
+            contents=["123", "456"],
         )
         self._assert_status_code_is(create_response, 200)
         contents_response = self._get(f"histories/{self.history_id}/contents?v=dev&keys=job_state_summary&view=summary")
@@ -667,7 +678,7 @@ class HistoryContentsApiTestCase(ApiTestCase):
         with self.dataset_populator.test_history() as history_id:
             first_time = datetime.utcnow().isoformat()
             assert len(self._get_content(history_id, update_time=first_time)) == 0
-            self.dataset_collection_populator.create_list_in_history(history_id=history_id)
+            self.dataset_collection_populator.create_list_in_history(history_id=history_id, wait=True)
             assert len(self._get_content(history_id, update_time=first_time)) == 4  # 3 datasets
             self.dataset_populator.wait_for_history(history_id)
             all_datasets_finished = first_time = datetime.utcnow().isoformat()
@@ -690,7 +701,7 @@ class HistoryContentsApiTestCase(ApiTestCase):
             assert history_contents.status_code == 204
 
             # add some stuff
-            self.dataset_collection_populator.create_list_in_history(history_id=history_id)
+            self.dataset_collection_populator.create_list_in_history(history_id=history_id, wait=True)
             self.dataset_populator.wait_for_history(history_id)
 
             # check to make sure the added stuff is there
@@ -739,7 +750,8 @@ class HistoryContentsApiTestCase(ApiTestCase):
     @skip_without_tool("cat_data_and_sleep")
     def test_history_contents_near_with_update_time_implicit_collection(self):
         with self.dataset_populator.test_history() as history_id:
-            hdca_id = self.dataset_collection_populator.create_list_in_history(history_id=history_id).json()["id"]
+            fetch_response = self.dataset_collection_populator.create_list_in_history(history_id=history_id).json()
+            hdca_id = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response)["id"]
             self.dataset_populator.wait_for_history(history_id)
             inputs = {
                 "input1": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]},
@@ -792,7 +804,7 @@ class HistoryContentsApiTestCase(ApiTestCase):
     def test_index_filter_by_type(self):
         history_id = self.dataset_populator.new_history()
         self.dataset_populator.new_dataset(history_id)
-        self.dataset_collection_populator.create_list_in_history(history_id=history_id)
+        self.dataset_collection_populator.create_list_in_history(history_id=history_id, wait=True)
 
         contents_response = self._get(f"histories/{history_id}/contents").json()
         num_items = len(contents_response)
@@ -830,7 +842,7 @@ class HistoryContentsApiTestCase(ApiTestCase):
 
     def _upload_collection_list_with_elements(self, history_id: str, collection_name: str, elements: List[Any]):
         create_homogeneous_response = self.dataset_collection_populator.upload_collection(
-            history_id, "list", elements=elements, name=collection_name
+            history_id, "list", elements=elements, name=collection_name, wait=True
         )
         self._assert_status_code_is_ok(create_homogeneous_response)
 
@@ -860,7 +872,7 @@ class HistoryContentsApiNearTestCase(ApiTestCase):
     def _create_list_in_history(self, history_id, n=2):
         # Creates list of size n*4 (n collections with 3 items each)
         for _ in range(n):
-            self.dataset_collection_populator.create_list_in_history(history_id=history_id)
+            self.dataset_collection_populator.create_list_in_history(history_id=history_id, wait=True)
 
     def _get_content(self, history_id, direction, *, hid, limit=1000):
         return self._get(f"/api/histories/{history_id}/contents/{direction}/{hid}/{limit}").json()
@@ -971,10 +983,6 @@ class HistoryContentsApiBulkOperationTestCase(ApiTestCase):
         with self.dataset_populator.test_history() as history_id:
             datasets_ids, collection_ids, history_contents = self._create_test_history_contents(history_id)
 
-            # All items are visible
-            for item in history_contents:
-                assert item["visible"]
-
             # Hide 2 collections and 3 datasets, 5 in total
             payload = {
                 "operation": "hide",
@@ -1016,10 +1024,6 @@ class HistoryContentsApiBulkOperationTestCase(ApiTestCase):
         with self.dataset_populator.test_history() as history_id:
             _, collection_ids, history_contents = self._create_test_history_contents(history_id)
 
-            # All items are visible
-            for item in history_contents:
-                assert item["visible"]
-
             # Hide all collections using query
             payload = {"operation": "hide"}
             query = "q=history_content_type-eq&qv=dataset_collection"
@@ -1167,12 +1171,20 @@ class HistoryContentsApiBulkOperationTestCase(ApiTestCase):
         datasets_ids = list(map(lambda dataset: dataset["id"], datasets))
         assert len(history_contents) == num_expected_datasets + num_expected_collections
         assert len(datasets_ids) == num_expected_datasets
+        for dataset_id in datasets_ids:
+            self._put(f"histories/{history_id}/contents/{dataset_id}", {"visible": True}, json=True).json()
+        # All items are visible
+        history_contents = self._get_history_contents(history_id)
+        for item in history_contents:
+            assert item["visible"]
         return datasets_ids, collection_ids, history_contents
 
     def _create_collection_in_history(self, history_id, num_collections=1) -> List[str]:
         collection_ids = []
         for _ in range(num_collections):
-            collection_id = self.dataset_collection_populator.create_pair_in_history(history_id=history_id).json()["id"]
+            collection_id = self.dataset_collection_populator.create_pair_in_history(
+                history_id=history_id, wait=True
+            ).json()["outputs"][0]["id"]
             collection_ids.append(collection_id)
         return collection_ids
 
@@ -1180,7 +1192,7 @@ class HistoryContentsApiBulkOperationTestCase(ApiTestCase):
         return self._get(f"histories/{history_id}/contents").json()
 
     def _get_hidden_items_from_history_contents(self, history_contents) -> List[Any]:
-        return list(filter(lambda item: item["visible"] is False, history_contents))
+        return [content for content in history_contents if not content["visible"]]
 
     def _apply_bulk_operation(self, history_id: str, payload, query: str = ""):
         if query:
diff --git a/lib/galaxy_test/api/test_jobs.py b/lib/galaxy_test/api/test_jobs.py
index 56919cad75f..bf8299f21a6 100644
--- a/lib/galaxy_test/api/test_jobs.py
+++ b/lib/galaxy_test/api/test_jobs.py
@@ -203,7 +203,7 @@ steps:
         # Initial number of ok jobs
         original_count = len(self.__uploads_with_state("ok", "new"))
 
-        # Run through dataset upload to ensure num uplaods at least greater
+        # Run through dataset upload to ensure num uploads at least greater
         # by 1.
         self.__history_with_ok_dataset(history_id)
 
@@ -280,9 +280,10 @@ steps:
             assert dataset["visible"]
 
     def _run_map_over_error(self, history_id):
-        hdca1 = self.dataset_collection_populator.create_list_in_history(
+        fetch_response = self.dataset_collection_populator.create_list_in_history(
             history_id, contents=[("sample1-1", "1 2 3")]
         ).json()
+        hdca1 = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response)
         inputs = {
             "error_bool": "true",
             "dataset": {
@@ -874,10 +875,10 @@ steps:
         self._assert_status_code_is(jobs_response, 200)
         jobs = jobs_response.json()
         assert not [j for j in jobs if not j["state"] in states]
-        return [j for j in jobs if j["tool_id"] == "upload1"]
+        return [j for j in jobs if j["tool_id"] == "__DATA_FETCH__"]
 
     def __history_with_new_dataset(self, history_id):
-        dataset_id = self.dataset_populator.new_dataset(history_id)["id"]
+        dataset_id = self.dataset_populator.new_dataset(history_id, wait=True)["id"]
         return dataset_id
 
     def __history_with_ok_dataset(self, history_id):
diff --git a/lib/galaxy_test/api/test_libraries.py b/lib/galaxy_test/api/test_libraries.py
index 81bdd541abf..6e8abac32f6 100644
--- a/lib/galaxy_test/api/test_libraries.py
+++ b/lib/galaxy_test/api/test_libraries.py
@@ -1,4 +1,3 @@
-import json
 import unittest
 
 import pytest
@@ -222,10 +221,10 @@ class LibrariesApiTestCase(ApiTestCase):
         targets = [{"destination": destination, "items": items}]
         payload = {
             "history_id": history_id,  # TODO: Shouldn't be needed :(
-            "targets": json.dumps(targets),
+            "targets": targets,
             "__files": {"files_0|file_data": open(self.test_data_resolver.get_filename("4.bed"))},
         }
-        self.dataset_populator.fetch(payload)
+        self.dataset_populator.fetch(payload, wait=True)
         dataset = self.library_populator.get_library_contents_with_path(library["id"], "/4.bed")
         assert dataset["file_size"] == 61, dataset
         assert dataset["genome_build"] == "hg19", dataset
@@ -245,7 +244,7 @@ class LibrariesApiTestCase(ApiTestCase):
         ]
         payload = {
             "history_id": history_id,  # TODO: Shouldn't be needed :(
-            "targets": json.dumps(targets),
+            "targets": targets,
             "__files": {"files_0|file_data": open(bed_test_data_path, "rb")},
         }
         self.dataset_populator.fetch(payload)
@@ -280,7 +279,7 @@ class LibrariesApiTestCase(ApiTestCase):
         ]
         payload = {
             "history_id": history_id,  # TODO: Shouldn't be needed :(
-            "targets": json.dumps(targets),
+            "targets": targets,
             "validate_hashes": True,
         }
         return library, self.dataset_populator.fetch(payload, assert_ok=assert_ok)
@@ -322,7 +321,7 @@ class LibrariesApiTestCase(ApiTestCase):
         ]
         payload = {
             "history_id": history_id,  # TODO: Shouldn't be needed :(
-            "targets": json.dumps(targets),
+            "targets": targets,
             "validate_hashes": True,
         }
         tool_response = self.dataset_populator.fetch(payload, assert_ok=False)
@@ -349,7 +348,7 @@ class LibrariesApiTestCase(ApiTestCase):
         ]
         payload = {
             "history_id": history_id,  # TODO: Shouldn't be needed :(
-            "targets": json.dumps(targets),
+            "targets": targets,
         }
         self.dataset_populator.fetch(payload)
         dataset = self.library_populator.get_library_contents_with_path(library["id"], "/4.bed")
@@ -368,7 +367,7 @@ class LibrariesApiTestCase(ApiTestCase):
         ]
         payload = {
             "history_id": history_id,  # TODO: Shouldn't be needed :(
-            "targets": json.dumps(targets),
+            "targets": targets,
             "__files": {"files_0|file_data": open(example_bag_path)},
         }
         self.dataset_populator.fetch(payload)
@@ -520,7 +519,7 @@ class LibrariesApiTestCase(ApiTestCase):
         folder_id = folder_response.json()[0]["id"]
         history_id = self.dataset_populator.new_history()
         hdca_id = self.dataset_collection_populator.create_list_in_history(
-            history_id, contents=["xxx", "yyy"], direct_upload=True
+            history_id, contents=["xxx", "yyy"], direct_upload=True, wait=True
         ).json()["outputs"][0]["id"]
         payload = {"from_hdca_id": hdca_id, "create_type": "file", "folder_id": folder_id}
         create_response = self._post(f"libraries/{library['id']}/contents", payload)
@@ -530,7 +529,7 @@ class LibrariesApiTestCase(ApiTestCase):
         library = self.library_populator.new_private_library("ForCreateDatasetsFromCollection")
         history_id = self.dataset_populator.new_history()
         hdca_id = self.dataset_collection_populator.create_list_in_history(
-            history_id, contents=["xxx", "yyy"], direct_upload=True
+            history_id, contents=["xxx", "yyy"], direct_upload=True, wait=True
         ).json()["outputs"][0]["id"]
         folder_response = self._create_folder(library)
         self._assert_status_code_is(folder_response, 200)
@@ -541,9 +540,9 @@ class LibrariesApiTestCase(ApiTestCase):
         assert len(create_response.json()) == 2
         # Also test that anything different from a flat dataset collection list
         # is refused
-        hdca_pair_id = self.dataset_collection_populator.create_list_of_pairs_in_history(history_id).json()["outputs"][
-            0
-        ]["id"]
+        hdca_pair_id = self.dataset_collection_populator.create_list_of_pairs_in_history(history_id, wait=True).json()[
+            "outputs"
+        ][0]["id"]
         payload = {"from_hdca_id": hdca_pair_id}
         create_response = self._post(f"folders/{folder_id}/contents", payload)
         self._assert_status_code_is(create_response, 501)
diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py
index 14c5f1e2899..54581b11b49 100644
--- a/lib/galaxy_test/api/test_tools.py
+++ b/lib/galaxy_test/api/test_tools.py
@@ -59,7 +59,10 @@ class TestsTools:
 
     def _build_pair(self, history_id, contents):
         create_response = self.dataset_collection_populator.create_pair_in_history(
-            history_id, contents=contents, direct_upload=True
+            history_id,
+            contents=contents,
+            direct_upload=True,
+            wait=True,
         )
         hdca_id = create_response.json()["outputs"][0]["id"]
         return hdca_id
@@ -518,7 +521,7 @@ class ToolsTestCase(ApiTestCase, TestsTools):
                 ],
             },
         ]
-        hdca1 = self.dataset_collection_populator.upload_collection(history_id, "list:paired", elements=data)
+        hdca1 = self.dataset_collection_populator.upload_collection(history_id, "list:paired", elements=data, wait=True)
         self._assert_status_code_is(hdca1, 200)
 
         payload = {
@@ -529,14 +532,12 @@ class ToolsTestCase(ApiTestCase, TestsTools):
             "history_id": history_id,
         }
         create_response = self._post("tools/CONVERTER_fasta_to_tabular/convert", payload)
-
-        self.dataset_populator.wait_for_job(create_response.json()["jobs"][0]["id"], assert_ok=True)
         create_response.raise_for_status()
-
         assert create_response.json()["implicit_collections"] != []
-        hdca_id = create_response.json()["implicit_collections"][0]["hid"]
-        fetchedResponse = self.dataset_populator.get_history_collection_details(history_id, hid=hdca_id)
-        for element in fetchedResponse["elements"][0]["object"]["elements"]:
+        hid = create_response.json()["implicit_collections"][0]["hid"]
+        self.dataset_populator.wait_for_history(history_id, assert_ok=True)
+        collection_details = self.dataset_populator.get_history_collection_details(history_id, hid=hid)
+        for element in collection_details["elements"][0]["object"]["elements"]:
             assert element["object"]["file_ext"] == "tabular"
 
     def test_unzip_collection(self):
@@ -592,6 +593,7 @@ class ToolsTestCase(ApiTestCase, TestsTools):
                         ],
                     }
                 ],
+                wait=True,
             )
             self._assert_status_code_is(response, 200)
             hdca_id = response.json()["outputs"][0]["id"]
@@ -601,7 +603,6 @@ class ToolsTestCase(ApiTestCase, TestsTools):
                     "values": [{"src": "hdca", "map_over_type": "paired", "id": hdca_id}],
                 }
             }
-            self.dataset_populator.wait_for_history(history_id, assert_ok=True)
             response = self._run("__UNZIP_COLLECTION__", history_id, inputs, assert_ok=True)
             implicit_collections = response["implicit_collections"]
             self.assertEqual(len(implicit_collections), 2)
@@ -645,7 +646,9 @@ class ToolsTestCase(ApiTestCase, TestsTools):
     @skip_without_tool("__UNZIP_COLLECTION__")
     @uses_test_history(require_new=False)
     def test_collection_operation_collection_input_permissions(self, history_id):
-        create_response = self.dataset_collection_populator.create_pair_in_history(history_id, direct_upload=True)
+        create_response = self.dataset_collection_populator.create_pair_in_history(
+            history_id, direct_upload=True, wait=True
+        )
         self._assert_status_code_is(create_response, 200)
         collection = create_response.json()["outputs"][0]
         self.dataset_populator.wait_for_history(history_id, assert_ok=True)
@@ -662,11 +665,12 @@ class ToolsTestCase(ApiTestCase, TestsTools):
     def test_zip_list_inputs(self):
         with self.dataset_populator.test_history() as history_id:
             hdca1_id = self.dataset_collection_populator.create_list_in_history(
-                history_id, contents=["a\nb\nc\nd", "e\nf\ng\nh"]
-            ).json()["id"]
+                history_id, contents=["a\nb\nc\nd", "e\nf\ng\nh"], wait=True
+            ).json()["outputs"][0]["id"]
+
             hdca2_id = self.dataset_collection_populator.create_list_in_history(
-                history_id, contents=["1\n2\n3\n4", "5\n6\n7\n8"]
-            ).json()["id"]
+                history_id, contents=["1\n2\n3\n4", "5\n6\n7\n8"], wait=True
+            ).json()["outputs"][0]["id"]
             inputs = {
                 "input_forward": {"batch": True, "values": [{"src": "hdca", "id": hdca1_id}]},
                 "input_reverse": {"batch": True, "values": [{"src": "hdca", "id": hdca2_id}]},
@@ -685,8 +689,8 @@ class ToolsTestCase(ApiTestCase, TestsTools):
     def test_filter_failed_list(self):
         with self.dataset_populator.test_history() as history_id:
             ok_hdca_id = self.dataset_collection_populator.create_list_in_history(
-                history_id, contents=["0", "1", "0", "1"]
-            ).json()["id"]
+                history_id, contents=["0", "1", "0", "1"], wait=True
+            ).json()["outputs"][0]["id"]
             response = self.dataset_populator.run_exit_code_from_file(history_id, ok_hdca_id)
 
             mixed_implicit_collections = response["implicit_collections"]
@@ -709,12 +713,12 @@ class ToolsTestCase(ApiTestCase, TestsTools):
     @skip_without_tool("__FILTER_FAILED_DATASETS__")
     def test_filter_failed_list_paired(self):
         with self.dataset_populator.test_history() as history_id:
-            pair1 = self.dataset_collection_populator.create_pair_in_history(history_id, contents=["0", "0"]).json()[
-                "id"
-            ]
-            pair2 = self.dataset_collection_populator.create_pair_in_history(history_id, contents=["0", "1"]).json()[
-                "id"
-            ]
+            pair1 = self.dataset_collection_populator.create_pair_in_history(
+                history_id, contents=["0", "0"], wait=True
+            ).json()["outputs"][0]["id"]
+            pair2 = self.dataset_collection_populator.create_pair_in_history(
+                history_id, contents=["0", "1"], wait=True
+            ).json()["outputs"][0]["id"]
             ok_hdca_id = self.dataset_collection_populator.create_list_from_pairs(history_id, [pair1, pair2]).json()[
                 "id"
             ]
@@ -1003,7 +1007,7 @@ class ToolsTestCase(ApiTestCase, TestsTools):
             return element0["id"]
 
         response = self.dataset_collection_populator.create_list_in_history(
-            history_id, contents=["a\nb\nc\nd", "e\nf\ng\nh"], direct_upload=True
+            history_id, contents=["a\nb\nc\nd", "e\nf\ng\nh"], direct_upload=True, wait=True
         )
         self._assert_status_code_is(response, 200)
         hdca = response.json()["output_collections"][0]
@@ -1123,15 +1127,15 @@ class ToolsTestCase(ApiTestCase, TestsTools):
     @uses_test_history(require_new=False)
     def test_list_collection_output(self, history_id):
         create_response = self.dataset_collection_populator.create_list_in_history(
-            history_id, contents=["a\nb\nc\nd", "e\nf\ng\nh"]
+            history_id, contents=["a\nb\nc\nd", "e\nf\ng\nh"], wait=True
         )
-        hdca_id = create_response.json()["id"]
+        hdca_id = create_response.json()["outputs"][0]["id"]
         create = self.dataset_populator.run_collection_creates_list(history_id, hdca_id)
         output_collection = self._assert_one_job_one_collection_run(create)
-        element0, element1 = self._assert_elements_are(output_collection, "data1", "data2")
+        element0, element1 = self._assert_elements_are(output_collection, "data0", "data1")
         self.dataset_populator.wait_for_history(history_id, assert_ok=True)
-        self._verify_element(history_id, element0, contents="identifier is data1\n", file_ext="txt")
-        self._verify_element(history_id, element1, contents="identifier is data2\n", file_ext="txt")
+        self._verify_element(history_id, element0, contents="identifier is data0\n", file_ext="txt")
+        self._verify_element(history_id, element1, contents="identifier is data1\n", file_ext="txt")
 
     @skip_without_tool("collection_creates_list_2")
     @uses_test_history(require_new=False)
@@ -1139,9 +1143,9 @@ class ToolsTestCase(ApiTestCase, TestsTools):
         # test using format_source with a tool
         new_dataset1 = self.dataset_populator.new_dataset(history_id, content="#col1\tcol2")
         create_response = self.dataset_collection_populator.create_list_in_history(
-            history_id, contents=["a\tb\nc\td", "e\tf\ng\th"]
+            history_id, contents=["a\tb\nc\td", "e\tf\ng\th"], wait=True
         )
-        hdca_id = create_response.json()["id"]
+        hdca_id = create_response.json()["outputs"][0]["id"]
         inputs = {
             "header": {"src": "hda", "id": new_dataset1["id"]},
             "input_collect": {"src": "hdca", "id": hdca_id},
@@ -1150,7 +1154,7 @@ class ToolsTestCase(ApiTestCase, TestsTools):
         self.dataset_populator.wait_for_history(history_id, assert_ok=True)
         create = self._run("collection_creates_list_2", history_id, inputs, assert_ok=True)
         output_collection = self._assert_one_job_one_collection_run(create)
-        element0, element1 = self._assert_elements_are(output_collection, "data1", "data2")
+        element0, element1 = self._assert_elements_are(output_collection, "data0", "data1")
         self.dataset_populator.wait_for_history(history_id, assert_ok=True)
         self._verify_element(history_id, element0, contents="#col1\tcol2\na\tb\nc\td\n", file_ext="txt")
         self._verify_element(history_id, element1, contents="#col1\tcol2\ne\tf\ng\th\n", file_ext="txt")
@@ -1605,7 +1609,9 @@ class ToolsTestCase(ApiTestCase, TestsTools):
     @skip_without_tool("cat1")
     @uses_test_history(require_new=False)
     def test_map_over_empty_collection(self, history_id):
-        hdca_id = self.dataset_collection_populator.create_list_in_history(history_id, contents=[]).json()["id"]
+        hdca_id = self.dataset_collection_populator.create_list_in_history(history_id, contents=[]).json()["outputs"][
+            0
+        ]["id"]
         inputs = {
             "input1": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]},
         }
@@ -1661,7 +1667,9 @@ class ToolsTestCase(ApiTestCase, TestsTools):
     @skip_without_tool("output_filter_with_input")
     @uses_test_history(require_new=False)
     def test_map_over_with_output_filter_no_filtering(self, history_id):
-        hdca_id = self.dataset_collection_populator.create_list_in_history(history_id).json()["id"]
+        hdca_id = self.dataset_collection_populator.create_list_in_history(history_id, wait=True).json()["outputs"][0][
+            "id"
+        ]
         inputs = {
             "input_1": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]},
             "produce_out_1": "true",
@@ -1677,9 +1685,9 @@ class ToolsTestCase(ApiTestCase, TestsTools):
     @skip_without_tool("output_filter_with_input_optional")
     @uses_test_history(require_new=False)
     def test_map_over_with_output_filter_on_optional_input(self, history_id):
-        hdca_id = self.dataset_collection_populator.create_list_in_history(history_id, contents=["myinputs"]).json()[
-            "id"
-        ]
+        hdca_id = self.dataset_collection_populator.create_list_in_history(
+            history_id, contents=["myinputs"], wait=True
+        ).json()["outputs"][0]["id"]
         inputs = {
             "input_1": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]},
         }
@@ -1694,7 +1702,9 @@ class ToolsTestCase(ApiTestCase, TestsTools):
     @skip_without_tool("output_filter_with_input")
     @uses_test_history(require_new=False)
     def test_map_over_with_output_filter_one_filtered(self, history_id):
-        hdca_id = self.dataset_collection_populator.create_list_in_history(history_id).json()["id"]
+        hdca_id = self.dataset_collection_populator.create_list_in_history(history_id, wait=True).json()["outputs"][0][
+            "id"
+        ]
         inputs = {
             "input_1": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]},
             "produce_out_1": "true",
@@ -1733,7 +1743,9 @@ class ToolsTestCase(ApiTestCase, TestsTools):
     @skip_without_tool("collection_creates_dynamic_list_of_pairs")
     @uses_test_history(require_new=False)
     def test_map_over_with_discovered_output_collection_elements(self, history_id):
-        hdca_id = self.dataset_collection_populator.create_list_in_history(history_id).json()["id"]
+        hdca_id = self.dataset_collection_populator.create_list_in_history(history_id, wait=True).json()["outputs"][0][
+            "id"
+        ]
         inputs = {"input": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]}}
         create = self._run("collection_creates_dynamic_list_of_pairs", history_id, inputs).json()
         implicit_collections = create["implicit_collections"]
@@ -1750,8 +1762,8 @@ class ToolsTestCase(ApiTestCase, TestsTools):
         bed1_contents = open(self.get_filename("1.bed")).read()
         bed2_contents = open(self.get_filename("2.bed")).read()
         contents = [bed1_contents, bed2_contents]
-        hdca = self.dataset_collection_populator.create_list_in_history(history_id, contents=contents).json()
-        return hdca["id"]
+        hdca = self.dataset_collection_populator.create_list_in_history(history_id, contents=contents, wait=True).json()
+        return hdca["outputs"][0]["id"]
 
     @skip_without_tool("identifier_single")
     @uses_test_history(require_new=False)
@@ -1799,7 +1811,7 @@ class ToolsTestCase(ApiTestCase, TestsTools):
     @skip_without_tool("identifier_multiple")
     @uses_test_history(require_new=False)
     def test_list_selectable_in_multidata_input(self, history_id):
-        self.dataset_collection_populator.create_list_in_history(history_id, contents=["123", "456"])
+        self.dataset_collection_populator.create_list_in_history(history_id, contents=["123", "456"], wait=True)
         build = self.dataset_populator.build_tool_state("identifier_multiple", history_id)
         assert len(build["inputs"][0]["options"]["hdca"]) == 1
 
@@ -2244,11 +2256,11 @@ class ToolsTestCase(ApiTestCase, TestsTools):
     def test_default_identifier_source_map_over(self):
         with self.dataset_populator.test_history() as history_id:
             input_a_hdca_id = self.dataset_collection_populator.create_list_in_history(
-                history_id, contents=[("A", "A content")]
-            ).json()["id"]
+                history_id, contents=[("A", "A content")], wait=True
+            ).json()["outputs"][0]["id"]
             input_b_hdca_id = self.dataset_collection_populator.create_list_in_history(
-                history_id, contents=[("B", "B content")]
-            ).json()["id"]
+                history_id, contents=[("B", "B content")], wait=True
+            ).json()["outputs"][0]["id"]
             inputs = {
                 "inputA": {"batch": True, "values": [dict(src="hdca", id=input_a_hdca_id)]},
                 "inputB": {"batch": True, "values": [dict(src="hdca", id=input_b_hdca_id)]},
@@ -2265,9 +2277,9 @@ class ToolsTestCase(ApiTestCase, TestsTools):
     def test_map_over_collection_output(self):
         with self.dataset_populator.test_history() as history_id:
             create_response = self.dataset_collection_populator.create_list_in_history(
-                history_id, contents=["a\nb\nc\nd", "e\nf\ng\nh"]
+                history_id, contents=["a\nb\nc\nd", "e\nf\ng\nh"], wait=True
             )
-            hdca_id = create_response.json()["id"]
+            hdca_id = create_response.json()["outputs"][0]["id"]
             inputs = {
                 "input1": {"batch": True, "values": [dict(src="hdca", id=hdca_id)]},
             }
@@ -2282,8 +2294,8 @@ class ToolsTestCase(ApiTestCase, TestsTools):
             outer_elements = implicit_collection["elements"]
             assert len(outer_elements) == 2
             element0, element1 = outer_elements
-            assert element0["element_identifier"] == "data1"
-            assert element1["element_identifier"] == "data2"
+            assert element0["element_identifier"] == "data0"
+            assert element1["element_identifier"] == "data1"
 
             pair0, pair1 = element0["object"], element1["object"]
             pair00, pair01 = pair0["elements"]
@@ -2313,7 +2325,7 @@ class ToolsTestCase(ApiTestCase, TestsTools):
     def test_cannot_map_over_incompatible_collections(self):
         with self.dataset_populator.test_history() as history_id:
             hdca1_id = self._build_pair(history_id, ["123\n", "456\n"])
-            hdca2_id = self.dataset_collection_populator.create_list_in_history(history_id).json()["id"]
+            hdca2_id = self.dataset_collection_populator.create_list_in_history(history_id).json()["outputs"][0]["id"]
             inputs = {
                 "input1": {
                     "batch": True,
@@ -2333,8 +2345,8 @@ class ToolsTestCase(ApiTestCase, TestsTools):
     def test_map_over_collection_structured_like(self):
         with self.dataset_populator.test_history() as history_id:
             hdca_id = self.dataset_collection_populator.create_list_in_history(
-                history_id, contents=[("A", "A"), ("B", "B")]
-            ).json()["id"]
+                history_id, contents=[("A", "A"), ("B", "B")], wait=True
+            ).json()["outputs"][0]["id"]
             self.dataset_populator.wait_for_history(history_id, assert_ok=True)
             inputs = {
                 "input": {"values": [dict(src="hdca", id=hdca_id)]},
@@ -2370,8 +2382,8 @@ class ToolsTestCase(ApiTestCase, TestsTools):
     def test_map_over_collection_type_source(self):
         with self.dataset_populator.test_history() as history_id:
             hdca_id = self.dataset_collection_populator.create_list_in_history(
-                history_id, contents=[("A", "A"), ("B", "B")]
-            ).json()["id"]
+                history_id, contents=[("A", "A"), ("B", "B")], wait=True
+            ).json()["outputs"][0]["id"]
             self.dataset_populator.wait_for_history(history_id, assert_ok=True)
             inputs = {
                 "input_collect": {"values": [dict(src="hdca", id=hdca_id)]},
@@ -2385,7 +2397,9 @@ class ToolsTestCase(ApiTestCase, TestsTools):
     def test_reduce_collections_legacy(self):
         with self.dataset_populator.test_history() as history_id:
             hdca1_id = self._build_pair(history_id, ["123\n", "456\n"])
-            hdca2_id = self.dataset_collection_populator.create_list_in_history(history_id).json()["id"]
+            hdca2_id = self.dataset_collection_populator.create_list_in_history(history_id, wait=True).json()[
+                "outputs"
+            ][0]["id"]
             inputs = {
                 "f1": f"__collection_reduce__|{hdca1_id}",
                 "f2": f"__collection_reduce__|{hdca2_id}",
@@ -2396,7 +2410,9 @@ class ToolsTestCase(ApiTestCase, TestsTools):
     def test_reduce_collections(self):
         with self.dataset_populator.test_history() as history_id:
             hdca1_id = self._build_pair(history_id, ["123\n", "456\n"])
-            hdca2_id = self.dataset_collection_populator.create_list_in_history(history_id).json()["id"]
+            hdca2_id = self.dataset_collection_populator.create_list_in_history(history_id, wait=True).json()[
+                "outputs"
+            ][0]["id"]
             inputs = {
                 "f1": {"src": "hdca", "id": hdca1_id},
                 "f2": {"src": "hdca", "id": hdca2_id},
@@ -2407,7 +2423,9 @@ class ToolsTestCase(ApiTestCase, TestsTools):
     def test_implicit_reduce_with_mapping(self):
         with self.dataset_populator.test_history() as history_id:
             hdca1_id = self._build_pair(history_id, ["123\n", "456\n"])
-            hdca2_id = self.dataset_collection_populator.create_list_of_list_in_history(history_id).json()["id"]
+            hdca2_id = self.dataset_collection_populator.create_list_of_list_in_history(history_id, wait=True).json()[
+                "id"
+            ]
             inputs = {
                 "f1": {"src": "hdca", "id": hdca1_id},
                 "f2": {
@@ -2454,6 +2472,7 @@ class ToolsTestCase(ApiTestCase, TestsTools):
                         "ext": "fasta",
                     }
                 ],
+                wait=True,
             )
             self._assert_status_code_is(response, 200)
             hdca_id = response.json()["outputs"][0]["id"]
@@ -2503,7 +2522,9 @@ class ToolsTestCase(ApiTestCase, TestsTools):
     def test_reduce_multiple_lists_on_multi_data(self):
         with self.dataset_populator.test_history() as history_id:
             hdca1_id = self._build_pair(history_id, ["123\n", "456\n"])
-            hdca2_id = self.dataset_collection_populator.create_list_in_history(history_id).json()["id"]
+            hdca2_id = self.dataset_collection_populator.create_list_in_history(history_id, wait=True).json()[
+                "outputs"
+            ][0]["id"]
             inputs = {
                 "f1": [{"src": "hdca", "id": hdca1_id}, {"src": "hdca", "id": hdca2_id}],
                 "f2": [{"src": "hdca", "id": hdca1_id}],
@@ -2561,9 +2582,9 @@ class ToolsTestCase(ApiTestCase, TestsTools):
         with self.dataset_populator.test_history() as history_id:
             nested_list_id = self.__build_nested_list(history_id)
             create_response = self.dataset_collection_populator.create_list_in_history(
-                history_id, contents=["xxx\n", "yyy\n"]
+                history_id, contents=["xxx\n", "yyy\n"], wait=True
             )
-            list_id = create_response.json()["id"]
+            list_id = create_response.json()["outputs"][0]["id"]
             inputs = {
                 "f1": {
                     "batch": True,
@@ -2724,6 +2745,7 @@ class ToolsTestCase(ApiTestCase, TestsTools):
                     "tags": ["group:type:single", "group:condition:untreated"],
                 },
             ],
+            wait=True,
         )
         self._assert_status_code_is(response, 200)
         hdca_list_id = response.json()["outputs"][0]["id"]
@@ -2749,6 +2771,7 @@ class ToolsTestCase(ApiTestCase, TestsTools):
                     ],
                 },
             ],
+            wait=True,
         )
         self._assert_status_code_is(response, 200)
         hdca_list_id = response.json()["outputs"][0]["id"]
@@ -2756,7 +2779,7 @@ class ToolsTestCase(ApiTestCase, TestsTools):
 
     def _build_pair(self, history_id, contents, run_cat=False):
         create_response = self.dataset_collection_populator.create_pair_in_history(
-            history_id, contents=contents, direct_upload=True
+            history_id, contents=contents, direct_upload=True, wait=True
         )
         hdca_id = create_response.json()["output_collections"][0]["id"]
         inputs = {
diff --git a/lib/galaxy_test/api/test_tools_upload.py b/lib/galaxy_test/api/test_tools_upload.py
index 28c5a921c77..38718758eeb 100644
--- a/lib/galaxy_test/api/test_tools_upload.py
+++ b/lib/galaxy_test/api/test_tools_upload.py
@@ -299,7 +299,7 @@ class ToolsUploadTestCase(ApiTestCase):
         ]
         payload = {
             "history_id": history_id,
-            "targets": json.dumps(targets),
+            "targets": targets,
         }
         fetch_response = self.dataset_populator.fetch(payload)
         self._assert_status_code_is(fetch_response, 200)
@@ -340,7 +340,7 @@ class ToolsUploadTestCase(ApiTestCase):
         ]
         payload = {
             "history_id": history_id,
-            "targets": json.dumps(targets),
+            "targets": targets,
         }
         fetch_response = self.dataset_populator.fetch(payload)
         self._assert_status_code_is(fetch_response, 200)
@@ -511,7 +511,7 @@ class ToolsUploadTestCase(ApiTestCase):
             ]
             payload = {
                 "history_id": history_id,
-                "targets": json.dumps(targets),
+                "targets": targets,
             }
             payload["__files"] = {"files_0|file_data": tar_f}
             fetch_response = self.dataset_populator.fetch(payload)
@@ -882,15 +882,17 @@ class ToolsUploadTestCase(ApiTestCase):
         return self.dataset_populator.get_history_dataset_content(history_id, dataset=new_dataset)
 
     def _upload_and_get_details(self, content, **upload_kwds):
+        assert_ok = upload_kwds.pop("assert_ok", True)
         history_id, new_dataset = self._upload(content, **upload_kwds)
-        assert_ok = upload_kwds.get("assert_ok", True)
         return self.dataset_populator.get_history_dataset_details(history_id, dataset=new_dataset, assert_ok=assert_ok)
 
     def _upload(self, content, api="upload1", history_id=None, **upload_kwds):
         assert_ok = upload_kwds.get("assert_ok", True)
         history_id = history_id or self.dataset_populator.new_history()
         if api == "upload1":
-            new_dataset = self.dataset_populator.new_dataset(history_id, content=content, **upload_kwds)
+            new_dataset = self.dataset_populator.new_dataset(
+                history_id, content=content, fetch_data=False, **upload_kwds
+            )
         else:
             assert api == "fetch"
             element = dict(src="files", **upload_kwds)
@@ -898,7 +900,7 @@ class ToolsUploadTestCase(ApiTestCase):
                 "destination": {"type": "hdas"},
                 "elements": [element],
             }
-            targets = json.dumps([target])
+            targets = [target]
             payload = {"history_id": history_id, "targets": targets, "__files": {"files_0|file_data": content}}
             new_dataset = self.dataset_populator.fetch(payload, assert_ok=assert_ok).json()["outputs"][0]
         self.dataset_populator.wait_for_history(history_id, assert_ok=assert_ok)
diff --git a/lib/galaxy_test/api/test_workflow_extraction.py b/lib/galaxy_test/api/test_workflow_extraction.py
index 4f78005d1b2..4cb86862edb 100644
--- a/lib/galaxy_test/api/test_workflow_extraction.py
+++ b/lib/galaxy_test/api/test_workflow_extraction.py
@@ -151,8 +151,8 @@ class WorkflowExtractionApiTestCase(BaseWorkflowsApiTestCase):
     @skip_without_tool("multi_data_param")
     def test_extract_reduction_from_history(self):
         hdca = self.dataset_collection_populator.create_pair_in_history(
-            self.history_id, contents=["1 2 3\n4 5 6", "7 8 9\n10 11 10"]
-        ).json()
+            self.history_id, contents=["1 2 3\n4 5 6", "7 8 9\n10 11 10"], wait=True
+        ).json()["outputs"][0]
         hdca_id = hdca["id"]
         inputs1 = {"input": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]}, "num_lines": 2}
         implicit_hdca1, job_id1 = self._run_tool_get_collection_and_job_id(self.history_id, "random_lines1", inputs1)
@@ -406,8 +406,8 @@ test_data:
 
     def __run_random_lines_mapped_over_pair(self, history_id):
         hdca = self.dataset_collection_populator.create_pair_in_history(
-            history_id, contents=["1 2 3\n4 5 6", "7 8 9\n10 11 10"]
-        ).json()
+            history_id, contents=["1 2 3\n4 5 6", "7 8 9\n10 11 10"], wait=True
+        ).json()["outputs"][0]
         hdca_id = hdca["id"]
         inputs1 = {"input": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]}, "num_lines": 2}
         implicit_hdca1, job_id1 = self._run_tool_get_collection_and_job_id(history_id, "random_lines1", inputs1)
@@ -474,8 +474,10 @@ test_data:
         workflow_request, history_id, workflow_id = self._setup_workflow_run(workflow, history_id=history_id)
         run_workflow_response = self._post(f"workflows/{workflow_id}/invocations", data=workflow_request)
         self._assert_status_code_is(run_workflow_response, 200)
-
-        self.dataset_populator.wait_for_history(history_id, assert_ok=True)
+        invocation_response = run_workflow_response.json()
+        self.workflow_populator.wait_for_invocation_and_jobs(
+            history_id=history_id, workflow_id=workflow_id, invocation_id=invocation_response["id"]
+        )
         return self.__cat_job_id(history_id)
 
     def _assert_first_step_is_paired_input(self, downloaded_workflow):
diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py
index b0fe908ce12..0a72bfc1ad1 100644
--- a/lib/galaxy_test/api/test_workflows.py
+++ b/lib/galaxy_test/api/test_workflows.py
@@ -339,7 +339,7 @@ class WorkflowsApiTestCase(BaseWorkflowsApiTestCase, ChangeDatatypeTestCase):
         assert not [w for w in workflow_index if w["id"] == workflow_id]
 
     def test_index_hidden(self):
-        workflow_id = self.workflow_populator.simple_workflow("test_hidden")
+        workflow_id = self.workflow_populator.simple_workflow("test_delete")
         workflow_index = self._get("workflows").json()
         workflow = [w for w in workflow_index if w["id"] == workflow_id][0]
         workflow["hidden"] = True
@@ -1364,9 +1364,10 @@ steps:
     def test_workflow_run_output_collection_mapping(self):
         workflow_id = self._upload_yaml_workflow(WORKFLOW_WITH_OUTPUT_COLLECTION_MAPPING)
         with self.dataset_populator.test_history() as history_id:
-            hdca1 = self.dataset_collection_populator.create_list_in_history(
+            fetch_response = self.dataset_collection_populator.create_list_in_history(
                 history_id, contents=["a\nb\nc\nd\n", "e\nf\ng\nh\n"]
             ).json()
+            hdca1 = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response)
             self.dataset_populator.wait_for_history(history_id, assert_ok=True)
             inputs = {
                 "0": self._ds_entry(hdca1),
@@ -2764,9 +2765,10 @@ input1:
             workflow = self.workflow_populator.load_workflow_from_resource("test_workflow_map_reduce_pause")
             uploaded_workflow_id = self.workflow_populator.create_workflow(workflow)
             hda1 = self.dataset_populator.new_dataset(history_id, content="reviewed\nunreviewed")
-            hdca1 = self.dataset_collection_populator.create_list_in_history(
+            fetch_response = self.dataset_collection_populator.create_list_in_history(
                 history_id, contents=["1\n2\n3", "4\n5\n6"]
             ).json()
+            hdca1 = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response)
             index_map = {
                 "0": self._ds_entry(hda1),
                 "1": self._ds_entry(hdca1),
@@ -2844,14 +2846,16 @@ steps:
 """
             )
             DELETED = 0
-            PAUSED_1 = 3
-            PAUSED_2 = 5
-            hdca1 = self.dataset_collection_populator.create_list_in_history(
-                history_id, contents=[("sample1-1", "1 2 3")]
+            PAUSED_1 = 1
+            PAUSED_2 = 2
+            fetch_response = self.dataset_collection_populator.create_list_in_history(
+                history_id, contents=[("sample1-1", "1 2 3")], wait=True
             ).json()
-            self.dataset_populator.wait_for_history(history_id, assert_ok=True)
+            hdca1 = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response)
             deleted_id = hdca1["elements"][DELETED]["object"]["id"]
-            r = self._delete(f"histories/{history_id}/contents/{deleted_id}?purge={purge}")
+            self.dataset_populator.delete_dataset(
+                history_id=history_id, content_id=deleted_id, purge=purge, wait_for_purge=True
+            )
             label_map = {"input1": self._ds_entry(hdca1)}
             workflow_request = dict(
                 history=f"hist_id={history_id}",
@@ -2865,14 +2869,12 @@ steps:
             self.workflow_populator.wait_for_invocation_and_jobs(
                 workflow_id, history_id, invocation_id, assert_ok=False
             )
-            # Why is this sleep needed? -John
-            if not purge:
-                time.sleep(5)
             contents = self.__history_contents(history_id)
-            assert contents[DELETED]["deleted"]
+            datasets = [content for content in contents if content["history_content_type"] == "dataset"]
+            assert datasets[DELETED]["deleted"]
             state = "error" if purge else "paused"
-            assert contents[PAUSED_1]["state"] == state
-            assert contents[PAUSED_2]["state"] == "paused"
+            assert datasets[PAUSED_1]["state"] == state
+            assert datasets[PAUSED_2]["state"] == "paused"
 
     def test_run_with_implicit_connection(self):
         with self.dataset_populator.test_history() as history_id:
@@ -3437,6 +3439,8 @@ outer_input:
             hdca2 = self.dataset_collection_populator.create_list_in_history(
                 history_id, contents=[("sample1-2", "4 5 6"), ("sample2-2", "0 a b")]
             ).json()
+            hdca1 = self.dataset_collection_populator.wait_for_fetched_collection(hdca1)
+            hdca2 = self.dataset_collection_populator.wait_for_fetched_collection(hdca2)
             self.dataset_populator.wait_for_history(history_id, assert_ok=True)
             label_map = {"list1": self._ds_entry(hdca1), "list2": self._ds_entry(hdca2)}
             workflow_request = dict(
@@ -4596,10 +4600,10 @@ steps:
         workflow = self.workflow_populator.load_workflow_from_resource("test_workflow_batch")
         workflow_id = self.workflow_populator.create_workflow(workflow)
         with self.dataset_populator.test_history() as history_id:
-            hda1 = self.dataset_populator.new_dataset(history_id, content="1 2 3")
-            hda2 = self.dataset_populator.new_dataset(history_id, content="4 5 6")
-            hda3 = self.dataset_populator.new_dataset(history_id, content="7 8 9")
-            hda4 = self.dataset_populator.new_dataset(history_id, content="10 11 12")
+            hda1 = self.dataset_populator.new_dataset(history_id, content="1 2 3", wait=True)
+            hda2 = self.dataset_populator.new_dataset(history_id, content="4 5 6", wait=True)
+            hda3 = self.dataset_populator.new_dataset(history_id, content="7 8 9", wait=True)
+            hda4 = self.dataset_populator.new_dataset(history_id, content="10 11 12", wait=True)
             parameters = {
                 "0": {
                     "input": {
@@ -5397,10 +5401,8 @@ test_data:
             inputs_by="name",
             inputs=json.dumps({"input1": self._ds_entry(hda1)}),
         )
-        self.workflow_populator.invoke_workflow_and_assert_ok(
-            workflow_id, history_id=history_id, request=workflow_request
-        )
+        self.workflow_populator.invoke_workflow_and_wait(workflow_id, history_id=history_id, request=workflow_request)
         self.assertEqual(
             "Hello World Second!\nhello world 2\n",
-            self.dataset_populator.get_history_dataset_content(history_id, hid=4),
+            self.dataset_populator.get_history_dataset_content(history_id),
         )
diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py
index 4ec0256d11a..8549c38ad76 100644
--- a/lib/galaxy_test/base/populators.py
+++ b/lib/galaxy_test/base/populators.py
@@ -295,21 +295,48 @@ class BaseDatasetPopulator(BasePopulator):
     Galaxy - implementations must implement _get, _post and _delete.
     """
 
-    def new_dataset(self, history_id: str, content=None, wait: bool = False, **kwds) -> dict:
+    def new_dataset(
+        self,
+        history_id: str,
+        content=None,
+        wait: bool = False,
+        fetch_data=True,
+        to_posix_lines=True,
+        auto_decompress=True,
+        **kwds,
+    ) -> dict:
         """Create a new history dataset instance (HDA) and return its ID.
 
         :returns: the HDA id of the new object
         """
-        run_response = self.new_dataset_request(history_id, content=content, wait=wait, **kwds)
+        run_response = self.new_dataset_request(
+            history_id,
+            content=content,
+            wait=wait,
+            fetch_data=fetch_data,
+            to_posix_lines=to_posix_lines,
+            auto_decompress=auto_decompress,
+            **kwds,
+        )
         assert run_response.status_code == 200, f"Failed to create new dataset with response: {run_response.text}"
+        if fetch_data and wait:
+            return self.get_history_dataset_details_raw(
+                history_id=history_id, dataset_id=run_response.json()["outputs"][0]["id"]
+            ).json()
         return run_response.json()["outputs"][0]
 
-    def new_dataset_request(self, history_id: str, content=None, wait: bool = False, **kwds) -> requests.Response:
+    def new_dataset_request(
+        self, history_id: str, content=None, wait: bool = False, fetch_data=True, **kwds
+    ) -> requests.Response:
         """Lower-level dataset creation that returns the upload tool response object."""
         if content is None and "ftp_files" not in kwds:
             content = "TestData123"
-        payload = self.upload_payload(history_id, content=content, **kwds)
-        run_response = self.tools_post(payload)
+        if not fetch_data:
+            payload = self.upload_payload(history_id, content=content, **kwds)
+            run_response = self.tools_post(payload)
+        else:
+            payload = self.fetch_payload(history_id, content=content, **kwds)
+            run_response = self.fetch(payload, wait=wait)
         if wait:
             self.wait_for_tool_run(history_id, run_response, assert_ok=kwds.get("assert_ok", True))
         return run_response
@@ -321,7 +348,7 @@ class BaseDatasetPopulator(BasePopulator):
         timeout: timeout_type = DEFAULT_TIMEOUT,
         wait: Optional[bool] = None,
     ):
-        tool_response = self._post("tools/fetch", data=payload)
+        tool_response = self._post("tools/fetch", data=payload, json=True)
         if wait is None:
             wait = assert_ok
         if wait:
@@ -344,7 +371,7 @@ class BaseDatasetPopulator(BasePopulator):
         ]
         payload = {
             "history_id": history_id,
-            "targets": json.dumps(targets),
+            "targets": targets,
         }
         fetch_response = self.fetch(payload, wait=wait)
         api_asserts.assert_status_code_is(fetch_response, 200)
@@ -375,8 +402,14 @@ class BaseDatasetPopulator(BasePopulator):
         return run_response
 
     def check_run(self, run_response: requests.Response) -> dict:
-        run_response.raise_for_status()
-        run = run_response.json()
+        run = None
+        try:
+            run = run_response.json()
+            run_response.raise_for_status()
+        except Exception:
+            if run and run["err_msg"]:
+                raise Exception(run["err_msg"])
+            raise
         job = run["jobs"][0]
         return job
 
@@ -440,10 +473,26 @@ class BaseDatasetPopulator(BasePopulator):
         delete_response = self._delete(f"histories/{history_id}")
         delete_response.raise_for_status()
 
-    def delete_dataset(self, history_id: str, content_id: str, purge: bool = False) -> Response:
-        delete_response = self._delete(f"histories/{history_id}/contents/{content_id}", {"purge": purge}, json=True)
+    def delete_dataset(
+        self, history_id: str, content_id: str, purge: bool = False, wait_for_purge: bool = False
+    ) -> Response:
+        dataset_url = f"histories/{history_id}/contents/{content_id}"
+        delete_response = self._delete(dataset_url, {"purge": purge}, json=True)
+        delete_response.raise_for_status()
+        if wait_for_purge and delete_response.status_code == 202:
+            return self.wait_for_purge(history_id, content_id)
         return delete_response
 
+    def wait_for_purge(self, history_id, content_id):
+        dataset_url = f"histories/{history_id}/contents/{content_id}"
+
+        def _wait_for_purge():
+            dataset = self._get(dataset_url).json()
+            return dataset["purged"] or None
+
+        wait_on(_wait_for_purge, "dataset to become purged", timeout=2)
+        return self._get(dataset_url)
+
     def create_tool_from_path(self, tool_path: str) -> Dict[str, Any]:
         tool_directory = os.path.dirname(os.path.abspath(tool_path))
         payload = dict(
@@ -518,6 +567,47 @@ class BaseDatasetPopulator(BasePopulator):
     def copy_history(self, history_id, name="API Test Copied History", **kwds) -> Response:
         return self._post("histories", data={"name": name, "history_id": history_id, **kwds})
 
+    def fetch_payload(
+        self,
+        history_id: str,
+        content: str,
+        auto_decompress: bool = False,
+        file_type: str = "txt",
+        dbkey: str = "?",
+        name: str = "Test_Dataset",
+        **kwds,
+    ) -> dict:
+        __files = {}
+        element = {
+            "ext": file_type,
+            "dbkey": dbkey,
+            "name": name,
+            "auto_decompress": auto_decompress,
+        }
+        for arg in ["to_posix_lines", "space_to_tab"]:
+            val = kwds.get(arg)
+            if val:
+                element[arg] = val
+        target = {
+            "destination": {"type": "hdas"},
+            "elements": [element],
+        }
+        if "ftp_files" in kwds:
+            element["src"] = "ftp_import"
+            element["ftp_path"] = kwds["ftp_files"]
+        elif hasattr(content, "read"):
+            element["src"] = "files"
+            __files["files_0|file_data"] = content
+        elif content and "://" in content:
+            element["src"] = "url"
+            element["url"] = content
+        else:
+            element["src"] = "pasted"
+            element["paste_content"] = content
+        targets = [target]
+        payload = {"history_id": history_id, "targets": targets, "__files": __files}
+        return payload
+
     def upload_payload(self, history_id: str, content: Optional[str] = None, **kwds) -> dict:
         name = kwds.get("name", "Test_Dataset")
         dbkey = kwds.get("dbkey", "?")
@@ -1400,8 +1490,8 @@ class BaseWorkflowPopulator(BasePopulator):
             workflow_id = self.create_workflow(workflow)
         if not history_id:
             history_id = self.dataset_populator.new_history()
-        hda1 = self.dataset_populator.new_dataset(history_id, content="1 2 3")
-        hda2 = self.dataset_populator.new_dataset(history_id, content="4 5 6")
+        hda1 = self.dataset_populator.new_dataset(history_id, content="1 2 3", wait=True)
+        hda2 = self.dataset_populator.new_dataset(history_id, content="4 5 6", wait=True)
         workflow_request = dict(
             history=f"hist_id={history_id}",
         )
@@ -2010,7 +2100,7 @@ class BaseDatasetCollectionPopulator:
         # this function uses recursive generation of history hdcas.
         collection_type = kwds.pop("collection_type", "list:list")
         collection_types = collection_type.split(":")
-        list = self.create_list_in_history(history_id, **kwds).json()["id"]
+        list = self.create_list_in_history(history_id, **kwds).json()["output_collections"][0]
         current_collection_type = "list"
         for collection_type in collection_types[1:]:
             current_collection_type = f"{current_collection_type}:{collection_type}"
@@ -2018,22 +2108,22 @@ class BaseDatasetCollectionPopulator:
                 history_id=history_id,
                 collection_type=current_collection_type,
                 name=current_collection_type,
-                collection=[list],
+                collection=[list["id"]],
             )
-            list = response.json()["id"]
+            list = response.json()
         return response
 
-    def create_pair_in_history(self, history_id, **kwds):
+    def create_pair_in_history(self, history_id, wait=False, **kwds):
         payload = self.create_pair_payload(history_id, instance_type="history", **kwds)
-        return self.__create(payload)
+        return self.__create(payload, wait=wait)
 
-    def create_list_in_history(self, history_id, **kwds):
+    def create_list_in_history(self, history_id, wait=False, **kwds):
         payload = self.create_list_payload(history_id, instance_type="history", **kwds)
-        return self.__create(payload)
+        return self.__create(payload, wait=wait)
 
-    def upload_collection(self, history_id, collection_type, elements, **kwds):
+    def upload_collection(self, history_id, collection_type, elements, wait=False, **kwds):
         payload = self.__create_payload_fetch(history_id, collection_type, contents=elements, **kwds)
-        return self.__create(payload)
+        return self.__create(payload, wait=wait)
 
     def create_list_payload(self, history_id, **kwds):
         return self.__create_payload(history_id, identifiers_func=self.list_identifiers, collection_type="list", **kwds)
@@ -2089,6 +2179,7 @@ class BaseDatasetCollectionPopulator:
                         element_identifier = "reverse"
                 element["name"] = element_identifier
                 element["paste_content"] = dataset_contents
+                element["to_posix_lines"] = kwds.get("to_posix_lines", True)
                 elements.append(element)
 
         name = kwds.get("name", "Test Dataset Collection")
@@ -2103,13 +2194,13 @@ class BaseDatasetCollectionPopulator:
         ]
         payload = dict(
             history_id=history_id,
-            targets=json.dumps(targets),
+            targets=targets,
         )
         return payload
 
     def wait_for_fetched_collection(self, fetch_response):
         self.dataset_populator.wait_for_job(fetch_response["jobs"][0]["id"], assert_ok=True)
-        initial_dataset_collection = fetch_response["outputs"][0]
+        initial_dataset_collection = fetch_response["output_collections"][0]
         dataset_collection = self.dataset_populator.get_history_collection_details(
             initial_dataset_collection["history_id"], hid=initial_dataset_collection["hid"]
         )
@@ -2158,14 +2249,14 @@ class BaseDatasetCollectionPopulator:
         element_identifiers = [hda_to_identifier(i, hda) for (i, hda) in enumerate(hdas)]
         return element_identifiers
 
-    def __create(self, payload):
+    def __create(self, payload, wait=False):
         # Create a colleciton - either from existing datasets using collection creation API
         # or from direct uploads with the fetch API. Dispatch on "targets" keyword in payload
         # to decide which to use.
         if "targets" not in payload:
             return self._create_collection(payload)
         else:
-            return self.dataset_populator.fetch(payload)
+            return self.dataset_populator.fetch(payload, wait=wait)
 
     def __datasets(self, history_id, count, contents=None):
         datasets = []
@@ -2243,7 +2334,7 @@ def load_data_dict(
                 if input_type == "File":
                     content = read_test_data(element_data)
                 else:
-                    content = element_data["content"]
+                    content = element_data.pop("content")
                 if content is not None:
                     element_data["src"] = "pasted"
                     element_data["paste_content"] = content
@@ -2287,7 +2378,7 @@ def load_data_dict(
                     new_dataset_kwds["name"] = value["name"]
                 if "file_type" in value:
                     new_dataset_kwds["file_type"] = value["file_type"]
-                hda = dataset_populator.new_dataset(history_id, **new_dataset_kwds)
+                hda = dataset_populator.new_dataset(history_id, wait=True, **new_dataset_kwds)
                 label_map[key] = dataset_populator.ds_entry(hda)
                 has_uploads = True
             elif input_type == "raw":
@@ -2357,7 +2448,7 @@ def wait_on_state(
             return state
 
     if skip_states is None:
-        skip_states = ["running", "queued", "new", "ready", "stop", "stopped", "setting_metadata"]
+        skip_states = ["running", "queued", "new", "ready", "stop", "stopped", "setting_metadata", "waiting"]
     if ok_states is None:
         ok_states = ["ok", "scheduled"]
     try:
diff --git a/lib/galaxy_test/selenium/framework.py b/lib/galaxy_test/selenium/framework.py
index 99f74890f01..4a0ad663fa2 100644
--- a/lib/galaxy_test/selenium/framework.py
+++ b/lib/galaxy_test/selenium/framework.py
@@ -31,6 +31,7 @@ from galaxy.selenium.navigates_galaxy import (
     NavigatesGalaxy,
     retry_during_transitions,
 )
+from galaxy.tool_util.verify.interactor import prepare_request_params
 from galaxy.util import (
     asbool,
     classproperty,
@@ -672,60 +673,37 @@ class SeleniumSessionGetPostMixin:
 
     def _post(self, route, data=None, files=None, headers=None, admin=False, json: bool = False) -> Response:
         full_url = self.selenium_context.build_url(f"api/{route}", for_selenium=False)
-        if data is None:
-            data = {}
-
-        if files is None:
-            files = data.get("__files", None)
-            if files is not None:
-                del data["__files"]
-
         cookies = None
         if admin:
             full_url = f"{full_url}?key={self._mixin_admin_api_key}"
         else:
             cookies = self.selenium_context.selenium_to_requests_cookies()
-        request_kwd = self._prepare_request_data(
-            dict(cookies=cookies, headers=headers, timeout=DEFAULT_SOCKET_TIMEOUT, files=files), data, as_json=json
-        )
-        response = requests.post(full_url, **request_kwd)
+        request_kwd = prepare_request_params(data=data, files=files, as_json=json, headers=headers, cookies=cookies)
+        response = requests.post(full_url, timeout=DEFAULT_SOCKET_TIMEOUT, **request_kwd)
         return response
 
     def _delete(self, route, data=None, headers=None, admin=False, json: bool = False) -> Response:
-        data = data or {}
         full_url = self.selenium_context.build_url(f"api/{route}", for_selenium=False)
         cookies = None
         if admin:
             full_url = f"{full_url}?key={self._mixin_admin_api_key}"
         else:
             cookies = self.selenium_context.selenium_to_requests_cookies()
-        request_kwd = self._prepare_request_data(
-            dict(cookies=cookies, headers=headers, timeout=DEFAULT_SOCKET_TIMEOUT), data, as_json=json
-        )
-        response = requests.delete(full_url, **request_kwd)
+        request_kwd = prepare_request_params(data=data, as_json=json, headers=headers, cookies=cookies)
+        response = requests.delete(full_url, timeout=DEFAULT_SOCKET_TIMEOUT, **request_kwd)
         return response
 
     def _put(self, route, data=None, headers=None, admin=False, json: bool = False) -> Response:
-        data = data or {}
         full_url = self.selenium_context.build_url(f"api/{route}", for_selenium=False)
         cookies = None
         if admin:
             full_url = f"{full_url}?key={self._mixin_admin_api_key}"
         else:
             cookies = self.selenium_context.selenium_to_requests_cookies()
-        request_kwd = self._prepare_request_data(
-            dict(cookies=cookies, headers=headers, timeout=DEFAULT_SOCKET_TIMEOUT), data, as_json=json
-        )
+        request_kwd = prepare_request_params(data=data, as_json=json, headers=headers, cookies=cookies)
         response = requests.put(full_url, **request_kwd)
         return response
 
-    def _prepare_request_data(self, request_kwd: Dict[str, Any], data: Dict[str, Any], as_json: bool = False):
-        if as_json:
-            request_kwd["json"] = data
-        else:
-            request_kwd["data"] = data
-        return request_kwd
-
 
 class SeleniumSessionDatasetPopulator(SeleniumSessionGetPostMixin, populators.BaseDatasetPopulator):
 
diff --git a/lib/galaxy_test/selenium/test_history_copy_elements.py b/lib/galaxy_test/selenium/test_history_copy_elements.py
index 840a307473a..0e3be143004 100644
--- a/lib/galaxy_test/selenium/test_history_copy_elements.py
+++ b/lib/galaxy_test/selenium/test_history_copy_elements.py
@@ -12,8 +12,8 @@ class HistoryCopyElementsTestCase(SeleniumTestCase):
     def test_copy_hdca(self):
         history_id = self.current_history_id()
         input_collection = self.dataset_collection_populator.create_list_in_history(
-            history_id, contents=["0", "1", "0", "1"]
-        ).json()
+            history_id, contents=["0", "1", "0", "1"], wait=True
+        ).json()["outputs"][0]
         input_hid = input_collection["hid"]
 
         failed_response = self.dataset_populator.run_exit_code_from_file(history_id, input_collection["id"])
diff --git a/lib/galaxy_test/selenium/test_history_multi_view.py b/lib/galaxy_test/selenium/test_history_multi_view.py
index 81b9ad9278f..f48fb5e352b 100644
--- a/lib/galaxy_test/selenium/test_history_multi_view.py
+++ b/lib/galaxy_test/selenium/test_history_multi_view.py
@@ -14,7 +14,7 @@ class HistoryMultiViewTestCase(SeleniumTestCase):
         input_collection = self.dataset_collection_populator.create_list_in_history(
             history_id, contents=["0", "1", "0", "1"]
         ).json()
-        input_hid = input_collection["hid"]
+        input_hid = input_collection["outputs"][0]["hid"]
 
         self.home()
 
@@ -28,7 +28,7 @@ class HistoryMultiViewTestCase(SeleniumTestCase):
     @selenium_test
     def test_list_list_display(self):
         history_id = self.current_history_id()
-        method = self.dataset_collection_populator.create_list_of_list_in_history(history_id).json
+        method = self.dataset_collection_populator.create_list_of_list_in_history(history_id, wait=True).json
         selector = self.prepare_multi_history_view(method)
         first_level_element_selector = selector.descendant(".dataset-collection-element")
         self.wait_for_and_click(first_level_element_selector)
@@ -150,6 +150,8 @@ class HistoryMultiViewTestCase(SeleniumTestCase):
 
     def prepare_multi_history_view(self, collection_populator_method):
         collection = collection_populator_method()
+        if "outputs" in collection:
+            collection = self.dataset_collection_populator.wait_for_fetched_collection(collection)
         collection_hid = collection["hid"]
 
         self.home()
diff --git a/lib/galaxy_test/selenium/test_history_panel_collections.py b/lib/galaxy_test/selenium/test_history_panel_collections.py
index 3e6fc64dcac..ea47a25a721 100644
--- a/lib/galaxy_test/selenium/test_history_panel_collections.py
+++ b/lib/galaxy_test/selenium/test_history_panel_collections.py
@@ -76,7 +76,7 @@ class HistoryPanelCollectionsTestCase(SeleniumTestCase):
         history_id = self.current_history_id()
         input_collection = self.dataset_collection_populator.create_list_in_history(
             history_id, contents=["0", "1", "0", "1"]
-        ).json()
+        ).json()["outputs"][0]
 
         ok_inputs = {"input1": {"src": "hdca", "id": input_collection["id"]}}
         ok_response = self.dataset_populator.run_tool("collection_creates_list", ok_inputs, history_id)
@@ -285,15 +285,17 @@ class HistoryPanelCollectionsTestCase(SeleniumTestCase):
     def _generate_partially_failed_collection_with_input(self):
         history_id = self.current_history_id()
         input_collection = self.dataset_collection_populator.create_list_in_history(
-            history_id, contents=["0", "1", "0", "1"]
-        ).json()
+            history_id, contents=["0", "1", "0", "1"], wait=True
+        ).json()["outputs"][0]
         failed_response = self.dataset_populator.run_exit_code_from_file(history_id, input_collection["id"])
         failed_collection = failed_response["implicit_collections"][0]
         return input_collection, failed_collection
 
     def _populated_paired_and_wait_for_it(self):
         history_id = self.current_history_id()
-        input_collection = self.dataset_collection_populator.create_pair_in_history(history_id).json()
+        input_collection = self.dataset_collection_populator.create_pair_in_history(history_id, wait=True).json()[
+            "outputs"
+        ][0]
         collection_hid = input_collection["hid"]
         if not self.is_beta_history():
             self.home()
diff --git a/test/integration/test_datatype_upload.py b/test/integration/test_datatype_upload.py
index da4347cb1e8..9f79c95a7de 100644
--- a/test/integration/test_datatype_upload.py
+++ b/test/integration/test_datatype_upload.py
@@ -72,7 +72,11 @@ def upload_datatype_helper(instance, test_data, temp_file, delete_cache_dir=Fals
         else:
             file_type = test_data.datatype.file_ext
         dataset = instance.dataset_populator.new_dataset(
-            instance.history_id, content=content, wait=False, file_type=file_type
+            instance.history_id,
+            content=content,
+            wait=False,
+            file_type=file_type,
+            auto_decompress=True,
         )
     dataset = instance.dataset_populator.get_history_dataset_details(
         instance.history_id, dataset=dataset, assert_ok=False
diff --git a/test/integration/test_kubernetes_runner.py b/test/integration/test_kubernetes_runner.py
index be335bf2831..9daf7f98455 100644
--- a/test/integration/test_kubernetes_runner.py
+++ b/test/integration/test_kubernetes_runner.py
@@ -143,7 +143,7 @@ def job_config(jobs_directory):
         </destination>
     </destinations>
     <tools>
-        <tool id="upload1" destination="local_dest"/>
+        <tool id="__DATA_FETCH__" destination="local_dest"/>
         <tool id="create_2" destination="k8s_destination_walltime_short"/>
         <tool id="galaxy_slots_and_memory" destination="k8s_destination_no_cleanup"/>
     </tools>
diff --git a/test/integration/test_kubernetes_staging.py b/test/integration/test_kubernetes_staging.py
index f923db679f2..1e155b4d0c3 100644
--- a/test/integration/test_kubernetes_staging.py
+++ b/test/integration/test_kubernetes_staging.py
@@ -68,7 +68,7 @@ execution:
     local_environment:
       runner: local
 tools:
-  - id: upload1
+  - id: __DATA_FETCH__
     environment: local_environment
 """
 
@@ -98,7 +98,7 @@ execution:
     local_environment:
       runner: local
 tools:
-  - id: upload1
+  - id: __DATA_FETCH__
     environment: local_environment
 """
 
diff --git a/test/integration/test_remote_files.py b/test/integration/test_remote_files.py
index e854f9df57e..1665ed9677b 100644
--- a/test/integration/test_remote_files.py
+++ b/test/integration/test_remote_files.py
@@ -216,11 +216,11 @@ class RemoteFilesIntegrationTestCase(ConfiguresRemoteFilesIntegrationTestCase):
         assert "test0" not in os.listdir(ftp_dir)
         _write_file_fixtures(self.root, ftp_dir)
         with dataset_populator.test_history() as history_id:
-            hdca = self.dataset_collection_populator.create_list_of_list_in_history(history_id).json()
+            hdca = self.dataset_collection_populator.create_list_of_list_in_history(history_id, wait=True).json()
             outer_elements = hdca["elements"][0]
             assert outer_elements["element_identifier"] == "test0"
             for i in range(2):
-                assert outer_elements["object"]["elements"][i]["element_identifier"] == f"data{i + 1}"
+                assert outer_elements["object"]["elements"][i]["element_identifier"] == f"data{i}"
                 assert outer_elements["object"]["elements"][i]["object"]["file_ext"] == "txt"
             incollection = {"src": "hdca", "id": hdca["id"]}
             inputs = {
@@ -232,7 +232,7 @@ class RemoteFilesIntegrationTestCase(ConfiguresRemoteFilesIntegrationTestCase):
             dataset_populator.wait_for_job(response["jobs"][0]["id"], assert_ok=True)
             assert "test0" in os.listdir(ftp_dir)
             subdir_content = os.listdir(os.path.join(ftp_dir, "test0"))
-            assert sorted(subdir_content) == ["data1.txt", "data2.txt", "data3.txt"]
+            assert sorted(subdir_content) == ["data0.txt", "data1.txt", "data2.txt"]
 
     def _assert_index_empty(self, index):
         assert len(index) == 0
diff --git a/test/integration/test_upload_configuration_options.py b/test/integration/test_upload_configuration_options.py
index 8eafc8d71e6..b182a3cc5cb 100644
--- a/test/integration/test_upload_configuration_options.py
+++ b/test/integration/test_upload_configuration_options.py
@@ -419,10 +419,10 @@ class SimpleFtpUploadConfigurationTestCase(BaseFtpUploadConfigurationTestCase):
         assert len(ftp_files) == 1, ftp_files
         assert ftp_files[0]["path"] == "test"
         assert os.path.exists(ftp_path)
-        # set to_posix_lines to None to exercise purging - by default this file type wouldn't
+        # set to_posix_lines to False to exercise purging - by default this file type wouldn't
         # be purged.
         dataset = self.dataset_populator.new_dataset(
-            self.history_id, ftp_files="test", file_type="txt", to_posix_lines=None, wait=True
+            self.history_id, ftp_files="test", file_type="txt", to_posix_lines=False, wait=True
         )
         self._check_content(dataset, content)
 
@@ -794,7 +794,7 @@ class FetchByPathTestCase(BaseUploadContentConfigurationTestCase):
         targets = [{"destination": destination, "items": items}]
         payload = {
             "history_id": history_id,  # TODO: Shouldn't be needed :(
-            "targets": json.dumps(targets),
+            "targets": targets,
         }
         self.dataset_populator.fetch(payload)
         dataset = self.library_populator.get_library_contents_with_path(library["id"], "/4.bed")
@@ -809,7 +809,7 @@ class FetchByPathTestCase(BaseUploadContentConfigurationTestCase):
         targets = [{"destination": destination, "items": items}]
         payload = {
             "history_id": history_id,  # TODO: Shouldn't be needed :(
-            "targets": json.dumps(targets),
+            "targets": targets,
         }
         self.dataset_populator.fetch(payload)
         dataset = self.library_populator.get_library_contents_with_path(library["id"], "/4.bed")
@@ -830,7 +830,7 @@ class FetchByPathTestCase(BaseUploadContentConfigurationTestCase):
         ]
         payload = {
             "history_id": history_id,  # TODO: Shouldn't be needed :(
-            "targets": json.dumps(targets),
+            "targets": targets,
         }
         self.dataset_populator.fetch(payload)
         dataset = self.library_populator.get_library_contents_with_path(library["id"], "/file1")
@@ -853,7 +853,7 @@ class FetchByPathTestCase(BaseUploadContentConfigurationTestCase):
         ]
         payload = {
             "history_id": self.history_id,  # TODO: Shouldn't be needed :(
-            "targets": json.dumps(targets),
+            "targets": targets,
         }
         fetch_response = self.dataset_populator.fetch(payload)
         self._assert_status_code_is(fetch_response, 200)
@@ -882,7 +882,7 @@ class FetchByPathTestCase(BaseUploadContentConfigurationTestCase):
         ]
         payload = {
             "history_id": self.history_id,  # TODO: Shouldn't be needed :(
-            "targets": json.dumps(targets),
+            "targets": targets,
         }
         self.dataset_populator.fetch(payload)
         contents_response = self.dataset_populator._get_contents_request(self.history_id)
@@ -902,7 +902,7 @@ class FetchByPathTestCase(BaseUploadContentConfigurationTestCase):
         ]
         payload = {
             "history_id": self.history_id,  # TODO: Shouldn't be needed :(
-            "targets": json.dumps(targets),
+            "targets": targets,
         }
         self.dataset_populator.fetch(payload)
         libraries = self.library_populator.get_libraries()
diff --git a/test/integration/test_workflow_scheduling_options.py b/test/integration/test_workflow_scheduling_options.py
index 85a99350957..66ade84db29 100644
--- a/test/integration/test_workflow_scheduling_options.py
+++ b/test/integration/test_workflow_scheduling_options.py
@@ -80,9 +80,10 @@ steps:
 """
         )
         with self.dataset_populator.test_history() as history_id:
-            hdca1 = self.dataset_collection_populator.create_list_in_history(
+            fetch_response = self.dataset_collection_populator.create_list_in_history(
                 history_id, contents=["a\nb\nc\nd\n", "e\nf\ng\nh\n"]
             ).json()
+            hdca1 = self.dataset_collection_populator.wait_for_fetched_collection(fetch_response)
             self.dataset_populator.wait_for_history(history_id, assert_ok=True)
             inputs = {
                 "0": {"src": "hdca", "id": hdca1["id"]},