From 991f82b136ec247745feb591701ea8b50b19e925 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 8 Feb 2022 22:56:17 +0100 Subject: [PATCH 1/4] Fix history exports import by url --- lib/galaxy/webapps/galaxy/api/histories.py | 4 ++-- lib/galaxy_test/base/populators.py | 14 +++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/histories.py b/lib/galaxy/webapps/galaxy/api/histories.py index 2628d6a5f95..4f58ef4993c 100644 --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -59,7 +59,7 @@ from galaxy.web import ( expose_api, expose_api_anonymous, expose_api_anonymous_and_sessionless, - expose_api_raw, + expose_api_raw_anonymous, ) from galaxy.webapps.galaxy.api.common import ( parse_serialization_params, @@ -765,7 +765,7 @@ class HistoriesController(BaseGalaxyAPIController): trans.response.status = 202 return export_result - @expose_api_raw + @expose_api_raw_anonymous def archive_download(self, trans, id, jeha_id, **kwds): """ GET /api/histories/{id}/exports/{jeha_id} diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index 78d70832ee7..fa3417054c7 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -913,6 +913,11 @@ class BaseDatasetPopulator(BasePopulator): assert update_response.status_code == 200, update_response.content return update_response.json() + def make_public(self, history_id: str) -> dict: + sharing_response = self._put(f"histories/{history_id}/publish") + assert sharing_response.status_code == 200 + return sharing_response.json() + def validate_dataset(self, history_id: str, dataset_id: str) -> Dict[str, Any]: url = f"histories/{history_id}/contents/{dataset_id}/validate" update_response = self._put(url) @@ -1035,13 +1040,12 @@ class BaseDatasetPopulator(BasePopulator): return len(contents) def reimport_history(self, history_id, history_name, wait_on_history_length, export_kwds, api_key): + # Make history public so we can import by url + self.make_public(history_id) # Export the history. - download_path = self.export_url(history_id, export_kwds, api_key, check_download=True) + download_url = self.export_url(history_id, export_kwds, api_key, check_download=True) - # Create download for history - full_download_url = urllib.parse.urljoin(download_path, f"?key={api_key}") - - import_data = dict(archive_source=full_download_url, archive_type="url") + import_data = dict(archive_source=download_url, archive_type="url") imported_history_id = self.import_history_and_wait_for_name(import_data, history_name) From 925f44a16ea48408e4a61cc89f2795dce415cfd8 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 9 Feb 2022 17:30:12 +0100 Subject: [PATCH 2/4] Improve (history import) job error reporting --- lib/galaxy_test/base/populators.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index fa3417054c7..5e42dbbe301 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -555,7 +555,7 @@ class BaseDatasetPopulator(BasePopulator): self.wait_for_history(history_id, assert_ok=True, timeout=timeout) def wait_for_job(self, job_id: str, assert_ok: bool = False, timeout: timeout_type = DEFAULT_TIMEOUT): - return wait_on_state(lambda: self.get_job_details(job_id), desc="job state", assert_ok=assert_ok, timeout=timeout) + return wait_on_state(lambda: self.get_job_details(job_id, full=True), desc="job state", assert_ok=assert_ok, timeout=timeout) def get_job_details(self, job_id: str, full: bool = False) -> Response: return self._get(f"jobs/{job_id}?full={full}") @@ -996,6 +996,7 @@ class BaseDatasetPopulator(BasePopulator): files["archive_file"] = archive_file import_response = self._post("histories", data=import_data, files=files) api_asserts.assert_status_code_is(import_response, 200) + return import_response.json()['id'] def import_history_and_wait_for_name(self, import_data, history_name): def history_names(): @@ -1004,7 +1005,8 @@ class BaseDatasetPopulator(BasePopulator): import_name = f"imported from archive: {history_name}" assert import_name not in history_names() - self.import_history(import_data) + job_id = self.import_history(import_data) + self.wait_for_job(job_id, assert_ok=True) def has_history_with_name(): histories = history_names() @@ -2155,12 +2157,13 @@ def wait_on_state(state_func: Callable, desc="state", skip_states=None, ok_state def get_state(): response = state_func() assert response.status_code == 200, f"Failed to fetch state update while waiting. [{response.content}]" - state = response.json()["state"] + state_response = response.json() + state = state_response["state"] if state in skip_states: return None else: if assert_ok: - assert state in ok_states, f"Final state - {state} - not okay." + assert state in ok_states, f"Final state - {state} - not okay. Full response: {state_response}" return state if skip_states is None: From db6e2db4dc99ed4bdce529c71200b19df1ad6911 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 9 Feb 2022 17:30:34 +0100 Subject: [PATCH 3/4] Endpoint needs to be sessionless as well --- lib/galaxy/webapps/galaxy/api/histories.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/histories.py b/lib/galaxy/webapps/galaxy/api/histories.py index 4f58ef4993c..d17bee888a0 100644 --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -59,7 +59,7 @@ from galaxy.web import ( expose_api, expose_api_anonymous, expose_api_anonymous_and_sessionless, - expose_api_raw_anonymous, + expose_api_raw_anonymous_and_sessionless, ) from galaxy.webapps.galaxy.api.common import ( parse_serialization_params, @@ -765,7 +765,7 @@ class HistoriesController(BaseGalaxyAPIController): trans.response.status = 202 return export_result - @expose_api_raw_anonymous + @expose_api_raw_anonymous_and_sessionless def archive_download(self, trans, id, jeha_id, **kwds): """ GET /api/histories/{id}/exports/{jeha_id} From 96d89a4f5c01e688e89621cb4c7504ed5dc9a8f7 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 9 Feb 2022 17:38:10 +0100 Subject: [PATCH 4/4] Drop API key from more places --- lib/galaxy_test/api/test_histories.py | 2 +- lib/galaxy_test/api/test_workflow_extraction.py | 2 +- lib/galaxy_test/base/populators.py | 13 ++++++------- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/lib/galaxy_test/api/test_histories.py b/lib/galaxy_test/api/test_histories.py index fc7983d10a9..4d1f08c842b 100644 --- a/lib/galaxy_test/api/test_histories.py +++ b/lib/galaxy_test/api/test_histories.py @@ -390,7 +390,7 @@ class ImportExportTests(BaseHistories): self.dataset_populator.wait_for_history(history_id, assert_ok=assert_ok) return self.dataset_populator.reimport_history( - history_id, history_name, wait_on_history_length=wait_on_history_length, export_kwds=export_kwds, api_key=self.galaxy_interactor.api_key + history_id, history_name, wait_on_history_length=wait_on_history_length, export_kwds=export_kwds, ) def _import_history_and_wait(self, import_data, history_name, wait_on_history_length=None): diff --git a/lib/galaxy_test/api/test_workflow_extraction.py b/lib/galaxy_test/api/test_workflow_extraction.py index 1a4a34ae504..7d509c0d9e5 100644 --- a/lib/galaxy_test/api/test_workflow_extraction.py +++ b/lib/galaxy_test/api/test_workflow_extraction.py @@ -499,7 +499,7 @@ test_data: history_length = self.dataset_populator.history_length(history_id) new_history_id = self.dataset_populator.reimport_history( - history_id, history_name, wait_on_history_length=history_length, export_kwds={}, api_key=self.galaxy_interactor.api_key + history_id, history_name, wait_on_history_length=history_length, export_kwds={}, ) # wait a little more for those jobs, todo fix to wait for history imported false or # for a specific number of jobs... diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index 5e42dbbe301..9183483c2c5 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -972,20 +972,19 @@ class BaseDatasetPopulator(BasePopulator): assert "job_id" in job_desc return self.wait_for_job(job_desc["job_id"]) - def export_url(self, history_id: str, data, api_key: str, check_download: bool = True) -> str: + def export_url(self, history_id: str, data, check_download: bool = True) -> str: put_response = self.prepare_export(history_id, data) response = put_response.json() api_asserts.assert_has_keys(response, "download_url") download_url = urllib.parse.urljoin(self.galaxy_interactor.api_url, response["download_url"].strip('/')) if check_download: - self.get_export_url(download_url, api_key) + self.get_export_url(download_url) return download_url - def get_export_url(self, export_url, api_key) -> Response: - full_download_url = f"{export_url}?key={api_key}" - download_response = self._get(full_download_url) + def get_export_url(self, export_url) -> Response: + download_response = self._get(export_url) api_asserts.assert_status_code_is(download_response, 200) return download_response @@ -1041,11 +1040,11 @@ class BaseDatasetPopulator(BasePopulator): contents = contents_response.json() return len(contents) - def reimport_history(self, history_id, history_name, wait_on_history_length, export_kwds, api_key): + def reimport_history(self, history_id, history_name, wait_on_history_length, export_kwds): # Make history public so we can import by url self.make_public(history_id) # Export the history. - download_url = self.export_url(history_id, export_kwds, api_key, check_download=True) + download_url = self.export_url(history_id, export_kwds, check_download=True) import_data = dict(archive_source=download_url, archive_type="url")