From a3961477894e1092bb9236a37c8c28ad1ca6a16c Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Wed, 8 Feb 2023 14:02:11 -0800 Subject: [PATCH 1/6] prevent leaking decoded ids through markdown error handling --- lib/galaxy/managers/markdown_util.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/managers/markdown_util.py b/lib/galaxy/managers/markdown_util.py index 1584f9c90da..4e5219251cc 100644 --- a/lib/galaxy/managers/markdown_util.py +++ b/lib/galaxy/managers/markdown_util.py @@ -110,13 +110,7 @@ class GalaxyInternalMarkdownDirectiveHandler(metaclass=abc.ABCMeta): raise MalformedContents(f"Missing object identifier [{line}].") def _remap(container, line): - id_match = re.search(UNENCODED_ID_PATTERN, line) - object_id = None - encoded_id = None - if id_match: - object_id = int(id_match.group(2)) - encoded_id = trans.security.encode_id(object_id) - line = line.replace(id_match.group(), f"{id_match.group(1)}={encoded_id}") + line, object_id, encoded_id = self._encode_line(trans, line) if container == "history_link": _check_object(object_id, line) history = history_manager.get_accessible(object_id, trans.user) @@ -196,11 +190,22 @@ class GalaxyInternalMarkdownDirectiveHandler(metaclass=abc.ABCMeta): try: return _remap(container, line) except Exception as e: + line, *_ = self._encode_line(trans, line) return self.handle_error(container, line, str(e)) export_markdown = _remap_galaxy_markdown_calls(_remap_container, internal_galaxy_markdown) return export_markdown + def _encode_line(self, trans, line): + id_match = re.search(UNENCODED_ID_PATTERN, line) + object_id = None + encoded_id = None + if id_match: + object_id = int(id_match.group(2)) + encoded_id = trans.security.encode_id(object_id) + line = line.replace(id_match.group(), f"{id_match.group(1)}={encoded_id}") + return line, object_id, encoded_id + @abc.abstractmethod def handle_history_link(self, line, history): pass From df8a4f1c9ab1e8e4ded93b477354efcd3d827cfb Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Thu, 9 Feb 2023 09:34:28 +0100 Subject: [PATCH 2/6] Update client API schema --- client/src/schema/schema.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/client/src/schema/schema.ts b/client/src/schema/schema.ts index ed9bc4fd31d..ed8ed9a1373 100644 --- a/client/src/schema/schema.ts +++ b/client/src/schema/schema.ts @@ -520,7 +520,7 @@ export interface paths { }; "/api/histories/{history_id}/contents/{id}": { /** - * Return detailed information about an HDA within a history. + * Return detailed information about an HDA within a history. ``/api/histories/{history_id}/contents/{type}s/{id}`` should be used instead. * @deprecated * @description Return detailed information about an `HDA` or `HDCA` within a history. * @@ -528,7 +528,7 @@ export interface paths { */ get: operations["history_content_api_histories__history_id__contents__id__get"]; /** - * Updates the values for the history content item with the given ``ID``. + * Updates the values for the history content item with the given ``ID``. ``/api/histories/{history_id}/contents/{type}s/{id}`` should be used instead. * @deprecated * @description Updates the values for the history content item with the given ``ID``. */ @@ -10605,7 +10605,7 @@ export interface operations { }; history_content_api_histories__history_id__contents__id__get: { /** - * Return detailed information about an HDA within a history. + * Return detailed information about an HDA within a history. ``/api/histories/{history_id}/contents/{type}s/{id}`` should be used instead. * @deprecated * @description Return detailed information about an `HDA` or `HDCA` within a history. * @@ -10658,7 +10658,7 @@ export interface operations { }; update_api_histories__history_id__contents__id__put: { /** - * Updates the values for the history content item with the given ``ID``. + * Updates the values for the history content item with the given ``ID``. ``/api/histories/{history_id}/contents/{type}s/{id}`` should be used instead. * @deprecated * @description Updates the values for the history content item with the given ``ID``. */ From 86a4f835058155bac8450ec290766173b7b3c9c7 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 8 Feb 2023 14:10:19 -0500 Subject: [PATCH 3/6] Fixes for Pulsar 0.15.0. --- .../dependencies/pinned-requirements.txt | 2 +- lib/galaxy/jobs/runners/pulsar.py | 6 ++- test/integration/test_coexecution.py | 49 ++++++++++++++++++- 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index b06e4cc7be7..1bc3767683b 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -117,7 +117,7 @@ pkgutil-resolve-name==1.3.10 ; python_version >= "3.7" and python_version < "3.9 prompt-toolkit==3.0.36 ; python_version >= "3.7" and python_version < "3.12" prov==1.5.1 ; python_version >= "3.7" and python_version < "3.12" psutil==5.9.4 ; python_version >= "3.7" and python_version < "3.12" -pulsar-galaxy-lib==0.15.0.dev0 ; python_version >= "3.7" and python_version < "3.12" +pulsar-galaxy-lib==0.15.0.dev1 ; python_version >= "3.7" and python_version < "3.12" pyasn1==0.4.8 ; python_version >= "3.7" and python_version < "3.12" pycparser==2.21 ; python_version >= "3.7" and python_version < "3.12" pycryptodome==3.16.0 ; python_version >= "3.7" and python_version < "3.12" diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index 39eb547a0fb..2f74366c478 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -99,6 +99,10 @@ PULSAR_PARAM_SPECS = dict( map=specs.to_str_or_none, default=None, ), + amqp_key_prefix=dict( + map=specs.to_str_or_none, + default=None, + ), galaxy_url=dict( map=specs.to_str_or_none, default=None, @@ -968,7 +972,7 @@ class PulsarMQJobRunner(PulsarJobRunner): ) -DEFAULT_PULSAR_CONTAINER = "galaxy/pulsar-pod-staging:0.15.0.1" +DEFAULT_PULSAR_CONTAINER = "galaxy/pulsar-pod-staging:0.15.0.2" COEXECUTION_DESTENTATION_DEFAULTS = { "default_file_action": "remote_transfer", "rewrite_parameters": "true", diff --git a/test/integration/test_coexecution.py b/test/integration/test_coexecution.py index 27709fcfc9c..4e542699a3b 100644 --- a/test/integration/test_coexecution.py +++ b/test/integration/test_coexecution.py @@ -8,7 +8,7 @@ communicate job status updates back. For this reason, this test will only work out of the box currently with Docker for Mac, rabbitmq installed via Homebrew, and if a fixed port is set for the test. - GALAXY_TEST_PORT=9234 pytest test/integration/test_kubernetes_staging.py + GALAXY_TEST_PORT=9234 pytest test/integration/test_coexecution.py """ import os @@ -186,6 +186,37 @@ tools: """ +TES_CONTAINERIZED_TEMPLATE_CUSTOM_AMQP_KEY = """ +runners: + local: + load: galaxy.jobs.runners.local:LocalJobRunner + workers: 1 + pulsar_tes: + load: galaxy.jobs.runners.pulsar:PulsarTesJobRunner + amqp_url: ${amqp_url} + amqp_key_prefix: pulsar_foobar34_ + +execution: + default: pulsar_tes_environment + environments: + pulsar_tes_environment: + runner: pulsar_tes + tes_url: ${tes_url} + docker_enabled: true + docker_default_container_id: busybox:ubuntu-14.04 + pulsar_app_config: + message_queue_url: '${container_amqp_url}' + env: + - name: SOME_ENV_VAR + value: '42' + local_environment: + runner: local +tools: + - id: __DATA_FETCH__ + environment: local_environment +""" + + def tes_job_config(template_str: str, jobs_directory: str) -> str: job_conf_template = string.Template(template_str) assert AMQP_URL @@ -312,6 +343,20 @@ class TestTesCoexecutionContainerIntegration(TestCoexecution): set_infrastucture_url(config) +@integration_util.skip_unless_environ("FUNNEL_SERVER_TARGET") +class TestTesCoexecutionCustomAmqpKeyContainerIntegration(TestCoexecution): + @classmethod + def handle_galaxy_config_kwds(cls, config) -> None: + config["jobs_directory"] = cls.jobs_directory + config["file_path"] = cls.jobs_directory + config["job_config_file"] = tes_job_config(TES_CONTAINERIZED_TEMPLATE_CUSTOM_AMQP_KEY, cls.jobs_directory) + + config["default_job_shell"] = "/bin/sh" + # Disable tool dependency resolution. + config["tool_dependency_dir"] = "none" + set_infrastucture_url(config) + + @integration_util.skip_unless_environ("FUNNEL_SERVER_TARGET") class TestTesDependencyResolutionIntegration(TestCoexecution): @classmethod @@ -342,7 +387,7 @@ def to_infrastructure_uri(uri: str) -> str: # remap MQ or file server URI hostnames for in-container versions, this is sloppy # should actually parse the URI and rebuild with correct host # Copied from Pulsar's integraiton tests. - infrastructure_host = os.environ.get("GALAXY_TEST_INFRASTRUCTURE_HOST") + infrastructure_host = GALAXY_TEST_INFRASTRUCTURE_HOST if infrastructure_host == "_PLATFORM_AUTO_": system = platform.system() if system in ["Darwin", "Windows"]: From 77eee0151bb1081122c671c47e6dc632f93f5ca0 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 9 Feb 2023 14:05:47 -0500 Subject: [PATCH 4/6] Overhaul workflow success to calculate target histories better --- .../Workflow/Run/WorkflowRunSuccess.vue | 33 ++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/client/src/components/Workflow/Run/WorkflowRunSuccess.vue b/client/src/components/Workflow/Run/WorkflowRunSuccess.vue index 00e207ab0dc..2713c1fa584 100644 --- a/client/src/components/Workflow/Run/WorkflowRunSuccess.vue +++ b/client/src/components/Workflow/Run/WorkflowRunSuccess.vue @@ -2,16 +2,17 @@

- Successfully invoked workflow {{ workflowName }} - {{ timesExecuted }} times. + Successfully invoked workflow {{ workflowName }} + - {{ timesExecuted }} times.

-

+ +

This workflow will generate results in multiple histories. You can observe progress in the - history multi-view. + history multi-view.

This workflow will generate results in a new history. - Switch to that history now. + Switch to that history now.

You can check the status of queued jobs and view the resulting data the History panel.

@@ -53,12 +54,22 @@ export default { multipleInvocations() { return this.timesExecuted > 1; }, - historyTarget() { - if (this.multipleInvocations) { - return `${getAppRoot()}histories/view_multiple`; - } else { - return `${getAppRoot()}history/switch_to_history?hist_id=${this.invocations[0].history_id}`; - } + multipleHistoryTargets() { + return this.targetHistories.length > 1; + }, + targetHistories() { + return this.invocations.reduce((histories, invocation) => { + if (invocation.history_id && !histories.includes(invocation.history_id)) { + histories.push(invocation.history_id); + } + return histories; + }, []); + }, + multiHistoryView() { + return `${getAppRoot()}histories/view_multiple`; + }, + newHistoryTarget() { + return `${getAppRoot()}history/switch_to_history?hist_id=${this.invocations[0].history_id}`; }, wasNewHistoryTarget() { if (this.invocations.length < 1) { From 232246746600931cfb7dfe6b581e27d8bbe5d787 Mon Sep 17 00:00:00 2001 From: John Davis Date: Thu, 9 Feb 2023 15:19:21 -0500 Subject: [PATCH 5/6] Add new release tags to manage_db.sh script We update these tags each time there's a new release, so that it's possible to run sh manage_db.sh upgrade release_23.0. --- lib/galaxy/model/migrations/dbscript.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy/model/migrations/dbscript.py b/lib/galaxy/model/migrations/dbscript.py index f1326fe39f0..ff5577f220c 100644 --- a/lib/galaxy/model/migrations/dbscript.py +++ b/lib/galaxy/model/migrations/dbscript.py @@ -36,6 +36,8 @@ REVISION_TAGS = { "22.01": "base", "release_22.05": "186d4835587b", "22.05": "186d4835587b", + "release_23.0": "186d4835587b", + "23.0": "186d4835587b", } From 1a2e15b353d7467cc8190e69339b451077eda3b7 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 9 Feb 2023 21:37:35 -0500 Subject: [PATCH 6/6] Adjust clientside search delay -- no reason to wait a full second here --- client/src/components/Panels/Common/ToolSearch.vue | 1 + 1 file changed, 1 insertion(+) diff --git a/client/src/components/Panels/Common/ToolSearch.vue b/client/src/components/Panels/Common/ToolSearch.vue index 167659c2870..2181b2fc349 100644 --- a/client/src/components/Panels/Common/ToolSearch.vue +++ b/client/src/components/Panels/Common/ToolSearch.vue @@ -4,6 +4,7 @@