From eef37e0cd77c9d79311f2770c6bd9974d554262f Mon Sep 17 00:00:00 2001 From: John Davis Date: Wed, 19 Oct 2022 12:04:19 -0400 Subject: [PATCH 01/13] Use create_time instead of dropped update_time --- lib/galaxy/jobs/handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/jobs/handler.py b/lib/galaxy/jobs/handler.py index 6eae4ad7bab..3697bbc9b04 100644 --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -738,7 +738,7 @@ class JobHandlerQueue(Monitors): # History is job.state_history started = None finished = None - for history in sorted(job.state_history, key=lambda h: h.update_time): + for history in sorted(job.state_history, key=lambda h: h.create_time): if history.state == "running": started = history.create_time elif history.state == "ok": From e3f2bab381164bfbd7bbe7c2563e5e3ccad28c46 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 24 Oct 2022 14:59:33 +0200 Subject: [PATCH 02/13] Add test for ignored comments --- test/unit/tool_util/test_tool_linters.py | 29 ++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test/unit/tool_util/test_tool_linters.py b/test/unit/tool_util/test_tool_linters.py index 6055c3dcbd2..5b37c932232 100644 --- a/test/unit/tool_util/test_tool_linters.py +++ b/test/unit/tool_util/test_tool_linters.py @@ -5,6 +5,7 @@ import pytest from galaxy.tool_util.lint import ( lint_tool_source_with, + lint_xml_with, LintContext, XMLLintMessageLine, XMLLintMessageXPath, @@ -22,6 +23,7 @@ from galaxy.tool_util.linters import ( ) from galaxy.tool_util.loader_directory import load_tool_sources_from_path from galaxy.tool_util.parser.xml import XmlToolSource +from galaxy.util import parse_xml from galaxy.util.xml_macros import load_with_references # TODO tests tool xml for general linter @@ -727,6 +729,17 @@ XML_ORDER = """ """ +TOOL_WITH_COMMENTS = """ + + + + + + + + +""" + @pytest.fixture() def lint_ctx(): @@ -746,6 +759,15 @@ def get_xml_tool_source(xml_string): return load_with_references(tool_path)[0] +def get_tool_xml_exact(xml_string): + """Returns the tool XML as it is, without stripping comments or anything else.""" + with tempfile.NamedTemporaryFile(mode="w", suffix="tool.xml") as tmp: + tmp.write(xml_string) + tmp.flush() + tool_path = tmp.name + return parse_xml(tool_path, strip_whitespace=False, remove_comments=False) + + def failed_assert_print(lint_ctx): return ( f"Valid: {lint_ctx.valid_messages}\n" @@ -1675,3 +1697,10 @@ def test_linting_cwl_tool(lint_ctx): assert len(lint_ctx.valid_messages) == 4 assert len(lint_ctx.warn_messages) == 2 assert not lint_ctx.error_messages + + +def test_xml_comments_are_ignored(lint_ctx: LintContext): + tool_xml = get_tool_xml_exact(TOOL_WITH_COMMENTS) + lint_xml_with(lint_ctx, tool_xml) + for lint_message in lint_ctx.message_list: + assert "Comment" not in lint_message.message From b9674d678bbd0a5be25070bed8ba9aadb88ffc5b Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 24 Oct 2022 15:01:43 +0200 Subject: [PATCH 03/13] Fix skip XML comments when linting --- lib/galaxy/tool_util/linters/outputs.py | 7 ++++++- lib/galaxy/tool_util/linters/stdio.py | 3 +++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/linters/outputs.py b/lib/galaxy/tool_util/linters/outputs.py index 87bacb3c77a..7d244d7a690 100644 --- a/lib/galaxy/tool_util/linters/outputs.py +++ b/lib/galaxy/tool_util/linters/outputs.py @@ -1,5 +1,8 @@ """This module contains a linting functions for tool outputs.""" -from galaxy.util import string_as_bool +from galaxy.util import ( + etree, + string_as_bool, +) from ._util import is_valid_cheetah_placeholder from ..parser.output_collection_def import NAMED_PATTERNS @@ -20,6 +23,8 @@ def lint_output(tool_xml, lint_ctx): labels = set() names = set() for output in list(outputs[0]): + if output.tag is etree.Comment: + continue if output.tag not in ["data", "collection"]: lint_ctx.warn(f"Unknown element found in outputs [{output.tag}]", node=output) continue diff --git a/lib/galaxy/tool_util/linters/stdio.py b/lib/galaxy/tool_util/linters/stdio.py index de769e824ed..981882bbde3 100644 --- a/lib/galaxy/tool_util/linters/stdio.py +++ b/lib/galaxy/tool_util/linters/stdio.py @@ -1,6 +1,7 @@ """This module contains a linting functions for tool error detection.""" import re +from galaxy.util import etree from .command import get_command @@ -37,6 +38,8 @@ def lint_stdio(tool_source, lint_ctx): stdio = stdios[0] for child in list(stdio): + if child.tag is etree.Comment: + continue if child.tag == "regex": _lint_regex(tool_xml, child, lint_ctx) elif child.tag == "exit_code": From af30b84081c4cc3667a0cdf2c4bae7ef0d07275e Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 25 Oct 2022 17:46:55 +0200 Subject: [PATCH 04/13] Fix invocation filtering Fixes https://github.com/galaxyproject/galaxy/issues/14851 --- client/src/components/providers/StoredWorkflowsProvider.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/components/providers/StoredWorkflowsProvider.js b/client/src/components/providers/StoredWorkflowsProvider.js index fdcbe91e56b..6740099f9f1 100644 --- a/client/src/components/providers/StoredWorkflowsProvider.js +++ b/client/src/components/providers/StoredWorkflowsProvider.js @@ -21,7 +21,7 @@ export function storedWorkflowsProvider(ctx, callback, extraParams = {}) { } async function storedWorkflowDetails({ storedWorkflowId }) { - const url = `${getAppRoot()}api/workflows/${storedWorkflowId}?instance=true`; + const url = `${getAppRoot()}api/workflows/${storedWorkflowId}`; try { const { data } = await axios.get(url); return data; From 0ed66646e1234a93193edbd4b5577c535ed32f99 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 25 Oct 2022 19:18:11 +0100 Subject: [PATCH 05/13] Rewrite `CondaContext._guess_conda_properties()` to use `conda info` --- lib/galaxy/tools/deps/conda_util.py | 42 +++++++---------------------- 1 file changed, 10 insertions(+), 32 deletions(-) diff --git a/lib/galaxy/tools/deps/conda_util.py b/lib/galaxy/tools/deps/conda_util.py index d6d6f626fe6..6b4919fa8b6 100644 --- a/lib/galaxy/tools/deps/conda_util.py +++ b/lib/galaxy/tools/deps/conda_util.py @@ -30,8 +30,6 @@ SHELL_UNSAFE_PATTERN = re.compile(r"[\s\"']") IS_OS_X = sys.platform == "darwin" -# BSD 3-clause -CONDA_LICENSE = "http://docs.continuum.io/anaconda/eula" VERSIONED_ENV_DIR_NAME = re.compile(r"__(.*)@(.*)") UNVERSIONED_ENV_DIR_NAME = re.compile(r"__(.*)@_uv_") USE_PATH_EXEC_DEFAULT = False @@ -103,7 +101,6 @@ class CondaContext(installable.InstallableContext): ensure_channels = None self.ensure_channels = ensure_channels self._conda_version = None - self._miniconda_version = None self._conda_build_available = None self.use_local = use_local @@ -120,35 +117,16 @@ class CondaContext(installable.InstallableContext): return self._conda_build_available def _guess_conda_properties(self): - conda_meta_path = self._conda_meta_path - # Perhaps we should call "conda info --json" and parse it but for now we are going - # to assume the default. - conda_version = packaging.version.parse(CONDA_VERSION) - conda_build_available = False - miniconda_version = "3" - - if os.path.exists(conda_meta_path): - for package in os.listdir(conda_meta_path): - package_parts = package.split("-") - if len(package_parts) < 3: - continue - package = '-'.join(package_parts[:-2]) - version = package_parts[-2] - # build = package_parts[-1] - if package == "conda": - conda_version = packaging.version.parse(version) - if package == "python" and version.startswith("2"): - miniconda_version = "2" - if package == "conda-build": - conda_build_available = True - - self._conda_version = conda_version - self._miniconda_version = miniconda_version - self._conda_build_available = conda_build_available - - @property - def _conda_meta_path(self): - return os.path.join(self.conda_prefix, "conda-meta") + info = self.conda_info() + self._conda_version = packaging.version.parse(info["conda_version"]) + self._conda_build_available = False + conda_build_version = info.get("conda_build_version") + if conda_build_version != "not installed": + try: + self._conda_version = packaging.version.parse(conda_build_version) + self._conda_build_available = True + except Exception: + pass @property def _override_channels_args(self): From 32e3287dbd05a836b40385565c0d3b0c889f9946 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 25 Oct 2022 19:21:41 +0100 Subject: [PATCH 06/13] Require also a recent pyOpenSSL package when installing conda Fix https://github.com/galaxyproject/galaxy/issues/14848 . --- lib/galaxy/tools/deps/conda_util.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/tools/deps/conda_util.py b/lib/galaxy/tools/deps/conda_util.py index 6b4919fa8b6..780eacd18ca 100644 --- a/lib/galaxy/tools/deps/conda_util.py +++ b/lib/galaxy/tools/deps/conda_util.py @@ -33,7 +33,7 @@ IS_OS_X = sys.platform == "darwin" VERSIONED_ENV_DIR_NAME = re.compile(r"__(.*)@(.*)") UNVERSIONED_ENV_DIR_NAME = re.compile(r"__(.*)@_uv_") USE_PATH_EXEC_DEFAULT = False -CONDA_VERSION = "4.6.14" +CONDA_PACKAGE_SPECS = ("conda=4.6.14", "'pyopenssl>=22.1.0'") CONDA_BUILD_VERSION = "3.17.8" USE_LOCAL_DEFAULT = False @@ -409,9 +409,7 @@ def install_conda(conda_context, force_conda_build=False): os.close(f) download_cmd = commands.download_command(conda_link(), to=script_path, quote_url=False) install_cmd = ['bash', script_path, '-b', '-p', conda_context.conda_prefix] - package_targets = [ - "conda=%s" % CONDA_VERSION, - ] + package_targets = list(CONDA_PACKAGE_SPECS) if force_conda_build or conda_context.use_local: package_targets.append("conda-build=%s" % CONDA_BUILD_VERSION) log.info("Installing conda, this may take several minutes.") From bb000b2403e51b2ecf2a4dfe9d2ac560b333a437 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 25 Oct 2022 19:28:57 +0100 Subject: [PATCH 07/13] Add `--strict-channel-priority` to conda create/install commands Fix https://github.com/galaxyproject/galaxy/issues/12790 . This is added only if using conda >=4.7.5 , which includes this bug fix: https://github.com/conda/conda/pull/8819 --- lib/galaxy/tools/deps/conda_util.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/tools/deps/conda_util.py b/lib/galaxy/tools/deps/conda_util.py index 780eacd18ca..580437bd7a8 100644 --- a/lib/galaxy/tools/deps/conda_util.py +++ b/lib/galaxy/tools/deps/conda_util.py @@ -224,15 +224,14 @@ class CondaContext(installable.InstallableContext): """ Return the process exit code (i.e. 0 in case of success). """ - create_base_args = [ - "-y", - "--quiet" - ] + create_args = ["-y", "--quiet"] + if self.conda_version >= packaging.version.parse("4.7.5"): + create_args.append("--strict-channel-priority") if allow_local and self.use_local: - create_base_args.extend(["--use-local"]) - create_base_args.extend(self._override_channels_args) - create_base_args.extend(args) - return self.exec_command("create", create_base_args, stdout_path=stdout_path) + create_args.extend(["--use-local"]) + create_args.extend(self._override_channels_args) + create_args.extend(args) + return self.exec_command("create", create_args, stdout_path=stdout_path) def exec_remove(self, args): """ @@ -252,14 +251,14 @@ class CondaContext(installable.InstallableContext): """ Return the process exit code (i.e. 0 in case of success). """ - install_base_args = [ - "-y" - ] + install_args = ["-y"] + if self.conda_version >= packaging.version.parse("4.7.5"): + install_args.append("--strict-channel-priority") if allow_local and self.use_local: - install_base_args.append("--use-local") - install_base_args.extend(self._override_channels_args) - install_base_args.extend(args) - return self.exec_command("install", install_base_args, stdout_path=stdout_path) + install_args.append("--use-local") + install_args.extend(self._override_channels_args) + install_args.extend(args) + return self.exec_command("install", install_args, stdout_path=stdout_path) def exec_clean(self, args=[], quiet=False): """ From 1076eadc4cd30c353f611dd5d1b8936ea581765f Mon Sep 17 00:00:00 2001 From: cat-bro Date: Mon, 17 Oct 2022 17:14:49 +1100 Subject: [PATCH 08/13] Add breaking tests for container_resolvers index and show --- lib/galaxy_test/api/test_container_resolvers.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 lib/galaxy_test/api/test_container_resolvers.py diff --git a/lib/galaxy_test/api/test_container_resolvers.py b/lib/galaxy_test/api/test_container_resolvers.py new file mode 100644 index 00000000000..be1f656305a --- /dev/null +++ b/lib/galaxy_test/api/test_container_resolvers.py @@ -0,0 +1,13 @@ +from ._framework import ApiTestCase + +class ContainerResolversApiTestCase(ApiTestCase): + + def test_index(self): + response = self._get("container_resolvers", admin=True) + assert response.status_code == 200 + assert isinstance(response.json(), list) + + def test_show(self): + response = self._get("container_resolvers/0", admin=True) + assert response.status_code == 200 + assert isinstance(response.json(), dict) From 13ef751f90c3cccc4fdbffc44c3d28dcd83d5262 Mon Sep 17 00:00:00 2001 From: cat-bro Date: Mon, 17 Oct 2022 19:41:40 +1100 Subject: [PATCH 09/13] lint --- lib/galaxy_test/api/test_container_resolvers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy_test/api/test_container_resolvers.py b/lib/galaxy_test/api/test_container_resolvers.py index be1f656305a..0b20b7387fe 100644 --- a/lib/galaxy_test/api/test_container_resolvers.py +++ b/lib/galaxy_test/api/test_container_resolvers.py @@ -1,5 +1,6 @@ from ._framework import ApiTestCase + class ContainerResolversApiTestCase(ApiTestCase): def test_index(self): @@ -8,6 +9,6 @@ class ContainerResolversApiTestCase(ApiTestCase): assert isinstance(response.json(), list) def test_show(self): - response = self._get("container_resolvers/0", admin=True) + response = self._get("container_resolvers/0", admin=True) assert response.status_code == 200 assert isinstance(response.json(), dict) From 460abaf8e38b8a3902aa19d53d2573dfb8ae3467 Mon Sep 17 00:00:00 2001 From: cat-bro Date: Mon, 17 Oct 2022 19:46:35 +1100 Subject: [PATCH 10/13] update lib/galaxy/webapps/galaxy/api/container_resolution.py and lib/galaxy/tool_util/deps/views.py --- lib/galaxy/tool_util/deps/views.py | 2 +- lib/galaxy/webapps/galaxy/api/container_resolution.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/tool_util/deps/views.py b/lib/galaxy/tool_util/deps/views.py index 5340adc181b..9bd1fccbd60 100644 --- a/lib/galaxy/tool_util/deps/views.py +++ b/lib/galaxy/tool_util/deps/views.py @@ -403,7 +403,7 @@ class ContainerResolutionView: @property def _container_resolvers(self): - return self._app.container_finder.container_resolvers + return self._app.container_finder.default_container_registry.container_resolvers def _container_resolver(self, index): index = int(index) diff --git a/lib/galaxy/webapps/galaxy/api/container_resolution.py b/lib/galaxy/webapps/galaxy/api/container_resolution.py index 07b9a9edaa3..b99f427a7f9 100644 --- a/lib/galaxy/webapps/galaxy/api/container_resolution.py +++ b/lib/galaxy/webapps/galaxy/api/container_resolution.py @@ -31,11 +31,11 @@ class ContainerResolutionAPIController(BaseGalaxyAPIController): @expose_api @require_admin - def show(self, trans, id): + def show(self, trans, index): """ GET /api/container_resolvers/ """ - return self._view.show(id) + return self._view.show(index) @expose_api @require_admin From 850b8fdcee1dd2ad360ebd7342cb76e6bc2132bc Mon Sep 17 00:00:00 2001 From: cat-bro Date: Tue, 18 Oct 2022 16:40:13 +1100 Subject: [PATCH 11/13] format and change name of class and file --- ...est_container_resolvers.py => test_container_resolution.py} | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) rename lib/galaxy_test/api/{test_container_resolvers.py => test_container_resolution.py} (88%) diff --git a/lib/galaxy_test/api/test_container_resolvers.py b/lib/galaxy_test/api/test_container_resolution.py similarity index 88% rename from lib/galaxy_test/api/test_container_resolvers.py rename to lib/galaxy_test/api/test_container_resolution.py index 0b20b7387fe..37a22618aff 100644 --- a/lib/galaxy_test/api/test_container_resolvers.py +++ b/lib/galaxy_test/api/test_container_resolution.py @@ -1,8 +1,7 @@ from ._framework import ApiTestCase -class ContainerResolversApiTestCase(ApiTestCase): - +class ContainerResolutionApiTestCase(ApiTestCase): def test_index(self): response = self._get("container_resolvers", admin=True) assert response.status_code == 200 From 792935178fc16eb642cb001c53e4eddba3983db8 Mon Sep 17 00:00:00 2001 From: cat-bro Date: Wed, 26 Oct 2022 14:45:10 +1100 Subject: [PATCH 12/13] add test for /api/container_resolvers/resolve --- lib/galaxy_test/api/test_container_resolution.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/galaxy_test/api/test_container_resolution.py b/lib/galaxy_test/api/test_container_resolution.py index 37a22618aff..81e6a6c631b 100644 --- a/lib/galaxy_test/api/test_container_resolution.py +++ b/lib/galaxy_test/api/test_container_resolution.py @@ -1,3 +1,4 @@ +from galaxy_test.base.populators import skip_without_tool from ._framework import ApiTestCase @@ -11,3 +12,15 @@ class ContainerResolutionApiTestCase(ApiTestCase): response = self._get("container_resolvers/0", admin=True) assert response.status_code == 200 assert isinstance(response.json(), dict) + + @skip_without_tool("cat1") + def test_resolve(self): + tool_id = "cat1" + + # no index + response = self._get(f"container_resolvers/resolve?tool_id={tool_id}", admin=True) + assert response.status_code == 200 + + # with index + response = self._get(f"container_resolvers/0/resolve?tool_id={tool_id}", admin=True) + assert response.status_code == 200 From 39d3dc96f90c832c8379412b43ce8bb223efe44f Mon Sep 17 00:00:00 2001 From: cat-bro Date: Wed, 26 Oct 2022 14:45:53 +1100 Subject: [PATCH 13/13] update lib/galaxy/tool_util/deps/views.py --- lib/galaxy/tool_util/deps/views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/tool_util/deps/views.py b/lib/galaxy/tool_util/deps/views.py index 9bd1fccbd60..885cebdbbfc 100644 --- a/lib/galaxy/tool_util/deps/views.py +++ b/lib/galaxy/tool_util/deps/views.py @@ -340,7 +340,7 @@ class ContainerResolutionView: def show(self, index): return self._container_resolver(index).to_dict() - def resolve(self, **kwds): + def resolve(self, index=None, **kwds): find_best_kwds = { "install": False, "enabled_container_types": ["docker", "singularity"], @@ -348,8 +348,8 @@ class ContainerResolutionView: "session": kwds.get("session"), } - if "index" in kwds: - find_best_kwds["index"] = int(kwds["index"]) + if index is not None: + find_best_kwds["index"] = int(index) if "container_type" in kwds: find_best_kwds["enabled_container_types"] = [kwds["container_type"]] if "resolver_type" in kwds: