From c27a01e2936af5faeace539afb27f3103962d114 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 18 Nov 2020 19:35:24 +0100 Subject: [PATCH 01/10] Include dataset and state in valid state exception --- lib/galaxy/tools/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 3ce0fe07254..06057173d7b 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -2773,7 +2773,7 @@ class DatabaseOperationTool(Tool): if self.require_dataset_ok: if input_dataset.state != input_dataset.dataset.states.OK: - raise ValueError("Tool requires inputs to be in valid state.") + raise ValueError("Tool requires inputs to be in valid state, but {} is in state '{}".format(input_dataset, input_dataset.state)) for input_dataset in input_datasets.values(): check_dataset_instance(input_dataset) From 40e20a40f1a0c0307cb81a08483599ed4c2bf41c Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Wed, 18 Nov 2020 19:43:50 +0100 Subject: [PATCH 02/10] Better exception message Co-authored-by: Nicola Soranzo --- lib/galaxy/tools/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 06057173d7b..1ab8813d2fa 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -2773,7 +2773,7 @@ class DatabaseOperationTool(Tool): if self.require_dataset_ok: if input_dataset.state != input_dataset.dataset.states.OK: - raise ValueError("Tool requires inputs to be in valid state, but {} is in state '{}".format(input_dataset, input_dataset.state)) + raise ValueError("Tool requires inputs to be in valid state, but dataset {} is in state '{}'".format(input_dataset, input_dataset.state)) for input_dataset in input_datasets.values(): check_dataset_instance(input_dataset) From b533d9928354fd5e85a40d6275c8eac9a53b75fe Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 18 Nov 2020 14:25:49 -0500 Subject: [PATCH 03/10] Drop (no idea where these came from?) samples that don't exist from announce --- doc/source/releases/20.09_announce.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/doc/source/releases/20.09_announce.rst b/doc/source/releases/20.09_announce.rst index 5c766f2bc44..6881ecc1b25 100644 --- a/doc/source/releases/20.09_announce.rst +++ b/doc/source/releases/20.09_announce.rst @@ -54,11 +54,9 @@ The following configuration options have been changed New Configuration Files ----------------------- -The following files are new, or recently converted to yaml +The following configuration file is new: - ``config/trs_servers_conf.yml.sample`` -- ``config/build_mulled_singularity_mapping_file.yml.sample`` -- ``config/singularity_mapping.yml.sample`` Get Galaxy From 0a818afaac6586666220bdcabcc63257b3cf3ede Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 19 Nov 2020 15:42:14 +0100 Subject: [PATCH 04/10] Fix collectl plugin We can't subscript filter objects. --- lib/galaxy/job_metrics/instrumenters/collectl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/job_metrics/instrumenters/collectl.py b/lib/galaxy/job_metrics/instrumenters/collectl.py index 439031b266b..0ce2760bdb1 100644 --- a/lib/galaxy/job_metrics/instrumenters/collectl.py +++ b/lib/galaxy/job_metrics/instrumenters/collectl.py @@ -99,9 +99,9 @@ class CollectlPlugin(InstrumentPlugin): pid = open(self.__pid_file(job_directory)).read().strip() contents = os.listdir(job_directory) try: - rel_path = filter(self._is_instrumented_collectl_log, contents)[0] + rel_path = next(iter(filter(self._is_instrumented_collectl_log, contents))) path = os.path.join(job_directory, rel_path) - except IndexError: + except StopIteration: message = "Failed to find collectl log in directory {}, files were {}".format(job_directory, contents) raise Exception(message) From dccb02327994b6588b0e9a78bec3b998b22dda3f Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 19 Nov 2020 16:09:20 +0100 Subject: [PATCH 05/10] Speed up quay.io interaction by using requests session Not sure how much faster this is, but it goes from timeout to no timeout when I load up usegalaxy.org's toolbox. --- .../tool_util/deps/container_resolvers/mulled.py | 10 +++++----- lib/galaxy/tool_util/deps/containers.py | 4 ++-- .../tool_util/deps/mulled/mulled_build_channel.py | 5 ++++- lib/galaxy/tool_util/deps/mulled/util.py | 14 ++++++++------ lib/galaxy/tool_util/deps/views.py | 1 + .../webapps/galaxy/api/container_resolution.py | 5 +++++ 6 files changed, 25 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/tool_util/deps/container_resolvers/mulled.py b/lib/galaxy/tool_util/deps/container_resolvers/mulled.py index b0eede03fa7..6664d22899f 100644 --- a/lib/galaxy/tool_util/deps/container_resolvers/mulled.py +++ b/lib/galaxy/tool_util/deps/container_resolvers/mulled.py @@ -234,7 +234,7 @@ def singularity_cached_container_description(targets, cache_directory, hash_func return container -def targets_to_mulled_name(targets, hash_func, namespace, resolution_cache=None): +def targets_to_mulled_name(targets, hash_func, namespace, resolution_cache=None, session=None): unresolved_cache_key = "galaxy.tool_util.deps.container_resolvers.mulled:unresolved" if resolution_cache is not None: if unresolved_cache_key not in resolution_cache: @@ -265,7 +265,7 @@ def targets_to_mulled_name(targets, hash_func, namespace, resolution_cache=None) if name: return name - tags = mulled_tags_for(namespace, target.package_name, resolution_cache=resolution_cache) + tags = mulled_tags_for(namespace, target.package_name, resolution_cache=resolution_cache, session=session) if tags: for tag in tags: @@ -284,7 +284,7 @@ def targets_to_mulled_name(targets, hash_func, namespace, resolution_cache=None) else: repo_name = image_name tag_prefix = None - tags = mulled_tags_for(namespace, repo_name, tag_prefix=tag_prefix, resolution_cache=resolution_cache) + tags = mulled_tags_for(namespace, repo_name, tag_prefix=tag_prefix, resolution_cache=resolution_cache, session=session) return tags[0] if tags else None if hash_func == "v2": @@ -388,7 +388,7 @@ class MulledDockerContainerResolver(ContainerResolver): command = container.build_pull_command() shell(command) - def resolve(self, enabled_container_types, tool_info, install=False, **kwds): + def resolve(self, enabled_container_types, tool_info, install=False, session=None, **kwds): resolution_cache = kwds.get("resolution_cache") if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: return None @@ -397,7 +397,7 @@ class MulledDockerContainerResolver(ContainerResolver): if len(targets) == 0: return None - name = targets_to_mulled_name(targets=targets, hash_func=self.hash_func, namespace=self.namespace, resolution_cache=resolution_cache) + name = targets_to_mulled_name(targets=targets, hash_func=self.hash_func, namespace=self.namespace, resolution_cache=resolution_cache, session=session) if name: container_id = "quay.io/{}/{}".format(self.namespace, name) if self.protocol: diff --git a/lib/galaxy/tool_util/deps/containers.py b/lib/galaxy/tool_util/deps/containers.py index 84e0d627cab..d2d23202d82 100644 --- a/lib/galaxy/tool_util/deps/containers.py +++ b/lib/galaxy/tool_util/deps/containers.py @@ -243,7 +243,7 @@ class ContainerRegistry: return None return None if resolved_container_description is None else resolved_container_description.container_description - def resolve(self, enabled_container_types, tool_info, index=None, resolver_type=None, install=True, resolution_cache=None): + def resolve(self, enabled_container_types, tool_info, index=None, resolver_type=None, install=True, resolution_cache=None, session=None): resolution_cache = resolution_cache or self.mulled_resolution_cache for i, container_resolver in enumerate(self.container_resolvers): if index is not None and i != index: @@ -259,7 +259,7 @@ class ContainerRegistry: if not install and container_resolver.builds_on_resolution: continue - container_description = container_resolver.resolve(enabled_container_types, tool_info, resolution_cache=resolution_cache) + container_description = container_resolver.resolve(enabled_container_types, tool_info, resolution_cache=resolution_cache, session=session) log.info("Checking with container resolver [{}] found description [{}]".format(container_resolver, container_description)) if container_description: assert container_description.type in enabled_container_types diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_build_channel.py b/lib/galaxy/tool_util/deps/mulled/mulled_build_channel.py index 0f69eda76ad..6a37c94b79f 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_build_channel.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_build_channel.py @@ -22,6 +22,8 @@ import subprocess import sys import time +import requests + from ._cli import arg_parser from .mulled_build import ( add_build_arguments, @@ -58,6 +60,7 @@ def _new_versions(quay, conda): def run_channel(args, build_last_n_versions=1): """Build list of involucro commands (as shell snippet) to run.""" + session = requests.session() for pkg_name, pkg_tests in get_affected_packages(args): repo_data = _fetch_repo_data(args) c = conda_versions(pkg_name, repo_data) @@ -66,7 +69,7 @@ def run_channel(args, build_last_n_versions=1): if not args.force_rebuild: time.sleep(1) - q = quay_versions(args.namespace, pkg_name) + q = quay_versions(args.namespace, pkg_name, session) versions = _new_versions(q, c) else: versions = c diff --git a/lib/galaxy/tool_util/deps/mulled/util.py b/lib/galaxy/tool_util/deps/mulled/util.py index f50355282cc..8bb5262f981 100644 --- a/lib/galaxy/tool_util/deps/mulled/util.py +++ b/lib/galaxy/tool_util/deps/mulled/util.py @@ -32,9 +32,9 @@ def create_repository(namespace, repo_name, oauth_token): requests.post("https://quay.io/api/v1/repository", json=data, headers=headers, timeout=QUAY_IO_TIMEOUT) -def quay_versions(namespace, pkg_name): +def quay_versions(namespace, pkg_name, session=None): """Get all version tags for a Docker image stored on quay.io for supplied package name.""" - data = quay_repository(namespace, pkg_name) + data = quay_repository(namespace, pkg_name, session=session) if 'error_type' in data and data['error_type'] == "invalid_token": return [] @@ -45,11 +45,13 @@ def quay_versions(namespace, pkg_name): return [tag for tag in data['tags'].keys() if tag != 'latest'] -def quay_repository(namespace, pkg_name): +def quay_repository(namespace, pkg_name, session=None): assert namespace is not None assert pkg_name is not None url = 'https://quay.io/api/v1/repository/{}/{}'.format(namespace, pkg_name) - response = requests.get(url, timeout=QUAY_IO_TIMEOUT) + if not session: + session = requests.session() + response = session.get(url, timeout=QUAY_IO_TIMEOUT) data = response.json() return data @@ -74,7 +76,7 @@ def _namespace_has_repo_name(namespace, repo_name, resolution_cache): return repo_name in repo_names -def mulled_tags_for(namespace, image, tag_prefix=None, resolution_cache=None): +def mulled_tags_for(namespace, image, tag_prefix=None, resolution_cache=None, session=None): """Fetch remote tags available for supplied image name. The result will be sorted so newest tags are first. @@ -101,7 +103,7 @@ def mulled_tags_for(namespace, image, tag_prefix=None, resolution_cache=None): tags_cached = True if not tags_cached: - tags = quay_versions(namespace, image) + tags = quay_versions(namespace, image, session) tag_cache[namespace][image] = tags if tag_prefix is not None: diff --git a/lib/galaxy/tool_util/deps/views.py b/lib/galaxy/tool_util/deps/views.py index b5d785ea3bc..985103d159a 100644 --- a/lib/galaxy/tool_util/deps/views.py +++ b/lib/galaxy/tool_util/deps/views.py @@ -333,6 +333,7 @@ class ContainerResolutionView: 'install': False, 'enabled_container_types': ['docker', 'singularity'], 'resolution_cache': kwds.get("resolution_cache"), + 'session': kwds.get('session'), } if 'index' in kwds: diff --git a/lib/galaxy/webapps/galaxy/api/container_resolution.py b/lib/galaxy/webapps/galaxy/api/container_resolution.py index 5ac3fe202c1..d545c40cecf 100644 --- a/lib/galaxy/webapps/galaxy/api/container_resolution.py +++ b/lib/galaxy/webapps/galaxy/api/container_resolution.py @@ -3,6 +3,8 @@ API operations allowing clients to manage container resolution. """ import logging +import requests + from galaxy.tool_util.deps import views from galaxy.web import ( expose_api, @@ -62,6 +64,7 @@ class ContainerResolutionAPIController(BaseAPIController): :returns: a dictified description of the container dependency, with attribute ``dependency_type: None`` if no match was found. """ + kwds['session'] = requests.session() return self._view.resolve(index=index, **kwds) @expose_api @@ -81,6 +84,7 @@ class ContainerResolutionAPIController(BaseAPIController): :rtype: list :returns: list of items returned from resolve() """ + kwds['session'] = requests.session() return self._view.resolve_toolbox(**kwds) @expose_api @@ -99,6 +103,7 @@ class ContainerResolutionAPIController(BaseAPIController): """ kwds.update(payload) kwds["install"] = True + kwds['session'] = requests.session() return self._view.resolve_toolbox(**kwds) @expose_api From c0de1c67b6cd1ced3ca93ed0fb2eb5719a87b2c0 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 4 Nov 2020 21:12:24 +0100 Subject: [PATCH 06/10] omit default bind type for singularity --- lib/galaxy/tool_util/deps/container_classes.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/deps/container_classes.py b/lib/galaxy/tool_util/deps/container_classes.py index 29dcaf766ff..c7c86e4e2a1 100644 --- a/lib/galaxy/tool_util/deps/container_classes.py +++ b/lib/galaxy/tool_util/deps/container_classes.py @@ -125,8 +125,10 @@ def preprocess_volumes(volumes_raw_str, container_type): ['/a/b:/a:ro', '/a/b/c:/a/b:rw'] >>> preprocess_volumes("/a/b:default_ro,/a/b/c:rw", DOCKER_CONTAINER_TYPE) ['/a/b:ro', '/a/b/c:rw'] + >>> preprocess_volumes("/a/b:default_ro,/a/b/c:ro", SINGULARITY_CONTAINER_TYPE) + ['/a/b:ro', '/a/b/c:ro'] >>> preprocess_volumes("/a/b:default_ro,/a/b/c:rw", SINGULARITY_CONTAINER_TYPE) - ['/a/b:rw', '/a/b/c:rw'] + ['/a/b', '/a/b/c'] """ volumes_raw_strs = [v.strip() for v in volumes_raw_str.split(",")] @@ -160,6 +162,12 @@ def preprocess_volumes(volumes_raw_str, container_type): volume[1] = how + # for a while singularity did not allow to specify the bind type rw + # (which is the default). so we omit this default + # see https://github.com/hpcng/singularity/pull/5487 + if container_type == SINGULARITY_CONTAINER_TYPE and volume[1] == 'rw': + del volume[1] + return [":".join(v) for v in volumes] From 84ddc63937297a8e0e8163606d16f2c83e49cb72 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 19 Nov 2020 17:02:30 +0100 Subject: [PATCH 07/10] Fix signature of singularity cached_container_description --- lib/galaxy/tool_util/deps/container_resolvers/mulled.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/deps/container_resolvers/mulled.py b/lib/galaxy/tool_util/deps/container_resolvers/mulled.py index b0eede03fa7..4b4e55aa855 100644 --- a/lib/galaxy/tool_util/deps/container_resolvers/mulled.py +++ b/lib/galaxy/tool_util/deps/container_resolvers/mulled.py @@ -447,7 +447,7 @@ class MulledSingularityContainerResolver(MulledDockerContainerResolver): self.hash_func = hash_func self.auto_install = string_as_bool(auto_install) - def cached_container_description(self, targets, namespace, hash_func): + def cached_container_description(self, targets, namespace, hash_func, resolution_cache): return singularity_cached_container_description(targets, cache_directory=self.cache_directory, hash_func=hash_func) From 0b30e75aa60fa422897fdbe5458af529e34afc79 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 19 Nov 2020 16:53:30 +0100 Subject: [PATCH 08/10] Fix selection of tool ids in container resolver admin view --- .../components/admin/Dependencies/ContainerIndex.vue | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/client/src/components/admin/Dependencies/ContainerIndex.vue b/client/src/components/admin/Dependencies/ContainerIndex.vue index 26630ffd831..b87d158bd10 100644 --- a/client/src/components/admin/Dependencies/ContainerIndex.vue +++ b/client/src/components/admin/Dependencies/ContainerIndex.vue @@ -170,16 +170,7 @@ export default { return params; }, selectedToolIds() { - const toolIds = []; - for (const item of this.items) { - if (item["selected"]) { - const toolId = item["tool"]; - if (toolId) { - toolIds.push(toolId[0]); - } - } - } - return toolIds; + return this.items.filter((item) => item.selected).map((item) => item.tool_id); }, }, watch: { From 833ef38c59be2174cadb3282d43e78196618fe74 Mon Sep 17 00:00:00 2001 From: tmcgowan Date: Fri, 20 Nov 2020 15:23:18 -0600 Subject: [PATCH 09/10] fix reference error --- config/plugins/visualizations/mvpapp/src/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/plugins/visualizations/mvpapp/src/index.js b/config/plugins/visualizations/mvpapp/src/index.js index 8c1424c1e18..616efb27d18 100644 --- a/config/plugins/visualizations/mvpapp/src/index.js +++ b/config/plugins/visualizations/mvpapp/src/index.js @@ -1971,7 +1971,7 @@ var PeptideView = (function (pv) { option.scoreSummary = true; - psmDetailDP = new AjaxDataProvider(option); + let psmDetailDP = new AjaxDataProvider(option); psmDetailDP.generateTable(); //Move to table $("html, body").animate( From b0b44287d866f9480a0ead7dc5a1dc1ba6707319 Mon Sep 17 00:00:00 2001 From: Oleg Zharkov Date: Fri, 20 Nov 2020 18:10:04 +0100 Subject: [PATCH 10/10] fix image as path --- .../Dataset/DatasetAsImage/DatasetAsImage.vue | 12 ++++++++---- .../Markdown/Elements/HistoryDatasetAsImage.vue | 10 ++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/client/src/components/Dataset/DatasetAsImage/DatasetAsImage.vue b/client/src/components/Dataset/DatasetAsImage/DatasetAsImage.vue index 1b7e19e201c..bf1ac33c718 100644 --- a/client/src/components/Dataset/DatasetAsImage/DatasetAsImage.vue +++ b/client/src/components/Dataset/DatasetAsImage/DatasetAsImage.vue @@ -1,6 +1,10 @@