diff --git a/client/galaxy/scripts/mvc/history/history-contents.js b/client/galaxy/scripts/mvc/history/history-contents.js index a0049811ff6..54eb7757e80 100644 --- a/client/galaxy/scripts/mvc/history/history-contents.js +++ b/client/galaxy/scripts/mvc/history/history-contents.js @@ -442,7 +442,8 @@ export var HistoryContents = _super.extend(BASE_MVC.LoggableMixin).extend({ data: JSON.stringify({ content: id, source: contentType, - type: type + type: type, + copy_elements: true }) }) .done(response => { diff --git a/client/galaxy/scripts/mvc/ui/ui-slider.js b/client/galaxy/scripts/mvc/ui/ui-slider.js index 6ce0d64ba94..d46dd2ce08e 100644 --- a/client/galaxy/scripts/mvc/ui/ui-slider.js +++ b/client/galaxy/scripts/mvc/ui/ui-slider.js @@ -47,6 +47,7 @@ var View = Backbone.View.extend({ v == 13 || v == 37 || v == 39 || + v == 69 || (v >= 48 && v <= 57) || (v >= 96 && v <= 105) || ((v == 190 || v == 110) && diff --git a/client/galaxy/scripts/mvc/workflow/workflow-view.js b/client/galaxy/scripts/mvc/workflow/workflow-view.js index 23875f61489..50465e6963f 100644 --- a/client/galaxy/scripts/mvc/workflow/workflow-view.js +++ b/client/galaxy/scripts/mvc/workflow/workflow-view.js @@ -258,7 +258,7 @@ export default Backbone.View.extend({ for (let i = 0; i < workflow_versions.length; i++) { const current_wf = workflow_versions[i]; - let version_text = `Version ${current_wf["version"]}, ${current_wf["steps"]} steps`; + let version_text = `Version ${current_wf.version}, ${current_wf.steps} steps`; let selected = false; if (i == self.workflow.workflow_version) { version_text = `${version_text} (active)`; @@ -287,7 +287,7 @@ export default Backbone.View.extend({ $("#workflow-version-switch").unbind("change"); if (this.value != self.workflow.workflow_version) { if (self.workflow && self.workflow.has_changes) { - const r = confirm("There are unsaved changes to your workflow which will be lost. Continue ?"); + const r = window.confirm("There are unsaved changes to your workflow which will be lost. Continue ?"); if (r == false) { // We rebuild the version select list, to reset the selected version self.build_version_select(); @@ -774,6 +774,7 @@ export default Backbone.View.extend({ // Fix width to computed width // Now add floats var buttons = $("
"); + buttons.append($(`
Node Details
`)); if (type !== "subworkflow") { buttons.append( $("
") diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index 5d05ea3e7a9..ea41b6dc6ff 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -36,7 +36,7 @@ cachetools==3.1.0 certifi==2019.3.9 cffi==1.12.3 chardet==3.0.4 -cheetah3==3.2.1 +cheetah3==3.2.2 cliff==2.14.1 cloudauthz==0.2.0 cloudbridge==2.0.0 diff --git a/lib/galaxy/jobs/runners/slurm.py b/lib/galaxy/jobs/runners/slurm.py index 333fd55d563..ec2ce131d45 100644 --- a/lib/galaxy/jobs/runners/slurm.py +++ b/lib/galaxy/jobs/runners/slurm.py @@ -28,13 +28,9 @@ SLURM_MEMORY_LIMIT_EXCEEDED_MSG = 'slurmstepd: error: Exceeded job memory limit' SLURM_MEMORY_LIMIT_EXCEEDED_PARTIAL_WARNINGS = [': Exceeded job memory limit at some point.', ': Exceeded step memory limit at some point.'] SLURM_MEMORY_LIMIT_SCAN_SIZE = 16 * 1024 * 1024 # 16MB -SLURM_UNABLE_TO_ADD_TASK_TO_MEMORY_CG_MSG_RE = re.compile(r"""slurmstepd: error: task/cgroup: unable to add task\[pid=\d+\] to memory cg '\(null\)'$""") -SLURM_UNABLE_TO_CREATE_CGROUP_MSG_RE = re.compile(r"""slurmstepd: error: xcgroup_instantiate: unable to create cgroup '[^']+' : No space left on device$""") -SLURM_UNABLE_TO_INSTANCIATE_CGROUP_MSG_RE = re.compile(r"""slurmstepd: error: jobacct_gather/cgroup: unable to instanciate (job|user) \d+ memory cgroup$""") +SLURM_CGROUP_RE = re.compile(r"""slurmstepd: .*cgroup.*$""") SLURM_TOP_WARNING_RES = ( - SLURM_UNABLE_TO_ADD_TASK_TO_MEMORY_CG_MSG_RE, - SLURM_UNABLE_TO_CREATE_CGROUP_MSG_RE, - SLURM_UNABLE_TO_INSTANCIATE_CGROUP_MSG_RE + SLURM_CGROUP_RE, ) # These messages are returned to the user diff --git a/lib/galaxy/objectstore/s3_multipart_upload.py b/lib/galaxy/objectstore/s3_multipart_upload.py index f564c1f8524..64b15ce76f9 100644 --- a/lib/galaxy/objectstore/s3_multipart_upload.py +++ b/lib/galaxy/objectstore/s3_multipart_upload.py @@ -5,13 +5,10 @@ This parallelizes the task over available cores using multiprocessing. Code mostly taken form CloudBioLinux. """ -import contextlib -import functools import glob -import multiprocessing import os import subprocess -from multiprocessing.pool import IMapIterator +import threading try: import boto @@ -20,13 +17,6 @@ except ImportError: boto = None -def map_wrap(f): - @functools.wraps(f) - def wrapper(args): - return f(*args) - return wrapper - - def mp_from_ids(s3server, mp_id, mp_keyname, mp_bucketname): """Get the multipart upload from the bucket and multipart IDs. @@ -51,7 +41,6 @@ def mp_from_ids(s3server, mp_id, mp_keyname, mp_bucketname): return mp -@map_wrap def transfer_part(s3server, mp_id, mp_keyname, mp_bucketname, i, part): """Transfer a part of a multipart upload. Designed to be run in parallel. """ @@ -64,8 +53,6 @@ def transfer_part(s3server, mp_id, mp_keyname, mp_bucketname, i, part): def multipart_upload(s3server, bucket, s3_key_name, tarball, mb_size): """Upload large files using Amazon's multipart upload functionality. """ - cores = multiprocessing.cpu_count() - def split_file(in_file, mb_size, split_num=5): prefix = os.path.join(os.path.dirname(in_file), "%sS3PART" % (os.path.basename(s3_key_name))) @@ -80,29 +67,11 @@ def multipart_upload(s3server, bucket, s3_key_name, tarball, mb_size): mp = bucket.initiate_multipart_upload(s3_key_name, reduced_redundancy=s3server['use_rr']) - with multimap(cores) as pmap: - for _ in pmap(transfer_part, ((s3server, mp.id, mp.key_name, mp.bucket_name, i, part) - for (i, part) in - enumerate(split_file(tarball, mb_size, cores)))): - pass + for (i, part) in enumerate(split_file(tarball, mb_size)): + t = threading.Thread( + target=transfer_part, + args=(s3server, mp.id, mp.key_name, mp.bucket_name, i, part)) + t.start() + t.join() + mp.complete_upload() - - -@contextlib.contextmanager -def multimap(cores=None): - """Provide multiprocessing imap like function. - - The context manager handles setting up the pool, worked around interrupt issues - and terminating the pool on completion. - """ - if cores is None: - cores = max(multiprocessing.cpu_count() - 1, 1) - - def wrapper(func): - def wrap(self, timeout=None): - return func(self, timeout=timeout if timeout is not None else 1e100) - return wrap - IMapIterator.next = wrapper(IMapIterator.next) - pool = multiprocessing.Pool(cores) - yield pool.imap - pool.terminate() diff --git a/lib/galaxy/tools/deps/containers.py b/lib/galaxy/tools/deps/containers.py index 5a8234f84db..35229b4e631 100644 --- a/lib/galaxy/tools/deps/containers.py +++ b/lib/galaxy/tools/deps/containers.py @@ -206,7 +206,9 @@ class ContainerRegistry(object): if self.enable_beta_mulled_containers: default_resolvers.extend([ CachedMulledDockerContainerResolver(self.app_info, namespace="biocontainers"), + CachedMulledDockerContainerResolver(self.app_info, namespace="local"), CachedMulledSingularityContainerResolver(self.app_info, namespace="biocontainers"), + CachedMulledSingularityContainerResolver(self.app_info, namespace="local"), MulledDockerContainerResolver(self.app_info, namespace="biocontainers"), MulledSingularityContainerResolver(self.app_info, namespace="biocontainers"), BuildMulledDockerContainerResolver(self.app_info), diff --git a/lib/galaxy/web/framework/webapp.py b/lib/galaxy/web/framework/webapp.py index f64e0a1d05b..88ca95eb8df 100644 --- a/lib/galaxy/web/framework/webapp.py +++ b/lib/galaxy/web/framework/webapp.py @@ -650,24 +650,27 @@ class GalaxyWebTransaction(base.DefaultWebTransaction, Associate the user's last accessed history (if exists) with their new session """ history = None + set_permissions = False try: users_last_session = user.galaxy_sessions[0] - last_accessed = True except Exception: users_last_session = None - last_accessed = False if (prev_galaxy_session and prev_galaxy_session.current_history and not prev_galaxy_session.current_history.deleted and - prev_galaxy_session.current_history.datasets): - if prev_galaxy_session.current_history.user is None or prev_galaxy_session.current_history.user == user: - # If the previous galaxy session had a history, associate it with the new - # session, but only if it didn't belong to a different user. - history = prev_galaxy_session.current_history - if prev_galaxy_session.user is None: - # Increase the user's disk usage by the amount of the previous history's datasets if they didn't already own it. - for hda in history.datasets: - user.adjust_total_disk_usage(hda.quota_amount(user)) + prev_galaxy_session.current_history.datasets and + (prev_galaxy_session.current_history.user is None or + prev_galaxy_session.current_history.user == user)): + # If the previous galaxy session had a history, associate it with the new session, but only if it didn't + # belong to a different user. + history = prev_galaxy_session.current_history + if prev_galaxy_session.user is None: + # Increase the user's disk usage by the amount of the previous history's datasets if they didn't already + # own it. + for hda in history.datasets: + user.adjust_total_disk_usage(hda.quota_amount(user)) + # Only set default history permissions if the history is from the previous session and anonymous + set_permissions = True elif self.galaxy_session.current_history: history = self.galaxy_session.current_history if (not history and users_last_session and @@ -681,8 +684,7 @@ class GalaxyWebTransaction(base.DefaultWebTransaction, if history.user is None: history.user = user self.galaxy_session.current_history = history - if not last_accessed: - # Only set default history permissions if current history is not from a previous session + if set_permissions: self.app.security_agent.history_set_default_permissions(history, dataset=True, bypass_manage_permission=True) self.sa_session.add_all((prev_galaxy_session, self.galaxy_session, history)) diff --git a/lib/galaxy/web/stack/handlers.py b/lib/galaxy/web/stack/handlers.py index 1290923427d..7754e8966b4 100644 --- a/lib/galaxy/web/stack/handlers.py +++ b/lib/galaxy/web/stack/handlers.py @@ -95,7 +95,7 @@ class ConfiguresHandlers(object): self._parse_handler(handler_id, process) self.add_handler(handler_id, process.get("tags") or [self.DEFAULT_HANDLER_TAG]) - self.default_handler_id = self._ensure_default_set(handling_config_dict.get("default"), self.handlers.keys()) + self.default_handler_id = self._ensure_default_set(handling_config_dict.get("default"), list(self.handlers.keys()), required=False) def _init_handler_assignment_methods(self, handling_config_dict=None): handling_config_dict = handling_config_dict or {} @@ -162,7 +162,7 @@ class ConfiguresHandlers(object): rval = config.config_dict.get(config_val, rval) return rval - def _get_default(self, config, parent, names, auto=False): + def _get_default(self, config, parent, names, auto=False, required=True): """ Returns the default attribute set in a parent tag like or , or return the ID of the child, if there is no explicit @@ -174,21 +174,25 @@ class ConfiguresHandlers(object): :type names: list of str :param auto: Automatically set a default if there is no default in the parent tag and there is only one child. :type auto: bool + :param required: Require a default to be set or determined automatically, else raise Exception + :type required: bool :returns: str -- id or tag representing the default. """ rval = ConfiguresHandlers.get_xml_default(config, parent) - return self._ensure_default_set(rval, names, auto=auto) + return self._ensure_default_set(rval, names, auto=auto, required=required) - def _ensure_default_set(self, rval, names, auto=False): + def _ensure_default_set(self, rval, names, auto=False, required=True): if rval is not None: # If the parent element has a 'default' attribute, use the id or tag in that attribute - if self.deterministic_handler_assignment and rval not in names: + if required and rval not in names: raise Exception("default attribute '%s' does not match a defined id or tag in a child element" % (rval)) log.debug("default set to child with id or tag '%s'" % (rval)) elif auto and len(names) == 1: log.info("Setting default to child with id '%s'" % (names[0])) rval = names[0] + elif required: + raise Exception("No default specified, please specify a valid id or tag with the 'default' attribute") return rval @staticmethod @@ -223,7 +227,7 @@ class ConfiguresHandlers(object): @property def deterministic_handler_assignment(self): - return self.handler_assignment_methods and all( + return self.handler_assignment_methods and any( filter(lambda x: x in ( HANDLER_ASSIGNMENT_METHODS.UWSGI_MULE_MESSAGE, HANDLER_ASSIGNMENT_METHODS.DB_PREASSIGN, diff --git a/test/unit/jobs/job_conf.sample_advanced.yml b/test/unit/jobs/job_conf.sample_advanced.yml index 216aba9ffca..7426838b7f3 100644 --- a/test/unit/jobs/job_conf.sample_advanced.yml +++ b/test/unit/jobs/job_conf.sample_advanced.yml @@ -150,6 +150,7 @@ handling: execution: + default: local environments: local: runner: local