opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'galaxyproject/release_19.05' into dev

Browse Source
This commit is contained in:
John Chilton
2019-05-07 17:08:21 -04:00
parent 89f54e59d6 7a4ca3a825
commit 199062cc4b
10 changed files with 45 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/galaxy/scripts/mvc/history/history-contents.js
+2 -1
View File
@@ -442,7 +442,8 @@ export var HistoryContents = _super.extend(BASE_MVC.LoggableMixin).extend({
data: JSON.stringify({
content: id,
source: contentType,
type: type
type: type,
copy_elements: true
})
})
.done(response => {
client/galaxy/scripts/mvc/ui/ui-slider.js
+1
View File
@@ -47,6 +47,7 @@ var View = Backbone.View.extend({
v == 13 ||
v == 37 ||
v == 39 ||
v == 69 ||
(v >= 48 && v <= 57) ||
(v >= 96 && v <= 105) ||
((v == 190 || v == 110) &&
client/galaxy/scripts/mvc/workflow/workflow-view.js
+3 -2
View File
@@ -258,7 +258,7 @@ export default Backbone.View.extend({
for (let i = 0; i < workflow_versions.length; i++) {
const current_wf = workflow_versions[i];
let version_text = `Version ${current_wf["version"]}, ${current_wf["steps"]} steps`;
let version_text = `Version ${current_wf.version}, ${current_wf.steps} steps`;
let selected = false;
if (i == self.workflow.workflow_version) {
version_text = `${version_text} (active)`;
@@ -287,7 +287,7 @@ export default Backbone.View.extend({
$("#workflow-version-switch").unbind("change");
if (this.value != self.workflow.workflow_version) {
if (self.workflow && self.workflow.has_changes) {
const r = confirm("There are unsaved changes to your workflow which will be lost. Continue ?");
const r = window.confirm("There are unsaved changes to your workflow which will be lost. Continue ?");
if (r == false) {
// We rebuild the version select list, to reset the selected version
self.build_version_select();
@@ -774,6 +774,7 @@ export default Backbone.View.extend({
// Fix width to computed width
// Now add floats
var buttons = $("<div class='buttons' style='float: right;'></div>");
buttons.append($(`<div class="sr-only"><a href="#right">Node Details</a></div>`));
if (type !== "subworkflow") {
buttons.append(
$("<div/>")
lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt
+1 -1
View File
@@ -36,7 +36,7 @@ cachetools==3.1.0
certifi==2019.3.9
cffi==1.12.3
chardet==3.0.4
cheetah3==3.2.1
cheetah3==3.2.2
cliff==2.14.1
cloudauthz==0.2.0
cloudbridge==2.0.0
lib/galaxy/jobs/runners/slurm.py
+2 -6
View File
@@ -28,13 +28,9 @@ SLURM_MEMORY_LIMIT_EXCEEDED_MSG = 'slurmstepd: error: Exceeded job memory limit'
SLURM_MEMORY_LIMIT_EXCEEDED_PARTIAL_WARNINGS = [': Exceeded job memory limit at some point.',
': Exceeded step memory limit at some point.']
SLURM_MEMORY_LIMIT_SCAN_SIZE = 16 * 1024 * 1024 # 16MB
SLURM_UNABLE_TO_ADD_TASK_TO_MEMORY_CG_MSG_RE = re.compile(r"""slurmstepd: error: task/cgroup: unable to add task\[pid=\d+\] to memory cg '\(null\)'$""")
SLURM_UNABLE_TO_CREATE_CGROUP_MSG_RE = re.compile(r"""slurmstepd: error: xcgroup_instantiate: unable to create cgroup '[^']+' : No space left on device$""")
SLURM_UNABLE_TO_INSTANCIATE_CGROUP_MSG_RE = re.compile(r"""slurmstepd: error: jobacct_gather/cgroup: unable to instanciate (job|user) \d+ memory cgroup$""")
SLURM_CGROUP_RE = re.compile(r"""slurmstepd: .*cgroup.*$""")
SLURM_TOP_WARNING_RES = (
SLURM_UNABLE_TO_ADD_TASK_TO_MEMORY_CG_MSG_RE,
SLURM_UNABLE_TO_CREATE_CGROUP_MSG_RE,
SLURM_UNABLE_TO_INSTANCIATE_CGROUP_MSG_RE
SLURM_CGROUP_RE,
)
# These messages are returned to the user
lib/galaxy/objectstore/s3_multipart_upload.py
+8 -39
View File
@@ -5,13 +5,10 @@ This parallelizes the task over available cores using multiprocessing.
Code mostly taken form CloudBioLinux.
"""
import contextlib
import functools
import glob
import multiprocessing
import os
import subprocess
from multiprocessing.pool import IMapIterator
import threading
try:
import boto
@@ -20,13 +17,6 @@ except ImportError:
boto = None
def map_wrap(f):
@functools.wraps(f)
def wrapper(args):
return f(*args)
return wrapper
def mp_from_ids(s3server, mp_id, mp_keyname, mp_bucketname):
"""Get the multipart upload from the bucket and multipart IDs.
@@ -51,7 +41,6 @@ def mp_from_ids(s3server, mp_id, mp_keyname, mp_bucketname):
return mp
@map_wrap
def transfer_part(s3server, mp_id, mp_keyname, mp_bucketname, i, part):
"""Transfer a part of a multipart upload. Designed to be run in parallel.
"""
@@ -64,8 +53,6 @@ def transfer_part(s3server, mp_id, mp_keyname, mp_bucketname, i, part):
def multipart_upload(s3server, bucket, s3_key_name, tarball, mb_size):
"""Upload large files using Amazon's multipart upload functionality.
"""
cores = multiprocessing.cpu_count()
def split_file(in_file, mb_size, split_num=5):
prefix = os.path.join(os.path.dirname(in_file),
"%sS3PART" % (os.path.basename(s3_key_name)))
@@ -80,29 +67,11 @@ def multipart_upload(s3server, bucket, s3_key_name, tarball, mb_size):
mp = bucket.initiate_multipart_upload(s3_key_name,
reduced_redundancy=s3server['use_rr'])
with multimap(cores) as pmap:
for _ in pmap(transfer_part, ((s3server, mp.id, mp.key_name, mp.bucket_name, i, part)
for (i, part) in
enumerate(split_file(tarball, mb_size, cores)))):
pass
for (i, part) in enumerate(split_file(tarball, mb_size)):
t = threading.Thread(
target=transfer_part,
args=(s3server, mp.id, mp.key_name, mp.bucket_name, i, part))
t.start()
t.join()
mp.complete_upload()
@contextlib.contextmanager
def multimap(cores=None):
"""Provide multiprocessing imap like function.
The context manager handles setting up the pool, worked around interrupt issues
and terminating the pool on completion.
"""
if cores is None:
cores = max(multiprocessing.cpu_count() - 1, 1)
def wrapper(func):
def wrap(self, timeout=None):
return func(self, timeout=timeout if timeout is not None else 1e100)
return wrap
IMapIterator.next = wrapper(IMapIterator.next)
pool = multiprocessing.Pool(cores)
yield pool.imap
pool.terminate()
lib/galaxy/tools/deps/containers.py
+2
View File
@@ -206,7 +206,9 @@ class ContainerRegistry(object):
if self.enable_beta_mulled_containers:
default_resolvers.extend([
CachedMulledDockerContainerResolver(self.app_info, namespace="biocontainers"),
CachedMulledDockerContainerResolver(self.app_info, namespace="local"),
CachedMulledSingularityContainerResolver(self.app_info, namespace="biocontainers"),
CachedMulledSingularityContainerResolver(self.app_info, namespace="local"),
MulledDockerContainerResolver(self.app_info, namespace="biocontainers"),
MulledSingularityContainerResolver(self.app_info, namespace="biocontainers"),
BuildMulledDockerContainerResolver(self.app_info),
lib/galaxy/web/framework/webapp.py
+15 -13
View File
@@ -650,24 +650,27 @@ class GalaxyWebTransaction(base.DefaultWebTransaction,
Associate the user's last accessed history (if exists) with their new session
"""
history = None
set_permissions = False
try:
users_last_session = user.galaxy_sessions[0]
last_accessed = True
except Exception:
users_last_session = None
last_accessed = False
if (prev_galaxy_session and
prev_galaxy_session.current_history and not
prev_galaxy_session.current_history.deleted and
prev_galaxy_session.current_history.datasets):
if prev_galaxy_session.current_history.user is None or prev_galaxy_session.current_history.user == user:
# If the previous galaxy session had a history, associate it with the new
# session, but only if it didn't belong to a different user.
history = prev_galaxy_session.current_history
if prev_galaxy_session.user is None:
# Increase the user's disk usage by the amount of the previous history's datasets if they didn't already own it.
for hda in history.datasets:
user.adjust_total_disk_usage(hda.quota_amount(user))
prev_galaxy_session.current_history.datasets and
(prev_galaxy_session.current_history.user is None or
prev_galaxy_session.current_history.user == user)):
# If the previous galaxy session had a history, associate it with the new session, but only if it didn't
# belong to a different user.
history = prev_galaxy_session.current_history
if prev_galaxy_session.user is None:
# Increase the user's disk usage by the amount of the previous history's datasets if they didn't already
# own it.
for hda in history.datasets:
user.adjust_total_disk_usage(hda.quota_amount(user))
# Only set default history permissions if the history is from the previous session and anonymous
set_permissions = True
elif self.galaxy_session.current_history:
history = self.galaxy_session.current_history
if (not history and users_last_session and
@@ -681,8 +684,7 @@ class GalaxyWebTransaction(base.DefaultWebTransaction,
if history.user is None:
history.user = user
self.galaxy_session.current_history = history
if not last_accessed:
# Only set default history permissions if current history is not from a previous session
if set_permissions:
self.app.security_agent.history_set_default_permissions(history, dataset=True, bypass_manage_permission=True)
self.sa_session.add_all((prev_galaxy_session, self.galaxy_session, history))
lib/galaxy/web/stack/handlers.py
+10 -6
View File
@@ -95,7 +95,7 @@ class ConfiguresHandlers(object):
self._parse_handler(handler_id, process)
self.add_handler(handler_id, process.get("tags") or [self.DEFAULT_HANDLER_TAG])
self.default_handler_id = self._ensure_default_set(handling_config_dict.get("default"), self.handlers.keys())
self.default_handler_id = self._ensure_default_set(handling_config_dict.get("default"), list(self.handlers.keys()), required=False)
def _init_handler_assignment_methods(self, handling_config_dict=None):
handling_config_dict = handling_config_dict or {}
@@ -162,7 +162,7 @@ class ConfiguresHandlers(object):
rval = config.config_dict.get(config_val, rval)
return rval
def _get_default(self, config, parent, names, auto=False):
def _get_default(self, config, parent, names, auto=False, required=True):
"""
Returns the default attribute set in a parent tag like <handlers> or
<destinations>, or return the ID of the child, if there is no explicit
@@ -174,21 +174,25 @@ class ConfiguresHandlers(object):
:type names: list of str
:param auto: Automatically set a default if there is no default in the parent tag and there is only one child.
:type auto: bool
:param required: Require a default to be set or determined automatically, else raise Exception
:type required: bool
:returns: str -- id or tag representing the default.
"""
rval = ConfiguresHandlers.get_xml_default(config, parent)
return self._ensure_default_set(rval, names, auto=auto)
return self._ensure_default_set(rval, names, auto=auto, required=required)
def _ensure_default_set(self, rval, names, auto=False):
def _ensure_default_set(self, rval, names, auto=False, required=True):
if rval is not None:
# If the parent element has a 'default' attribute, use the id or tag in that attribute
if self.deterministic_handler_assignment and rval not in names:
if required and rval not in names:
raise Exception("default attribute '%s' does not match a defined id or tag in a child element" % (rval))
log.debug("default set to child with id or tag '%s'" % (rval))
elif auto and len(names) == 1:
log.info("Setting default to child with id '%s'" % (names[0]))
rval = names[0]
elif required:
raise Exception("No default specified, please specify a valid id or tag with the 'default' attribute")
return rval
@staticmethod
@@ -223,7 +227,7 @@ class ConfiguresHandlers(object):
@property
def deterministic_handler_assignment(self):
return self.handler_assignment_methods and all(
return self.handler_assignment_methods and any(
filter(lambda x: x in (
HANDLER_ASSIGNMENT_METHODS.UWSGI_MULE_MESSAGE,
HANDLER_ASSIGNMENT_METHODS.DB_PREASSIGN,
test/unit/jobs/job_conf.sample_advanced.yml
+1
View File
@@ -150,6 +150,7 @@ handling:
execution:
default: local
environments:
local:
runner: local