mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-19 02:21:32 +08:00
Merge branch 'release_18.09' into dev
This commit is contained in:
Nicola Soranzo
@@ -354,7 +354,7 @@ div.unified-panel-body-background {
|
||||
|
||||
#left > div.unified-panel-body,
|
||||
#right > div.unified-panel-body {
|
||||
bottom: $panel_footer_height;
|
||||
padding-bottom: $panel_footer_height;
|
||||
overflow: auto;
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,87 @@
|
||||
|
||||
===========================================================
|
||||
September 2018 Galaxy Release (v 18.09)
|
||||
===========================================================
|
||||
|
||||
.. include:: _header.rst
|
||||
|
||||
Highlights
|
||||
===========================================================
|
||||
|
||||
**Extensive Workflow Enhancements**
|
||||
Workflows got a lot of love this time around, with new runtime parameters for
|
||||
subworkflows, exposed workflow versions, and zoom capability in the editor, to
|
||||
name a few. There were also a number of usability enhancements including
|
||||
better labeling, links, overhauled workflow import interfaces, and many more.
|
||||
|
||||
`Pull Request 6664`_, `Pull Request 6683`_, `Pull Request 6142`_,
|
||||
`Pull Request 6240`_, `Pull Request 6290`_, `Pull Request 6428`_,
|
||||
`Pull Request 6441`_, `Pull Request 6580`_, `Pull Request 6596`_,
|
||||
`Pull Request 6678`_, `Pull Request 6076`_, `Pull Request 6274`_,
|
||||
`Pull Request 6570`_, `Pull Request 6680`_, `Pull Request 6774`_
|
||||
|
||||
**Group Tags**
|
||||
Galaxy now contains powerful new features for multiple factor analysis of collections
|
||||
of datasets. The concept of group tags has been added to Galaxy. These are a special
|
||||
class of tags that describe key-value pairs that can be attached to the contents of a
|
||||
collection during upload or using collection operation tools. These tags can describe
|
||||
multiple sets of variables for the contents of a collection. Once set, these tags can
|
||||
be consumed intelligently by tools that need to divide collections into multiple
|
||||
overlapping factors or sets of datasets. A special thanks to `@mvdbeek <https://github.com/mvdbeek>`__ for devising and implementing this approach.
|
||||
|
||||
`Pull Request 5457`_, `Pull Request 6491`_, `Pull Request 6661`_,
|
||||
`Pull Request 6750`_, `Pull Request 6499`_, `Pull Request 6500`_,
|
||||
`Pull Request 6572`_, `Pull Request 6545`_
|
||||
|
||||
**Python 3 Beta Support**
|
||||
After almost 3 years of work and more than 100 pull requests, we are proud
|
||||
to announce the Beta-stage support for running Galaxy under Python 3.
|
||||
Lint, unit, API, framework, integration and Selenium tests all pass, time
|
||||
for you to give it a try and report any bug you find!
|
||||
|
||||
`Tracking issue <https://github.com/galaxyproject/galaxy/issues/1715>`__
|
||||
|
||||
Get Galaxy
|
||||
==========
|
||||
|
||||
The code lives at `GitHub <https://github.com/galaxyproject/galaxy>`__ and you should have `Git <https://git-scm.com/>`__ to obtain it.
|
||||
|
||||
To get a new Galaxy repository run:
|
||||
.. code-block:: shell
|
||||
|
||||
$ git clone -b release_18.09 https://github.com/galaxyproject/galaxy.git
|
||||
|
||||
To update an existing Galaxy repository run:
|
||||
.. code-block:: shell
|
||||
|
||||
$ git fetch origin && git checkout release_18.09 && git pull --ff-only origin release_18.09
|
||||
|
||||
See the `community hub <https://galaxyproject.org/develop/source-code/>`__ for additional details regarding the source code locations.
|
||||
|
||||
Security
|
||||
========
|
||||
|
||||
Unauthorized File System Operations via New Upload API
|
||||
------------------------------------------------------
|
||||
|
||||
Tracked as GX-2018-0006. Servers running Galaxy 18.05 should be updated as soon as possible.
|
||||
See `the public announcement for full details
|
||||
<http://announce.list.galaxyproject.org/GX-2018-0006-Unauthorized-File-System-Operations-via-New-Upload-API-td4639432.html>`__.
|
||||
|
||||
Deprecation Notice
|
||||
==================
|
||||
|
||||
With 19.01, all Galaxy users will be forced to have a username. This
|
||||
requirement has been enforced on user creation for years, though we have never
|
||||
taken steps to coerce users created in the past who did not have one. The
|
||||
19.01 release will come with a migration script that will coerce this in the
|
||||
database, which will allow us to have consistent handling of this field.
|
||||
|
||||
|
||||
Release Notes
|
||||
===========================================================
|
||||
|
||||
.. include:: 18.09.rst
|
||||
:start-after: announce_start
|
||||
|
||||
.. include:: _thanks.rst
|
||||
@@ -0,0 +1,10 @@
|
||||
|
||||
===========================================================
|
||||
January 2019 Galaxy Release (v 19.01)
|
||||
===========================================================
|
||||
|
||||
|
||||
Schedule
|
||||
===========================================================
|
||||
* Planned Freeze Date: 2019-01-07
|
||||
* Planned Release Date: 2019-01-28
|
||||
@@ -93,7 +93,7 @@ oslo.log==3.39.0
|
||||
oslo.serialization==2.27.0; python_version != '3.3.*'
|
||||
oslo.utils==3.37.0; python_version != '3.3.*'
|
||||
packaging==17.1
|
||||
paramiko==2.4.1
|
||||
paramiko==2.4.2
|
||||
parsley==1.3
|
||||
paste==2.0.3
|
||||
pastedeploy==1.5.2
|
||||
|
||||
@@ -44,7 +44,7 @@ class LocalShell(BaseShellExec):
|
||||
|
||||
def execute(self, cmd, persist=False, timeout=DEFAULT_TIMEOUT, timeout_check_interval=DEFAULT_TIMEOUT_CHECK_INTERVAL, **kwds):
|
||||
outf = TemporaryFile()
|
||||
p = Popen(cmd, shell=True, stdin=None, stdout=outf, stderr=PIPE)
|
||||
p = Popen(cmd, stdin=None, stdout=outf, stderr=PIPE)
|
||||
# poll until timeout
|
||||
|
||||
for i in range(int(timeout / timeout_check_interval)):
|
||||
|
||||
@@ -19,20 +19,23 @@ __all__ = ('RemoteShell', 'SecureShell', 'GlobusSecureShell', 'ParamikoShell')
|
||||
|
||||
class RemoteShell(LocalShell):
|
||||
|
||||
def __init__(self, rsh='rsh', rcp='rcp', hostname='localhost', username=None, **kwargs):
|
||||
def __init__(self, rsh='rsh', rcp='rcp', hostname='localhost', username=None, options=None, **kwargs):
|
||||
super(RemoteShell, self).__init__(**kwargs)
|
||||
self.rsh = rsh
|
||||
self.rcp = rcp
|
||||
self.hostname = hostname
|
||||
self.username = username
|
||||
self.options = options
|
||||
self.sessions = {}
|
||||
|
||||
def execute(self, cmd, persist=False, timeout=60):
|
||||
# TODO: implement persistence
|
||||
if self.username is None:
|
||||
fullcmd = '%s %s %s' % (self.rsh, self.hostname, cmd)
|
||||
else:
|
||||
fullcmd = '%s -l %s %s %s' % (self.rsh, self.username, self.hostname, cmd)
|
||||
fullcmd = [self.rsh]
|
||||
if self.options:
|
||||
fullcmd.extend(self.options)
|
||||
if self.username:
|
||||
fullcmd.extend(["-l", self.username])
|
||||
fullcmd.extend([self.hostname, cmd])
|
||||
return super(RemoteShell, self).execute(fullcmd, persist, timeout)
|
||||
|
||||
|
||||
@@ -41,15 +44,13 @@ class SecureShell(RemoteShell):
|
||||
|
||||
def __init__(self, rsh='ssh', rcp='scp', private_key=None, port=None, strict_host_key_checking=True, **kwargs):
|
||||
strict_host_key_checking = "yes" if strict_host_key_checking else "no"
|
||||
rsh += " -oStrictHostKeyChecking=%s -oConnectTimeout=60" % strict_host_key_checking
|
||||
rcp += " -oStrictHostKeyChecking=%s -oConnectTimeout=60" % strict_host_key_checking
|
||||
options = ["-o", "StrictHostKeyChecking=%s" % strict_host_key_checking]
|
||||
options.extend(["-o", "ConnectTimeout=60"])
|
||||
if private_key:
|
||||
rsh += " -i %s" % private_key
|
||||
rcp += " -i %s" % private_key
|
||||
options.extend(['-i', private_key])
|
||||
if port:
|
||||
rsh += " -p %s" % port
|
||||
rcp += " -p %s" % port
|
||||
super(SecureShell, self).__init__(rsh=rsh, rcp=rcp, **kwargs)
|
||||
options.extend(['-p', str(port)])
|
||||
super(SecureShell, self).__init__(rsh=rsh, rcp=rcp, options=options, **kwargs)
|
||||
|
||||
|
||||
class ParamikoShell(object):
|
||||
|
||||
@@ -3,6 +3,7 @@ from __future__ import absolute_import
|
||||
import logging
|
||||
import threading
|
||||
|
||||
from galaxy.web.stack import register_postfork_function
|
||||
from .sleeper import Sleeper
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -27,7 +28,11 @@ class Monitors(object):
|
||||
self.sleeper = Sleeper()
|
||||
self.monitor_thread = threading.Thread(name=name, target=monitor_func)
|
||||
self.monitor_thread.setDaemon(True)
|
||||
if start:
|
||||
self._start = start
|
||||
register_postfork_function(self.start_monitoring)
|
||||
|
||||
def start_monitoring(self):
|
||||
if self._start:
|
||||
self.monitor_thread.start()
|
||||
|
||||
def stop_monitoring(self):
|
||||
|
||||
@@ -40,7 +40,8 @@ DEVTEAM = [
|
||||
"afgane", "dannon", "blankenberg",
|
||||
"davebx", "martenson", "jmchilton",
|
||||
"tnabtaf", "natefoo", "jgoecks",
|
||||
"guerler", "jennaj", "nekrut", "jxtx"
|
||||
"guerler", "jennaj", "nekrut", "jxtx",
|
||||
"VJalili"
|
||||
]
|
||||
|
||||
TEMPLATE = """
|
||||
@@ -102,7 +103,7 @@ Highlights
|
||||
Get Galaxy
|
||||
==========
|
||||
|
||||
The code lives at `Github <https://github.com/galaxyproject/galaxy>`__ and you should have `Git <https://git-scm.com/>`__ to obtain it.
|
||||
The code lives at `GitHub <https://github.com/galaxyproject/galaxy>`__ and you should have `Git <https://git-scm.com/>`__ to obtain it.
|
||||
|
||||
To get a new Galaxy repository run:
|
||||
.. code-block:: shell
|
||||
@@ -202,7 +203,7 @@ RELEASE_ISSUE_TEMPLATE = string.Template("""
|
||||
- [ ] Ensure all [blocking milestone PRs](https://github.com/galaxyproject/galaxy/pulls?q=is%3Aopen+is%3Apr+milestone%3A${version}) have been merged or closed.
|
||||
|
||||
make release-check-blocking-prs RELEASE_CURR=${version}
|
||||
- [ ] Ensure previous release is merged into current. [Github branch comparison](https://github.com/galaxyproject/galaxy/compare/release_${version}...release_${previous_version})
|
||||
- [ ] Ensure previous release is merged into current. [GitHub branch comparison](https://github.com/galaxyproject/galaxy/compare/release_${version}...release_${previous_version})
|
||||
- [ ] Create and push release tag:
|
||||
|
||||
make release-create RELEASE_CURR=${version}
|
||||
|
||||
@@ -12289,7 +12289,7 @@ div.unified-panel-body-background {
|
||||
|
||||
#left > div.unified-panel-body,
|
||||
#right > div.unified-panel-body {
|
||||
bottom: 25px;
|
||||
padding-bottom: 25px;
|
||||
overflow: auto; }
|
||||
|
||||
#dd-helper {
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
echo `pwd` > '$pwd';
|
||||
echo "\$HOME" > '$home';
|
||||
echo "\$TMP" > '$tmp';
|
||||
echo "\$SOME_ENV_VAR" > '$some_env_var';
|
||||
]]></command>
|
||||
<inputs>
|
||||
</inputs>
|
||||
@@ -17,6 +18,7 @@
|
||||
<data name="pwd" format="txt" label="pwd" />
|
||||
<data name="home" format="txt" label="home" />
|
||||
<data name="tmp" format="txt" label="tmp" />
|
||||
<data name="some_env_var" format="txt" label="env_var" />
|
||||
</outputs>
|
||||
<help>
|
||||
</help>
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
echo `pwd` > '$pwd';
|
||||
echo "\$HOME" > '$home';
|
||||
echo "\$TMP" > '$tmp';
|
||||
echo "\$SOME_ENV_VAR" > '$some_env_var';
|
||||
]]></command>
|
||||
<inputs>
|
||||
</inputs>
|
||||
@@ -17,6 +18,7 @@
|
||||
<data name="pwd" format="txt" label="pwd" />
|
||||
<data name="home" format="txt" label="home" />
|
||||
<data name="tmp" format="txt" label="tmp" />
|
||||
<data name="some_env_var" format="txt" label="env_var" />
|
||||
</outputs>
|
||||
<help>
|
||||
</help>
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
echo `pwd` > '$pwd';
|
||||
echo "\$HOME" > '$home';
|
||||
echo "\$TMP" > '$tmp';
|
||||
echo "\$SOME_ENV_VAR" > '$some_env_var';
|
||||
]]></command>
|
||||
<inputs>
|
||||
</inputs>
|
||||
@@ -17,6 +18,7 @@
|
||||
<data name="pwd" format="txt" label="pwd" />
|
||||
<data name="home" format="txt" label="home" />
|
||||
<data name="tmp" format="txt" label="tmp" />
|
||||
<data name="some_env_var" format="txt" label="env_var" />
|
||||
</outputs>
|
||||
<help>
|
||||
</help>
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
"""Integration tests for the CLI shell plugins and runners."""
|
||||
import collections
|
||||
import os
|
||||
import string
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from Crypto.PublicKey import RSA
|
||||
|
||||
from base import integration_util # noqa: I100,I202
|
||||
from base.populators import skip_without_tool
|
||||
from .test_job_environments import BaseJobEnvironmentIntegrationTestCase # noqa: I201
|
||||
|
||||
|
||||
def generate_keys():
|
||||
key = RSA.generate(2048)
|
||||
return (key.export_key(), key.publickey().export_key(format='OpenSSH'))
|
||||
|
||||
|
||||
RemoteConnection = collections.namedtuple('remote_connection', ['hostname', 'username', 'password', 'port', 'private_key', 'public_key'])
|
||||
|
||||
|
||||
@integration_util.skip_unless_docker()
|
||||
def start_ssh_docker(container_name, jobs_directory, port=10022, image='agaveapi/slurm'):
|
||||
private_key, public_key = generate_keys()
|
||||
with tempfile.NamedTemporaryFile(delete=False) as f:
|
||||
f.write(private_key)
|
||||
private_key_file = f.name
|
||||
with tempfile.NamedTemporaryFile(delete=False) as f:
|
||||
f.write(public_key)
|
||||
public_key_file = f.name
|
||||
START_SLURM_DOCKER = ['docker',
|
||||
'run',
|
||||
'-h',
|
||||
'localhost',
|
||||
'-p',
|
||||
'{port}:22'.format(port=port),
|
||||
'-d',
|
||||
'--name',
|
||||
container_name,
|
||||
'--rm',
|
||||
'-v',
|
||||
"{jobs_directory}:{jobs_directory}".format(jobs_directory=jobs_directory),
|
||||
"-v",
|
||||
"{public_key_file}:/home/testuser/.ssh/authorized_keys".format(public_key_file=public_key_file),
|
||||
'--ulimit',
|
||||
'nofile=2048:2048',
|
||||
image]
|
||||
subprocess.check_call(START_SLURM_DOCKER)
|
||||
return RemoteConnection('localhost', 'testuser', 'testuser', port, private_key_file, public_key_file)
|
||||
|
||||
|
||||
def stop_ssh_docker(container_name, remote_connection):
|
||||
subprocess.check_call(['docker', 'rm', '-f', container_name])
|
||||
os.remove(remote_connection.private_key)
|
||||
os.remove(remote_connection.public_key)
|
||||
|
||||
|
||||
def cli_job_config(remote_connection, shell_plugin='ParamikoShell', job_plugin='Slurm'):
|
||||
job_conf_template = string.Template("""<job_conf>
|
||||
<plugins>
|
||||
<plugin id="cli" type="runner" load="galaxy.jobs.runners.cli:ShellJobRunner" workers="1"/>
|
||||
</plugins>
|
||||
<destinations default="ssh_slurm">
|
||||
<destination id="ssh_slurm" runner="cli">
|
||||
<param id="shell_plugin">$shell_plugin</param>
|
||||
<param id="job_plugin">$job_plugin</param>
|
||||
<param id="shell_username">$username</param>
|
||||
<param id="shell_private_key">$private_key</param>
|
||||
<param id="shell_hostname">$hostname</param>
|
||||
<param id="shell_port">$port</param>
|
||||
<param id="embed_metadata_in_job">False</param>
|
||||
<env id="SOME_ENV_VAR">42</env>
|
||||
</destination>
|
||||
</destinations>
|
||||
</job_conf>
|
||||
""")
|
||||
job_conf_str = job_conf_template.substitute(shell_plugin=shell_plugin,
|
||||
job_plugin=job_plugin,
|
||||
**remote_connection._asdict())
|
||||
with tempfile.NamedTemporaryFile(suffix="_slurm_integration_job_conf", delete=False) as job_conf:
|
||||
job_conf.write(job_conf_str)
|
||||
return job_conf.name
|
||||
|
||||
|
||||
class BaseCliIntegrationTestCase(BaseJobEnvironmentIntegrationTestCase):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
if cls is BaseCliIntegrationTestCase:
|
||||
raise unittest.SkipTest("Base class")
|
||||
cls.container_name = "%s_container" % cls.__name__
|
||||
cls.jobs_directory = tempfile.mkdtemp()
|
||||
cls.remote_connection = start_ssh_docker(container_name=cls.container_name,
|
||||
jobs_directory=cls.jobs_directory,
|
||||
image=cls.image)
|
||||
super(BaseCliIntegrationTestCase, cls).setUpClass()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
stop_ssh_docker(cls.container_name, cls.remote_connection)
|
||||
super(BaseCliIntegrationTestCase, cls).tearDownClass()
|
||||
|
||||
@classmethod
|
||||
def handle_galaxy_config_kwds(cls, config, ):
|
||||
config["jobs_directory"] = cls.jobs_directory
|
||||
config["file_path"] = cls.jobs_directory
|
||||
config["job_config_file"] = cli_job_config(remote_connection=cls.remote_connection,
|
||||
shell_plugin=cls.shell_plugin,
|
||||
job_plugin=cls.job_plugin)
|
||||
|
||||
@skip_without_tool("job_environment_default")
|
||||
def test_running_cli_job(self):
|
||||
job_env = self._run_and_get_environment_properties()
|
||||
assert job_env.some_env == '42'
|
||||
|
||||
|
||||
class TorqueSetup(object):
|
||||
job_plugin = 'Torque'
|
||||
image = 'mvdbeek/galaxy-integration-docker-images:torque_latest'
|
||||
|
||||
|
||||
class SlurmSetup(object):
|
||||
job_plugin = 'Slurm'
|
||||
image = 'mvdbeek/galaxy-integration-docker-images:slurm_latest'
|
||||
|
||||
|
||||
class ParamikoShell(object):
|
||||
shell_plugin = 'ParamikoShell'
|
||||
|
||||
|
||||
class SecureShell(object):
|
||||
shell_plugin = 'SecureShell'
|
||||
|
||||
|
||||
class ParamikoCliSlurmIntegrationTestCase(SlurmSetup, ParamikoShell, BaseCliIntegrationTestCase):
|
||||
pass
|
||||
|
||||
|
||||
class ShellJobCliSlurmIntegrationTestCase(SlurmSetup, SecureShell, BaseCliIntegrationTestCase):
|
||||
pass
|
||||
|
||||
|
||||
class ParamikoCliTorqueIntegrationTestCase(TorqueSetup, ParamikoShell, BaseCliIntegrationTestCase):
|
||||
pass
|
||||
|
||||
|
||||
class ShellJobCliTorqueIntegrationTestCase(TorqueSetup, SecureShell, BaseCliIntegrationTestCase):
|
||||
pass
|
||||
@@ -21,6 +21,7 @@ JobEnviromentProperties = collections.namedtuple("JobEnvironmentProperties", [
|
||||
"pwd",
|
||||
"home",
|
||||
"tmp",
|
||||
"some_env",
|
||||
])
|
||||
|
||||
|
||||
@@ -38,8 +39,8 @@ class RunsEnvironmentJobs(object):
|
||||
pwd = self.dataset_populator.get_history_dataset_content(history_id, hid=3).strip()
|
||||
home = self.dataset_populator.get_history_dataset_content(history_id, hid=4).strip()
|
||||
tmp = self.dataset_populator.get_history_dataset_content(history_id, hid=5).strip()
|
||||
|
||||
return JobEnviromentProperties(user_id, group_id, pwd, home, tmp)
|
||||
some_env = self.dataset_populator.get_history_dataset_content(history_id, hid=6).strip()
|
||||
return JobEnviromentProperties(user_id, group_id, pwd, home, tmp, some_env)
|
||||
|
||||
|
||||
class BaseJobEnvironmentIntegrationTestCase(integration_util.IntegrationTestCase, RunsEnvironmentJobs):
|
||||
|
||||
Reference in New Issue
Block a user