opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-21 13:50:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/dev' into feature/auto_create_userfolder

Browse Source
This commit is contained in:
Nate Coraor
2018-01-19 11:39:30 -05:00
parent 4947ab0738 080148a161
commit ea7dbd1906
118 changed files with 11213 additions and 2572 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.ci/jenkins/selenium/run_tests.sh
+2
View File
@@ -16,6 +16,8 @@ fi
mkdir -p "$GALAXY_TEST_ERRORS_DIRECTORY"
mkdir -p "$GALAXY_TEST_SCREENSHOTS_DIRECTORY"
docker run -v `pwd`:`pwd`:rw -w `pwd` -u $UID node /bin/bash -c 'make client-production-maps'
# Start Selenium server in the test Docker container.
DOCKER_RUN_EXTRA_ARGS="-e USE_SELENIUM=1 -e GALAXY_TEST_SELENIUM_RETRIES=${GALAXY_TEST_SELENIUM_RETRIES} -e GALAXY_TEST_ERRORS_DIRECTORY=${GALAXY_TEST_ERRORS_DIRECTORY} -e GALAXY_TEST_SCREENSHOTS_DIRECTORY=${GALAXY_TEST_SCREENSHOTS_DIRECTORY} ${DOCKER_RUN_EXTRA_ARGS}"
export DOCKER_RUN_EXTRA_ARGS
.gitignore
+3
View File
@@ -133,6 +133,9 @@ client/galaxy/scripts/libs/*
# Documentation build files.
doc/build
doc/schema.md
doc/source/admin/config_logging_default_yaml.rst
doc/source/api/api.rst
doc/source/api/ts_api.rst
doc/source/dev/schema.rst
# Misc
.travis.yml
+1 -1
View File
@@ -30,7 +30,7 @@ matrix:
install:
- pip install tox
- if [ "$TOX_ENV" == "first_startup" ]; then bash -c "bash scripts/common_startup.sh && wget -q https://github.com/jmchilton/galaxy-downloads/raw/master/db_gx_rev_0127.sqlite && mv db_gx_rev_0127.sqlite database/universe.sqlite && bash manage_db.sh -c ./config/galaxy.ini.sample upgrade"; fi
- if [ "$TOX_ENV" == "first_startup" ]; then bash -c "bash scripts/common_startup.sh && wget -q https://github.com/jmchilton/galaxy-downloads/raw/master/db_gx_rev_0127.sqlite && mv db_gx_rev_0127.sqlite database/universe.sqlite && bash manage_db.sh -c ./config/galaxy.yml.sample upgrade"; fi
script: tox -e $TOX_ENV
Makefile
+18
View File
@@ -93,6 +93,24 @@ reports-config-lint: ## lint reports YAML configuration file
reports-config-rebuild-rst: ## Rebuild sample reports RST docs
$(CONFIG_MANAGE) build_rst reports > doc/source/admin/reports_options.rst
config-validate: ## validate galaxy YAML configuration file
$(CONFIG_MANAGE) validate galaxy
config-convert-dry-run: ## convert old style galaxy ini to yaml (dry run)
$(CONFIG_MANAGE) convert galaxy --dry-run
config-convert: ## convert old style galaxy ini to yaml
$(CONFIG_MANAGE) convert galaxy
config-rebuild-sample: ## Rebuild sample galaxy yaml file from schema
$(CONFIG_MANAGE) build_sample_yaml galaxy --add-comments
config-lint: ## lint galaxy YAML configuration file
$(CONFIG_MANAGE) lint galaxy
config-rebuild-rst: ## Rebuild sample galaxy RST docs
$(CONFIG_MANAGE) build_rst galaxy > doc/source/admin/galaxy_options.rst
release-ensure-upstream: ## Ensure upstream branch for release commands setup
ifeq (shell git remote -v | grep $(RELEASE_UPSTREAM), )
git remote add $(RELEASE_UPSTREAM) git@github.com:galaxyproject/galaxy.git
client/galaxy/scripts/mvc/library/library-dataset-view.js
+6 -1
View File
@@ -409,6 +409,11 @@ var LibraryDatasetView = Backbone.View.extend({
ld.set("misc_info", new_info);
is_changed = true;
}
var new_message = this.$el.find(".input_dataset_message").val();
if (typeof new_message !== "undefined" && new_message !== ld.get("message")) {
ld.set("message", new_message);
is_changed = true;
}
var new_genome_build = this.select_genome.$el.select2("data").id;
if (typeof new_genome_build !== "undefined" && new_genome_build !== ld.get("genome_build")) {
ld.set("genome_build", new_genome_build);
@@ -972,7 +977,7 @@ var LibraryDatasetView = Backbone.View.extend({
"</tr>",
"<tr>",
'<th scope="row">Message</th>',
'<td scope="row"><%= _.escape(item.get("message")) %></td>',
'<td scope="row"><input class="input_dataset_message form-control" type="text" placeholder="message" value="<%= _.escape(item.get("message")) %>"></td>',
"</tr>",
"<tr>",
'<th scope="row">Misc. blurb</th>',
client/galaxy/scripts/mvc/tool/tool-form.js
+9 -3
View File
@@ -136,16 +136,22 @@ var View = Backbone.View.extend({
// remap feature
if (options.job_id && options.job_remap) {
if (options.job_remap === "job_produced_collection_elements") {
var label = "Replace elements in collection ?";
var help = "The previous run of this tool failed. Use this option to replace the failed element(s) in the dataset collectio that were produced during the previous tool run.";
} else {
var label = "Resume dependencies from this job ?";
var help = "The previous run of this tool failed and other tools were waiting for it to finish successfully. Use this option to resume those tools using the new output(s) of this tool run.";
}
options.inputs.push({
label: "Resume dependencies from this job",
label: label,
name: "rerun_remap_job_id",
type: "select",
display: "radio",
ignore: "__ignore__",
value: "__ignore__",
options: [["Yes", options.job_id], ["No", "__ignore__"]],
help:
"The previous run of this tool failed and other tools were waiting for it to finish successfully. Use this option to resume those tools using the new output(s) of this tool run."
help: help,
});
}
client/galaxy/scripts/mvc/tours.js
+11 -1
View File
@@ -34,7 +34,7 @@ Select any tour to get started (and remember, you can click 'End Tour' at any ti
<a href="/tours/<%- tour.id %>" class="tourItem" data-tour.id=<%- tour.id %>>
<%- tour.name || tour.id %>
</a>
- <%- tour.attributes.description || \"No description given.\" %>
- <%- tour.attributes.description || "No description given." %>
<% _.each(tour.attributes.tags, function(tag) { %>
<span class="label label-primary sm-label-pad">
<%- tag.charAt(0).toUpperCase() + tag.slice(1) %>
@@ -83,6 +83,16 @@ var hooked_tour_from_data = data => {
.trigger("change");
};
}
if (step.path) {
// Galaxy does *not* support automagic path navigation right now in
// Tours -- too many ways to get your client 'stuck' in automatic
// navigation loops. We can probably re-enable this as our client
// routing matures.
console.warn(
"This Galaxy Tour is attempting to use path navigation. This is known to be unstable and can possibly get the Galaxy client 'stuck' in a tour, and at this time is not allowed."
);
delete step.path;
}
});
return data;
};
config/datatypes_conf.xml.sample
+5
View File
@@ -16,6 +16,10 @@
<display file="igb/bam.xml" />
<display file="iobio/bam.xml" />
</datatype>
<datatype extension="bam_native" type="galaxy.datatypes.binary:BamNative" mimetype="application/octet-stream" display_in_upload="true" description="A binary file compressed in the BGZF format with a '.bam' file extension." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#BAM">
<converter file="bam_to_bigwig_converter.xml" target_datatype="bigwig"/>
<converter file="bam_native_to_bam_converter.xml" target_datatype="bam"/>
</datatype>
<datatype extension="cram" type="galaxy.datatypes.binary:CRAM" mimetype="application/octet-stream" display_in_upload="true" description="CRAM is a file format for highly efficient and tunable reference-based compression of alignment data." description_url="http://www.ebi.ac.uk/ena/software/cram-usage">
<converter file="cram_to_bam_converter.xml" target_datatype="bam"/>
</datatype>
@@ -284,6 +288,7 @@
<datatype extension="qual454" type="galaxy.datatypes.qualityscore:QualityScore454" display_in_upload="true"/>
<datatype extension="roadmaps" type="galaxy.datatypes.assembly:Roadmaps" display_in_upload="false"/>
<datatype extension="sam" type="galaxy.datatypes.tabular:Sam" display_in_upload="true">
<converter file="sam_to_bam_native.xml" target_datatype="bam_native"/>
<converter file="sam_to_bam.xml" target_datatype="bam"/>
<converter file="sam_to_bigwig_converter.xml" target_datatype="bigwig"/>
</datatype>
config/galaxy.ini.sample
-1448
View File
File diff suppressed because it is too large Load Diff
config/galaxy.yml.sample
+1665
View File
File diff suppressed because it is too large Load Diff
config/job_conf.xml.sample_advanced
+17
View File
@@ -542,6 +542,23 @@
itself.
-->
</destination>
<!-- Following three destinations demonstrate setting up per-job temp directory handling.
In these cases TEMP, TMP, and TMPDIR will be set for each job dispatched to these
destinations.
The first simply tells Galaxy to create a temp directory in the job directory, the
other forms can be used to issue shell commands before the job runs on the worker node to
allocate a temp directory. In these other cases, Galaxy will not clean up these
directories so either use directories managed by the job resource manager or setup
tooling to clean old temp directories up outside of Galaxy. -->
<destination id="clean_tmp_by_job" runner="drmma">
<param id="tmp_dir">True</param>
</destination>
<destination id="clean_tmp_drm" runner="drmma">
<param id="tmp_dir">"$DRM_SET_VARIABLES_FOR_THIS_JOB"</param>
</destination>
<destination id="clean_tmp_fast_scratch" runner="drmma">
<param id="tmp_dir">$(mktemp -d /mnt/scratch/fastest/gxyjobXXXXXXXXXXX)</param>
</destination>
<destination id="real_user_cluster" runner="drmaa">
<!-- Make sure to setup 3 real user parameters in galaxy.ini. -->
</destination>
doc/Makefile
+7 -1
View File
@@ -15,7 +15,7 @@ ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) sou
# the i18n builder cannot share the environment and doctrees with the others
I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source
GENERATED_RST = source/api/api.rst source/api/ts_api.rst source/dev/schema.rst
GENERATED_RST = source/api/api.rst source/api/ts_api.rst source/dev/schema.rst source/admin/config_logging_default_yaml.rst
.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext updaterst
@@ -61,6 +61,12 @@ source/dev/schema.rst: schema.md ## Convert Galaxy Tool XSD Markdown docs into r
pandoc schema.md -f markdown_github-hard_line_breaks -s -o $@
./fix_schema_rst.sh $@
source/admin/config_logging_default_yaml.rst: ../lib/galaxy/config.py
printf '.. code-block:: yaml\n\n' > $@
printf ' galaxy:\n' >> $@
printf ' logging:\n ' >> $@
PYTHONPATH=../lib ../.venv/bin/python -c 'import yaml, galaxy.config; print "\n ".join(yaml.dump(galaxy.config.LOGGING_CONFIG_DEFAULT, indent=4, default_flow_style=False).splitlines())' >> $@
# might also want to do
# cd source/lib; hg revert; rm *.rst.orig; or not.
clean:
doc/source/admin/_inc_proxy_prereq.rst
+22
View File
@@ -0,0 +1,22 @@
This documentation should be used in conjunction with the `Scaling and Load Balancing` documentation, which you should
familiarize yourself with prior to setting up your proxy.
You will need to ensure that inbound (and outbound) traffic on the HTTP (TCP port 80) and HTTPS (TCP port 443) ports is
permitted by your server's firewall/security.
**Documentation Conventions:**
For the purposes of this example, we assume that:
- **Debian** refers to any *Debian*-based Linux distribution (including Ubuntu)
- **EL** refers to any *RedHat Enterprise Linux*-based Linux distribution (including CentOS)
- the Galaxy server is installed at ``/srv/galaxy/server``
- |PROXY| runs as the user ``www-data`` (this is the default under Debian)
- Galaxy runs as the user ``galaxy`` with primary group ``galaxy``
- Galaxy is served from the hostname ``galaxy.example.org``
Throughout the configuration examples in this document, in order to avoid repetition, ``#...`` is used to denote a
location where existing or previously given configuration statements would appear.
.. danger:: Please note that Galaxy's files - code, datasets, and so forth - should *never* be located on disk inside
|PROXY|'s document root. By default, this would expose all of Galaxy (including datasets) to anyone on the web.
doc/source/admin/_inc_proxy_serving_root.rst
+10
View File
@@ -0,0 +1,10 @@
This configuration assumes that Galaxy will be the only site on your server using the given hostname (e.g.
``https://galaxy.example.org``).
Beginning with Galaxy Release 18.01, the default application server that Galaxy runs under is uWSGI. Because of this,
the native high performance uWSGI protocol should be used for communication between |PROXY| and Galaxy, rather
than HTTP. Legacy instructions for proxying via HTTP can be found in the `Galaxy Release 17.09 proxy documentation`_.
Since |PROXY| is more efficient than uWSGI at serving static content, it is best to serve it directly, reducing the load
on the Galaxy process and allowing for more effective compression (if enabled), caching, and pipelining. Directives to
do so are included in the example below.
doc/source/admin/_inc_proxy_ssl.rst
+25
View File
@@ -0,0 +1,25 @@
The use of SSL is **strongly encouraged** to avoid exposure of confidential information such as datasets and user
credentials to eavesdroppers. The instructions in this document are for setting up an SSL-enabled Galaxy server.
When setting up an SSL server, simply enabling SSL with the default options is not enough to have a secure server. In
most cases, the configuration is weak and vulnerable to one or more of the multitude of SSL attacks that have been
recently prevalent. The `Qualys SSL/TLS Deployment Best Practices`_ is an excellent and up-to-date guide covering
everything necessary for securing an SSL server. In addition, the `Mozilla SSL Configuration Generator`_ can provide you
with a best practices config tailored to your desired security level and software versions.
Finally, Google's `PageSpeed Insights`_ tool is helpful for determining how you can improve responsiveness as related to
proxying, such as verifying that caching and compression are configured properly.
If you need to run more than one site on your Galaxy server, there are two options:
- Run them on the same server but serve them on different hostnames
- Serve them from different URL prefixes on a single hostname
The former option is typically cleaner, but if serving more than one SSL site, you will need an SSL certificate with
subjectAltNames_ for each hostname served by the server.
.. _Qualys SSL/TLS Deployment Best Practices: https://www.ssllabs.com/projects/best-practices/
.. _Mozilla SSL Configuration Generator: https://mozilla.github.io/server-side-tls/ssl-config-generator/
.. _PageSpeed Insights: https://developers.google.com/speed/pagespeed/insights/
.. _subjectAltNames: http://wiki.cacert.org/FAQ/subjectAltName
doc/source/admin/apache.md
+341
View File
@@ -0,0 +1,341 @@
```eval_rst
.. |PROXY| replace:: Apache
```
# Proxying Galaxy with Apache
In a production environment, it is recommended to run Galaxy behind a proxy web server for performance and security
reasons. The proxy server sits between clients and your Galaxy server, relaying requests between them and offloading
some of the more menial and resource-intensive tasks.
[The Apache HTTP Server][apache] is a widely deployed and very featureful general purpose web server with mature
proxying capabilities.
Instructions for [proxying with NGINX](nginx.html), which is the proxy server used by The Galaxy Project's public
servers, [usegalaxy.org][main] ("Main") and [Test][test], as well as the [Docker Galaxy project][docker-galaxy], are
also available.
[apache]: https://httpd.apache.org/
[main]: https://galaxyproject.org/main/
[test]: https://galaxyproject.org/test/
[docker-galaxy]: https://github.com/bgruening/docker-galaxy-stable
## Prerequisites
```eval_rst
.. include:: _inc_proxy_prereq.rst
```
### Apache Proxy Prerequisites
Currently, the only recommended way to proxy Galaxy with Apache is using `mod_rewrite`, `mod_proxy`, and
`mod_proxy_uwsgi`. These modules must be enabled in the Apache config. The main proxy directives, `ProxyRequests` and
`ProxyVia` do **not** need to be enabled.
Additionally, these directions are written for Apache 2.4+. Apache 2.4 for EL 6 can be obtained from the [CentOS SCLo
SIG Repo][sclo-sig-repo]. Otherwise, your system package manager's version of Apache should be suitable. On EL, you will
need to enable the [EPEL][epel] repository to obtain the `mod_proxy_uwsgi` package.
```eval_rst
.. caution:: ``mod_uwsgi`` is not the same module as ``mod_proxy_uwsgi``. The former is the old and unsupported module.
Be sure that you have installed ``mod_proxy_uwsgi``.
```
Ensure that the `mod_headers`, `mod_rewrite`, `mod_proxy`, and `mod_proxy_uwsgi` modules are loaded. Although not
required, the configuration examples also use `mod_deflate` and `mod_expires` for increased client/server performance,
so these should also be enabled.
On Debian you can install the necessary packages and enable the modules this with the following:
```shell-session
# apt-get install apache2 libapache2-mod-proxy-uwsgi
# a2enmod headers deflate expires rewrite proxy proxy_uwsgi
Enabling module headers.
Considering dependency filter for deflate:
Module filter already enabled
Module deflate already enabled
Enabling module expires.
Enabling module rewrite.
Enabling module proxy.
Considering dependency proxy for proxy_uwsgi:
Module proxy already enabled
Enabling module proxy_uwsgi.
To activate the new configuration, you need to run:
service apache2 restart
```
And on EL:
```shell-session
# yum install httpd mod_proxy_uwsgi
# echo "LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so" > /etc/httpd/conf.modules.d/10-proxy-uwsgi.conf
```
[sclo-sig-repo]: https://wiki.centos.org/SpecialInterestGroup/SCLo/CollectionsList
[epel]: https://fedoraproject.org/wiki/EPEL
## Basic configuration
```eval_rst
.. include:: _inc_proxy_ssl.rst
```
### Serving Galaxy at the Web Server Root
```eval_rst
.. include:: _inc_proxy_serving_root.rst
.. _Galaxy Release 17.09 Proxy Documentation: https://docs.galaxyproject.org/en/release_17.09/admin/special_topics/apache.html
```
The following configuration is not exhaustive, only the portions most relevant to serving Galaxy are shown, these should
be incorporated with your existing/default Apache config as is appropriate for your server. Notably, the Apache package
you installed most likely has a multi-file config layout. If you are not already familiar with that layout and where
best to place your configuration, you can learn more in the [Proxy Package Layouts](proxy_package_layouts.html)
documentation.
```apache
SSLProtocol all -SSLv3
SSLCipherSuite ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA256
SSLHonorCipherOrder on
SSLCompression off
SSLSessionTickets off
# OCSP stapling
SSLUseStapling on
SSLStaplingResponderTimeout 5
SSLStaplingReturnResponderErrors off
SSLStaplingCache shmcb:/var/run/ocsp(128000)
<VirtualHost _default_:80>
Redirect permanent / https://galaxy.example.org
</VirtualHost>
<VirtualHost _default_:443>
SSLEngine on
SSLCertificateFile /etc/apache2/ssl/server.crt
SSLCertificateKeyFile /etc/apache2/ssl/server.key
# Enable HSTS
Header always set Strict-Transport-Security "max-age=15552000; includeSubdomains"
# use a variable for convenience
Define galaxy_root /srv/galaxy/server
# don't decode encoded slashes in path info
AllowEncodedSlashes NoDecode
# enable compression on all relevant types
AddOutputFilterByType DEFLATE text/html text/plain text/xml
AddOutputFilterByType DEFLATE text/css
AddOutputFilterByType DEFLATE application/x-javascript application/javascript application/ecmascript
AddOutputFilterByType DEFLATE application/rss+xml
AddOutputFilterByType DEFLATE application/xml
AddOutputFilterByType DEFLATE application/json
# allow access to static content
<Directory "${galaxy_root}/static">
AllowOverride None
Require all granted
</Directory>
# Galaxy needs to know that this is https for generating URLs
RequestHeader set X-URL-SCHEME "%{REQUEST_SCHEME}e"
# allow up to 3 minutes for Galaxy to respond to slow requests before timing out
ProxyTimeout 180
# proxy all requests not matching other locations to uWSGI
ProxyPass / unix:///srv/galaxy/var/uwsgi.sock|uwsgi://
# or uWSGI on a TCP socket
#ProxyPass / uwsgi://127.0.0.1:4001/
# serve framework static content
RewriteEngine On
RewriteRule ^/static/style/(.*) ${galaxy_root}/static/style/blue/$1 [L]
RewriteRule ^/static/(.*) ${galaxy_root}/static/$1 [L]
RewriteRule ^/favicon.ico ${galaxy_root}/static/favicon.ico [L]
RewriteRule ^/robots.txt ${galaxy_root}/static/robots.txt [L]
# enable caching on static content
<Location "/static">
ExpiresActive On
ExpiresDefault "access plus 24 hours"
</Location>
# serve visualization and interactive environment plugin static content
<Directory "${galaxy_root}/config/plugins/(.+)/(.+)/static">
AllowOverride None
Require all granted
</Directory>
RewriteRule ^/plugins/(.+)/(.+)/static/(.*)$ ${galaxy_root}/config/plugins/$1/$2/static/$3 [L]
</VirtualHost>
```
Be sure to set `galaxy_root` to the path to your copy of Galaxy and modify the value of `ProxyPass /` to match your
uWSGI socket path. With the default configuration, uWSGI will bind to a random TCP socket, so you will need to set it to
a fixed value as described in the [Scaling and Load Balancing](scaling.html) documentation. If using a UNIX domain
socket, be sure to pay particular attention to the discussion of users and permissions.
### Additional Notes
- **Do not** simply copy the SSL configuration directives and expect them to work on your server or to be secure! These
are provided as examples of some of the best practices as of the time of writing, but will not always be up to date.
Use the guides referenced in [basic configuration](#basic-configuration) section to configure SSL properly.
- If your existing Apache configuration contains a line or included config file defining a default server, be sure to
disable it by commenting its `<VirtualHost>` or preventing its inclusion (under Debian this is done by removing its
symlink from `/etc/apache2/sites-enabled`).
- `ProxyTimeout` can be adjusted as appropriate for your site. This is the amount of time allowed for communication
between Apache and uWSGI to block while waiting for a response from Galaxy, and is useful for holding client (browser)
connections while uWSGI is restarting Galaxy subprocesses or Galaxy is performing a slow operation.
- If your Apache server is set up to use `mod_security`, you may need to modify the value of the `SecRequestBodyLimit`.
The default value on some systems will limit uploads to only a few kilobytes.
- Some Galaxy URLs contain encoded slashes (%2F) in the path and Apache will not serve these URLs by default, which is
the reason for inclusion of the `AllowEncodedSlashes` directive. Note: The `NoDecode` value was added in Apache2
2.2.18, which is newer than EL 6's provided 2.2.15.
- If you must serve Galaxy without SSL, you would simply replace the `443` with `80` in the SSL `VirtualHost` block
and remove the non-SSL block and all SSL directives.
- If the proxy works but you are getting 404 errors for Galaxy's static content, be sure that the user that Apache runs
as has access to Galaxy's `static/` directory (and all its parent directories) on the filesystem. You can test this on
the command line with e.g. `sudo -u www-data ls /srv/galaxy/server/static`.
### Serving Galaxy at a URL Prefix
It may be necessary to serve Galaxy from an address other than the web server root (`https://www.example.org/galaxy`),
instead of `https://galaxy.example.org`). To do this, you need to make the following changes to the configuration in the
previous section:
1. In the Apache config, prefix all of the location directives with your prefix, like so:
```apache
#...
# proxy all requests not matching other locations to uWSGI
ProxyPass /galaxy unix:///srv/galaxy/var/uwsgi.sock|uwsgi://
# or uWSGI on a TCP socket
#ProxyPass /galaxy uwsgi://127.0.0.1:4001/
# serve framework static content
RewriteEngine On
RewriteRule ^/galaxy/$ /galaxy [R,L]
RewriteRule ^/galaxy/static/style/(.*) ${galaxy_root}/static/style/blue/$1 [L]
RewriteRule ^/galaxy/static/(.*) ${galaxy_root}/static/$1 [L]
RewriteRule ^/galaxy/favicon.ico ${galaxy_root}/static/favicon.ico [L]
RewriteRule ^/galaxy/robots.txt ${galaxy_root}/static/robots.txt [L]
```
2. The Galaxy application needs to be aware that it is running with a prefix (for generating URLs in dynamic pages).
This is accomplished by configuring uWSGI and Galaxy (the `uwsgi` and `galaxy` sections in `config/galaxy.yml`
respectively) like so and restarting Galaxy:
```yaml
uwsgi:
#...
socket: unix:///srv/galaxy/var/uwsgi.sock
mount: /galaxy=galaxy.webapps.galaxy.buildapp:uwsgi_app()
manage-script-name: true
# `module` MUST NOT be set when `mount` is in use
#module: galaxy.webapps.galaxy.buildapp:uwsgi_app()
galaxy:
#...
cookie_path: /galaxy
```
`cookie_path` should be set to prevent Galaxy's session cookies from clobbering each other if you are running more
than one instance of Galaxy under different URL prefixes on the same hostname.
Be sure to consult the [Scaling and Load Balancing](scaling.html) documentation, other options unrelated to proxying
should also be set in the `uwsgi` section of the config.
## Advanced Configuration Topics
### Sending Files With Apache
Galaxy sends files (e.g. dataset downloads) by opening the file and streaming it in chunks through the proxy server.
However, this ties up the Galaxy process, which can impact the performance of other operations (see [Production Server
Configuration](production.html) for a more in-depth explanation).
Apache can assume this task instead and as an added benefit, speed up downloads. This is accomplished through the use of
`mod_xsendfile`, a 3rd-party Apache module. Dataset security is maintained in this configuration because Apache will
still check with Galaxy to ensure that the requesting user has permission to access the dataset before sending it.
To enable it, you must first install `mod_xsendfile`. This is usually available via your package manager
(`libapache2-mod-xsendfile` on Debian and `mod_xsendfile` from EPEL on EL). Once installed, add the appropriate
`LoadModule` directive to your Apache configuration (`LoadModule xsendfile_module /path/to/mod_xsendfile.so`, but both
the Debian and EPEL packages do this for you upon installation).
The, add `XSendFile` directives to your proxy configuration:
```apache
<Location "/">
XSendFile on
XSendFilePath /
</Location>
```
Next, edit `galaxy.yml` and make the following change before restarting Galaxy:
```yaml
galaxy:
# ...
apache_xsendfile: true
```
For this to work, the user under which your Apache server runs will need read access to Galaxy's `files_path` directory
(by default, `database/files/`) and its contents. This is most easily done by adding the Apache user to the Galaxy user's
primary group and setting the `umask(2)` to create files with the group read permission set. If you start Galaxy from
the command line, you can do this like so:
```shell-session
admin@server$ sudo usermod -a -G galaxy www-data # add `www-data` user to `galaxy` group
admin@server$ sudo -iu galaxy
galaxy@server$ umask 027
galaxy@server$ sh run.sh
```
If you start Galaxy from supervisord, you can set the `umask` option in the [program
section](http://supervisord.org/configuration.html#program-x-section-settings) after adding the Apache user to the Galaxy
group as shown above.
### External user authentication
- [Apache for External Authentication](https://galaxyproject.org/admin/config/apache-external-user-auth/)
- [Built-in Galaxy External Authentication](authentication.html)
#### Display Sites
Display sites such as UCSC work not by sending data directly from Galaxy to UCSC via the client's browser, but by
sending UCSC a URL to the data in Galaxy that the UCSC server will retrieve data from. Since enabling authentication
will place **all** of Galaxy behind authentication, such display sites will no longer be able to access data via that
URL. If `display_servers` is set to a non-empty value in `$galaxy_root/config/galaxy.yml`, this tells Galaxy it should
allow the named servers access to data in Galaxy. However, you still need to configure Apache to allow access to the
datasets. An example config is provided here that allows the UCSC Main/Test backends:
```apache
<Location "/root/display_as">
Satisfy Any
Order deny,allow
Deny from all
Allow from hgw1.cse.ucsc.edu
Allow from hgw2.cse.ucsc.edu
Allow from hgw3.cse.ucsc.edu
Allow from hgw4.cse.ucsc.edu
Allow from hgw5.cse.ucsc.edu
Allow from hgw6.cse.ucsc.edu
Allow from hgw7.cse.ucsc.edu
Allow from hgw8.cse.ucsc.edu
</Location>
```
**PLEASE NOTE that this introduces a security hole** , the impact of which depends on whether you have restricted access
to the dataset via Galaxy's [internal dataset permissions](https://galaxyproject.org/learn/security-features/).
- By default, data in Galaxy is public. Normally with a Galaxy server behind authentication in a proxy server this is of
little concern since only clients who've authenticated can access Galaxy. However, if display site exceptions are made
as shown above, anyone could use those public sites to bypass authentication and view any **public** dataset on your
Galaxy server. If you have not changed from the default and most of your datasets are public, you should consider
running your own display sites that are also behind authentication rather than using the public ones.
- For datasets for which access has been restricted to one or more roles (i.e. it is no longer "public"), access for
reading via external browsers is only allowed for a brief period, when someone with access permission clicks the
"display at..." link. During this period, anyone who has the dataset ID would then be able to use the browser to view
this dataset. Although such a scenario is unlikely, it is technically possible.
doc/source/admin/authentication.md
+16 -15
View File
@@ -33,8 +33,8 @@ of documentation on configuring these plugins.
[OpenID](https://en.wikipedia.org/wiki/OpenID) is becoming less popular and probably shouldn't be used the primary mechanism
for authentication in Galaxy but it is an available option.
Enabling OpenID requires you to edit Galaxy's configuration file and set `enable_openid` to `True`. This file is
likely located in `config/galaxy.ini` and can be created by copying Galaxy's sample `config/galaxy.ini.sample`.
Enabling OpenID requires you to edit Galaxy's configuration file and set `enable_openid` to `true`. This file is
likely located in `config/galaxy.yml` and can be created by copying Galaxy's sample `config/galaxy.yml.sample`.
Enabling this option enables OpenID and causes the OpenID form to be displayed on the login screen.
@@ -55,8 +55,8 @@ and the "external" flag should itself prohibit the traditional mechanism being u
user returns to Galaxy and is not already logged in, the details of the user are retrieved according to the identity
information supplied by the Web server.
Enabling remote user authentication requires you to edit Galaxy's configuration file and set `use_remote_user` to `True`.
This file is likely located in `config/galaxy.ini` and can be created by copying Galaxy's sample `config/galaxy.ini.sample`.
Enabling remote user authentication requires you to edit Galaxy's configuration file and set `use_remote_user` to `true`.
This file is likely located in `config/galaxy.yml` and can be created by copying Galaxy's sample `config/galaxy.yml.sample`.
Additional Galaxy configuration options related to remote user authentication are documented in Galaxy's sample
configuration file. The options ``remote_user_maildomain``, ``remote_user_header``, and ``normalize_remote_user_email`` can
@@ -75,29 +75,30 @@ How to set up this config is presented here.
### Account activation feature
In the Galaxy config file **config/galaxy.ini** there is the user activation setting that you have to turn on.
In the Galaxy config file **config/galaxy.yml** there is the user activation setting that you have to turn on.
```
user_activation_on = True
```yaml
user_activation_on: true
```
There is also the option for tracking jobs in database that is required to be turned on for the account activation to be effective. By default it is off.
```
track_jobs_in_database = True
```yaml
track_jobs_in_database: true
```
After you turn on both of these every user that will try to register after this configuration file takes effect will have the verification email sent to the email address provided. Unless the Grace period (see below) is set, the user won't be able to login before the verification happens.
Furthermore in order for this to work correctly smtp server and admin email should be set:
```
#smtp_server = some.server.edu:587
#smtp_username = example_username
#smtp_password = example_passsword
#activation_email = activation-noreply@example.com
#error_email_to = admin@example.com
```yaml
smtp_server: some.server.edu:587
smtp_username: example_username
smtp_password: example_passsword
activation_email: activation-noreply@example.com
error_email_to: admin@example.com
```
Smtp server takes care of the email sending and the activation_email email is used as the *From* address in the verification email. Furthermore the error_email_to is being shown to the user if the Galaxy detects its own misconfiguration.
doc/source/admin/cluster.md
+10 -10
View File
@@ -53,20 +53,20 @@ galaxy_user@node1%
```
If your cluster nodes have Internet access (NAT is okay) and you want to run the data source tools (upload, ucsc, etc.) on the cluster (doing so is highly recommended), set `new_file_path` in `galaxy.ini` to a directory somewhere in your shared filesystem:
If your cluster nodes have Internet access (NAT is okay) and you want to run the data source tools (upload, ucsc, etc.) on the cluster (doing so is highly recommended), set `new_file_path` in `galaxy.yml` to a directory somewhere in your shared filesystem:
```ini
new_file_path = /clusterfs/galaxy/tmp
```yaml
new_file_path: /clusterfs/galaxy/tmp
```
Additionally some of the runners including DRMAA may use the ``cluster_files_directory`` for sharing files with the cluster, which defaults to ``database/pbs``. You may need to create this folder.
```ini
cluster_files_directory = database/pbs
```yaml
cluster_files_directory: database/pbs
```
You may also find that attribute caching in your filesystem causes problems with job completion since it interferes with Galaxy detecting the presence and correct sizes of output files. In NFS caching can be disabled with the `-noac` mount option on Linux (on the Galaxy server), but this may have a significant impact on performance since all attributes will have to be read from the file server upon every file access. You should try the `retry_output_collection` option in `galaxy.ini` first to see if this solves the problem.
You may also find that attribute caching in your filesystem causes problems with job completion since it interferes with Galaxy detecting the presence and correct sizes of output files. In NFS caching can be disabled with the `-noac` mount option on Linux (on the Galaxy server), but this may have a significant impact on performance since all attributes will have to be read from the file server upon every file access. You should try the `retry_output_collection` option in `galaxy.yml` first to see if this solves the problem.
## Runner Configuration
@@ -336,9 +336,9 @@ The directory specified in `new_file_path` in the Galaxy config should be world-
The `outputs_to_working_directory` option in the Galaxy config **must** be set to `True`. This ensures that a tool/job's outputs are written to the temporary working directory, which (when using the real user system) is owned by the real user who submitted the job. If left set to the default (`False`), the tool will attempt to write directly to the directory specified in `file_path` (by default, `galaxy-app/database/files/`), which must be owned by the Galaxy user (and thus will not be writable by the real user).
For releases later than 17.05 you can configure the method how the system user is determined in `config/galaxy.ini` via the variable `real_system_username`. For determining the system user from the email adress stored in Galaxy set it to `user_email`, otherwise for determining the system user from the Galaxy user name set it to `username`.
For releases later than 17.05 you can configure the method how the system user is determined in `config/galaxy.yml` via the variable `real_system_username`. For determining the system user from the email adress stored in Galaxy set it to `user_email`, otherwise for determining the system user from the Galaxy user name set it to `username`.
Once these are set, you must set the `drmaa_external_*` and `external_chown_script` settings in the Galaxy config and configure `sudo(8)` to allow them to be run. A sudo config using the three scripts set in the sample `galaxy.ini` would be:
Once these are set, you must set the `drmaa_external_*` and `external_chown_script` settings in the Galaxy config and configure `sudo(8)` to allow them to be run. A sudo config using the three scripts set in the sample `galaxy.yml` would be:
```
galaxy ALL = (root) NOPASSWD: SETENV: /opt/galaxy/scripts/drmaa_external_runner.py
@@ -348,8 +348,8 @@ galaxy ALL = (root) NOPASSWD: SETENV: /opt/galaxy/scripts/external_chown_script
If your sudo config contains `Defaults requiretty`, this option must be disabled.
For Galaxy releases > 17.05 the sudo call has been moved to `galaxy.ini` and is thereby configurable by the Galaxy admin. This can be of interest because sudo removes `PATH`, `LD_LIBRARY_PATH`, etc. variables per default in some installations. In such cases the sudo calls in the three variables in galaxy.ini can be adapted, e.g., `sudo -E PATH=... LD_LIBRARY_PATH=... /PATH/TO/GALAXY/scripts/drmaa_external_runner.py`. In order to allow setting the variables this way adaptions to the sudo configuration might be necessary.
Also for Galaxy releases > 17.05: In order to allow `external_chown_script.py` to chown only path below certain entry points the variable `ALLOWED_PATHS` in the python script can be adapted. It is sufficient to include the directorries `job_working_directory` and `new_file_path` as configured in `galaxy.ini`.
For Galaxy releases > 17.05, the sudo call has been moved to `galaxy.yml` and is thereby configurable by the Galaxy admin. This can be of interest because sudo removes `PATH`, `LD_LIBRARY_PATH`, etc. variables per default in some installations. In such cases the sudo calls in the three variables in galaxy.yml can be adapted, e.g., `sudo -E PATH=... LD_LIBRARY_PATH=... /PATH/TO/GALAXY/scripts/drmaa_external_runner.py`. In order to allow setting the variables this way adaptions to the sudo configuration might be necessary.
Also for Galaxy releases > 17.05: In order to allow `external_chown_script.py` to chown only path below certain entry points the variable `ALLOWED_PATHS` in the python script can be adapted. It is sufficient to include the directorries `job_working_directory` and `new_file_path` as configured in `galaxy.yml`.
It is also a good idea to make sure that only trusted users, e.g. root, have write access to all three scripts.
doc/source/admin/conda_faq.rst
+21 -16
View File
@@ -4,6 +4,11 @@
Conda for Tool Dependencies
===========================
.. note:: This document describes configuring Galaxy using YAML based configuraiton
options. For Galaxy instances before version 18.01, `this variant
<https://docs.galaxyproject.org/en/release_17.09/admin/conda_faq.html>`__ of this
document will be more directly relatable.
Galaxy tools (also called wrappers) have tradionally used Tool Shed package
recipes to install their dependencies. These were too tightly tied to Galaxy
and to the Tool Shed and so have been replaced with Conda as the package
@@ -63,19 +68,19 @@ entry should remain first. This means that tools that have specified Tool Shed p
as their dependencies will work without a change.
The most common configuration settings related to Conda are listed in Table 1.
See `galaxy.ini.sample`_ for the complete list.
See `galaxy.yml.sample`_ for the complete list.
+-------------------------+------------------------------------+---------------------------+
| Setting | Default setting | Meaning |
+-------------------------+------------------------------------+---------------------------+
| ``conda_auto_init`` | ``True`` | If ``True``, Galaxy will |
| ``conda_auto_init`` | ``true`` | If ``true``, Galaxy will |
| | | try to install Conda |
| | | (the package manager) |
| | | automatically if it |
| | | cannot find a local copy |
| | | already on the system |
+-------------------------+------------------------------------+---------------------------+
| ``conda_auto_install`` | ``False`` | If ``True``, Galaxy will |
| ``conda_auto_install`` | ``false`` | If ``true``, Galaxy will |
| | | look for and install |
| | | Conda packages for |
| | | missing tool dependencies |
@@ -99,8 +104,8 @@ admin to install and manage packages without touching Galaxy at all. Galaxy can
handle these dependencies for you, but admins are not required to use Galaxy for
dependency management.
There are a few new config options in the ``galaxy.ini`` file (see Table 1 or
`galaxy.ini.sample`_ for more information), but by default Galaxy will install
There are a few new config options in the ``galaxy.yml`` file (see Table 1 or
`galaxy.yml.sample`_ for more information), but by default Galaxy will install
Conda (the package manager) and the required packages in the
``<tool_dependency_dir>/_conda/`` directory. In this directory, Galaxy will
create an ``envs`` folder with all of the environments managed by Galaxy. Each
@@ -111,7 +116,7 @@ available before the tool is executed on your system.
To summarize, there are four ways to manage Conda dependencies for use
with Galaxy. For all of these options, Conda dependency management must
be configured in the ``dependency_resolvers_conf.xml`` and the ``galaxy.ini`` file.
be configured in the ``dependency_resolvers_conf.xml`` and the ``galaxy.yml`` file.
#. Galaxy Admin Interface (>= 16.07) - Galaxy will install Conda tool
dependencies when tools are installed from the Tool Shed if the
@@ -121,7 +126,7 @@ be configured in the ``dependency_resolvers_conf.xml`` and the ``galaxy.ini`` fi
#. Manual Install - Conda dependencies may be installed by
administrators from the command line. Conda (and thus the Conda
environments) should be installed in the location specified by the
``conda_prefix`` path (defined in ``galaxy.ini`` and by default
``conda_prefix`` path (defined in ``galaxy.yml`` and by default
``<tool_dependency_dir>/_conda/`` directory). Galaxy will search
these environments for required packages when tools are run. Conda
environment names have to follow a specific naming pattern. As an
@@ -157,11 +162,11 @@ will run on all major \*nix operating systems newer than 2007.
4. If I have Conda enabled, what do I need to do to install tools using it? For example, how can I install the latest Trinity? And how will I know the dependencies are installed?
**********************************************************************************************************************************************************************************
This depends on your ``galaxy.ini`` setting. Starting with release 16.07, Galaxy
This depends on your ``galaxy.yml`` settings. Starting with release 16.07, Galaxy
can automatically install the Conda package manager for you if you have enabled
``conda_auto_init``. Galaxy can then install Trinity along with its dependencies
using one of the methods listed in question 2 above. In particular, if
``conda_auto_install`` is ``True`` and Trinity is not installed yet, Galaxy will
``conda_auto_install`` is ``true`` and Trinity is not installed yet, Galaxy will
try to install it via Conda when a Trinity job is launched.
With release 16.07 you can see which dependencies are being used
@@ -176,7 +181,7 @@ dependency resolvers configuration with regards to what will actually be used du
the tool execution.
To check if Galaxy has created a Trinity environment, have a look at folders under
``<tool_dependency_dir>/_conda/envs/`` (or ``<conda_prefix>/envs`` if you have changed ``conda_prefix`` in your galaxy.ini file).
``<tool_dependency_dir>/_conda/envs/`` (or ``<conda_prefix>/envs`` if you have changed ``conda_prefix`` in your ``galaxy.yml`` file).
We recommend to use Conda on a tool-per-tool basis, by unchecking the checkbox
for TS dependencies during the tool installation, and for tools where there
@@ -224,7 +229,7 @@ The simple answer is: you don't need to do much to make Conda work for a tool.
The ``<requirement>`` tag in the tool XML file is enough. The name and the
version should correspond to a Conda package in the ``default``, ``r``,
``bioconda`` or ``iuc`` Conda channel (you can extend this list if you
like in your ``galaxy.ini`` ). If this is the case you are ready to go. Read
like in your ``galaxy.yml`` ). If this is the case you are ready to go. Read
more about `Conda channels`_ and browse their packages on https://anaconda.org/ url followed by the channel name (e.g.
`https://anaconda.org/bioconda <https://anaconda.org/bioconda>`__
).
@@ -328,14 +333,14 @@ message appears in your logs:
You can also use: $ conda clean --lock
First, you may wish to enable cached dependencies. This can be done by setting
``use_cached_dependency_manager`` to ``True`` in ``galaxy.ini``. Without this
``use_cached_dependency_manager`` to ``true`` in ``galaxy.yml``. Without this
option, many jobs will create a per-job Conda environment with just the
dependencies needed for that job installed.
This will be placed on the filesystem containg the job working directory. This
is an expensive operation and Conda doesn't always link environments correctly
across filesystems. Enabling this dependency caching will create a cache
directory for each required combination of requirements inside the directory
specified by ``tool_dependency_cache_dir`` in ``galaxy.ini`` (defaulting to
specified by ``tool_dependency_cache_dir`` in ``galaxy.yml`` (defaulting to
``<tool_dependency_dir>/_cache``).
The cached dependency manager was added to the 16.10 release of Galaxy (see
@@ -364,8 +369,8 @@ newer version of Conda than shipped with Galaxy as of 17.01. See the question be
on upgrading Conda if you must use this trick.
Alternatively, copying can be used when creating environments instead of links (either
symbolic or hard). To enable this set ``conda_copy_dependencies`` to ``True`` in
``galaxy.ini``. This requires at least version 16.07 of Galaxy.
symbolic or hard). To enable this set ``conda_copy_dependencies`` to ``true`` in
``galaxy.yml``. This requires at least version 16.07 of Galaxy.
More reading on this can be found at `Conda Pull Request #3870`_, `Conda Issue #3308`,
and Galaxy `Issue #3193`_.
@@ -409,7 +414,7 @@ The command can obviously be adapted to install any version of Conda.
.. _submit: https://bioconda.github.io/#step-4-join-the-team
.. _BioConda: https://bioconda.github.io
.. _contact with the IUC: https://gitter.im/galaxy-iuc/iuc
.. _galaxy.ini.sample: https://github.com/galaxyproject/galaxy/blob/dev/config/galaxy.ini.sample
.. _galaxy.yml.sample: https://docs.galaxyproject.org/en/master/admin/options.html
.. _Pull Request #3106: https://github.com/galaxyproject/galaxy/pull/3106
.. _Pull Request #3348: https://github.com/galaxyproject/galaxy/pull/3348
.. _Pull Request #3391: https://github.com/galaxyproject/galaxy/pull/3391
doc/source/admin/config.rst
+120
View File
@@ -0,0 +1,120 @@
Galaxy Configuration
========================================
Overview
----------------------------
Galaxy has a large number of configuration files in an array of formats. Although it is not required to modify *any*
Galaxy configuration files to run the server, most Galaxy servers will modify at least the core configuration file.
These have grown organically over time as new features with the needs for advanced syntaxes and config isolation have
been added. An effort is under way to `standardize and unify configuration formats`_.
Configuration files can be found underneath the ``config/`` subdirectory, wherein you can find ``<name>.sample`` files
corresponding to configuration files that you can modify by copying the ``<name>.sample`` to ``<name>``. In many cases,
you will find that the sample configuration provides the most up-to-date and detailed documentation about the features
configured therein.
Most config files are administered by hand, although a few (ones which begin with ``shed_*``) are modified by Galaxy
when installing from the Galaxy Tool Shed. When starting Galaxy for the first time, these files are copied from their
sample files automatically. You should not need to modify these unless you want to change the directory in to which Tool
Shed tools are installed.
Configuration Files
----------------------------
The primary Galaxy configuration file is ``galaxy.yml``. You will need to use this file to modify core functionality
such as the port on which Galaxy listens, the directory in which Galaxy datasets are stored, the database connection
options, and so forth.
The most commonly modified configuration files include:
- ``galaxy.yml``: Core Galaxy configuration file.
- ``tool_conf.xml``: Describes the paths to local tool configurations that Galaxy should attempt to load. Tools that
are installed via the Tool Shed are configured to load in the ``shed_tool_conf.xml`` file. See the :ref:`Tool Panel
Administration <tool_panel>` and `Installing Tools into Galaxy`_ documentation for more.
- ``datatypes_conf.xml``: Describes the file formats that are supported in Galaxy. See the `Datatypes documentation`_
for more.
- ``job_conf.xml``: Controls how Galaxy runs tools, e.g. to run them on a compute cluster. See the `Cluster
documentation`_ for more.
Some configuration files are only used when adding local components, rather than ones installed from the Tool Shed:
- ``tool_conf.xml``: As described above.
- ``tool_data_table_conf.xml``: Describes the mapping between Data Tables - the structured format that allow tools to
work with locally cached reference data - and the location files that describe the actual data that is available (e.g.
paths, genome builds, etc.). Data table configurations are also provided by tools in the Tool Shed, those are
configured in ``shed_tool_data_table_conf.xml``. See the `Data Preparation documentation`_ for more.
- ``data_manager_conf.xml``: Describes the paths to local Data Managers, special Galaxy tools that automatically fetch
or create data for Data Tables and manage the corresponding data table and location configuration files. See the `Data
Managers documentation`_ for more.
- ``local_conda_mapping.yml``: Define mappings between the names specified in the tool configuration (``<requirement>``
tags) and the conda resolver's names (conda package name).
- ``lmod_modules_mapping.yml``: Define mappings between the names specified in the tool configuration (``<requirement>``
tags) and the Lmod system.
Some configuration files are used to control the way that Galaxy resolves tool dependencies. Most Galaxy tools are only
descriptions of how to run a particular command line tool, and they do not contain the dependent command line tool. The
task of locating and making available these command line tools is performed by the Galaxy tool dependencies system,
which has configuration files of its own:
Additional configuration files and their purposes are:
- ``auth_conf.xml``: Configures the pluggable authentication service. By default, Galaxy users are created and managed
internally.
- ``build_sites.yml``: Controls which display applications are available and their configuration paths
- ``containers_conf.yml``: Configures the beta Galaxy containers interface, currently only used by Galaxy Interactive
Environments, and only neccesary for Docker Swarm support.
- ``dependency_resolvers_conf.xml``: Describes how Galaxy tools (which are typically just descriptions of how to run a
particular command line tool) should locate their dependencies (the command line tool) that are not part of the tool.
See the `Dependency Resolvers documentation <dependency_resolvers>` for more.
- ``error_report.yml``: Controls how user-initiated error reporting (e.g. due to tool failure) is performed. See the
:ref:`Bug Reports documentation <bug_reports>` for more.
- ``job_metrics_conf.xml``: Enables reporting of certain conditions and collection of metrics when jobs run.
- ``job_resource_params_conf.xml``: Describes tool form elements that should be inserted into tool forms that can be
used by users to control runtime parameters such as memory allocations, cluster selection, and so forth.
- ``object_store_conf.xml``: Configures more advanced storage paradigms for Galaxy datasets, including layout across
multiple filesystems, or in object storage systems such as Swift or Amazon S3.
- ``openid_conf.xml``: Controls which OpenID (if enabled) providers should be presented as options to the user on the
login form.
- ``swarm_manager_conf.yml``: Configures the experimental Docker Swarm manager.
- ``tool_destinations.yml``: Configures dynamic tool destinations, which allow for mapping tools to job destinations
based on certain runtime job properties, such as the user submitting it, input sizes, and so forth.
- ``tool_sheds_conf.xml``: Defines the list of Tool Shed servers that should appear in the Galaxy Administration
interface when searching for new tools.
- ``workflow_schedulers_conf.xml``: Similar to the job configuration, controls the scheduling of workflows as jobs.
.. _standardize and unify configuration formats: https://github.com/galaxyproject/galaxy/issues/5148
.. _Installing Tools into Galaxy: https://galaxyproject.org/admin/tools/add-tool-from-toolshed-tutorial/
.. _Datatypes documentation: https://galaxyproject.org/learn/datatypes/
.. _Cluster documentation: https://galaxyproject.org/admin/config/performance/cluster/
.. _Data Preparation documentation: https://galaxyproject.org/admin/data-preparation/
.. _Data Managers documentation: https://galaxyproject.org/admin/tools/data-managers/
Configuration Basics
----------------------------
- Edit ``config/galaxy.yml`` (copy it from ``config/galaxy.yml.sample`` if it does not exist) to make configuration
changes. This is a `uWSGI YAML configuration file`_ and should contain two sections, one named ``uwsgi`` for uWSGI and
one named ``galaxy`` for Galaxy.
- The default port for the Galaxy web server is ``8080``, and it only binds to localhost by default. To configure
uWSGI to listen on all available network addresses, set ``http`` to ``0.0.0.0:<port>`` (e.g. ``http:
0.0.0.0:8080``).
- Some uWSGI options are required for uWSGI to run Galaxy properly and will be added to the ``uwsgi`` command line
by ``run.sh`` if not specified in ``galaxy.yml``.
- uWSGI has a `large number of options`_. The Galaxy documentation refers to some of them, but many additional
advanced deployment scenarios are available.
- Run Galaxy with ``sh run.sh``
- Use a web browser and go to the address you configured in ``galaxy.yml`` (defaults to http://localhost:8080/)
.. _uWSGI YAML configuration file: http://uwsgi-docs.readthedocs.io/en/latest/Configuration.html
.. _large number of options: http://uwsgi-docs.readthedocs.io/en/latest/Options.html
----------------------------
Configuration Options
----------------------------
.. include:: galaxy_options.rst
doc/source/admin/config_logging.rst
+209
View File
@@ -0,0 +1,209 @@
Logging Configuration
========================================
Overview
----------------------------
There are two ways in which you can configure logging for Galaxy servers:
1. Basic/automatic configuration with control over log level and log destination (standard output or a named log file).
2. More complex configuration using the Python :mod:`logging` module's :func:`logging.config.dictConfig` or :func:`logging.config.fileConfig`.
By default, Galaxy logs all messages to standard output at the ``DEBUG`` logging level, unless the ``--daemon`` argument
is passed to ``run.sh``, in which case, output is logged to the file ``galaxy.log`` in the current directory.
The way in which you configure logging depends on whether you are using a YAML or INI configuration file, and also on
whether you are using the uWSGI application server, or Python Paste. Galaxy servers that were created starting with
Galaxy Release 18.01 or later use a YAML configuration file with uWSGI. Galaxy servers that were created with 17.09 or
older use an INI configuration file, and Python Paste by default, but they could be configured to run under uWSGI (and
this was the recommendation for production servers). If you upgrade a pre-18.01 server running under Paste to 18.01 or
later but do not convert your INI config (``galaxy.ini``) to a YAML config (``galaxy.yml``), the INI config and Paste
will still be used.
uWSGI, Paste, and related terminology are explained in detail in the :doc:`Scaling and Load Balancing <scaling>`
documentation.
Basic Configuration
----------------------------
Basic logging configuration can be used to modify the level of log messages and the file to which Galaxy logs. The level
is controlled by the ``log_level`` configuration option.
If not set, Galaxy logs all messages at the ``DEBUG`` level (versions prior to 18.01 defaulted to ``INFO`` if unset, but
the default config file shipped with ``log_level`` explicitly set to ``DEBUG`` for development purposes).
Galaxy logs all messages to standard output by default if running in the foreground. If running in the background (``sh
run.sh --daemon``) under uWSGI, the log is written to ``galaxy.log`` in the current directory. If running in the
background under Paste, the log is written to ``paster.log``.
**Setting the log level:**
In ``galaxy.yml``, set ``log_level``:
.. code-block:: yaml
galaxy:
log_level: LEVEL
Or if using ``galaxy.ini``:
.. code-block:: ini
[app:main]
log_level = LEVEL
Where ``LEVEL`` is one of the `logging levels`_ documented in the :mod:`logging` module.
**Logging to a file:**
To change the log file name or location, use the ``$GALAXY_LOG`` environment variable like so:
.. code-block:: shell-session
$ GALAXY_LOG=/path/to/galaxy/logfile sh run.sh --daemon
Advanced Configuration
----------------------------
With the improved uWSGI support added in Galaxy release 18.01, additional fields identifying the uWSGI worker ID and
mule ID can be added to log messages. These are implemented as the custom Python logging filter
:class:`galaxy.web.stack.UWSGILogFilter` which provides two new Python :class:`logging.LogRecord` attributes:
``%(worker_id)s`` and ``%(mule_id)s``. These aid in identifying which log messages are being emitted by which process
and are used in the default message format when running under uWSGI, but are available to you if you wish to change the
message format. The default message format under uWSGI can be found in
:data:`galaxy.web.stack.UWSGIApplicationStack.log_format`.
Additionally, because uWSGI can start multiple distinct Galaxy processes (e.g. job handler mules) from a single config
file, by default it would not be possible to log each process to a separate file, meaning that the combined log file
could be quite verbose. In order to alleviate this, a ``filename_template`` attribute has been added to
:class:`logging.FileHandler` (or derivative classes) definitions so that multiple file logging is possible.
If you are still using Paste or an INI configuration file, it is still possible to use :func:`logging.config.fileConfig`
logging, but ``filename_template`` is not available in this scenario.
YAML
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The full syntax of Python's :func:`logging.config.dictConfig` is available under the ``logging`` key in the ``galaxy``
section of ``galaxy.yml``. The default as of this release can be found in the
:data:`galaxy.config.LOGGING_CONFIG_DEFAULT` constant and has been converted to YAML format here:
.. include:: config_logging_default_yaml.rst
Using ``run.sh --daemon`` causes Galaxy to log to ``galaxy.log``, but this is done using uWSGI's logging functionality
and does not allow for splitting logging in to multiple files. The following logging definition will cause the creation
of log files ``galaxy_web_0.log`` (the combined messages of all web workers) and ``galaxy_job-handlers_N.log`` where
``N`` is the instance ID of the server process in its pool (aka the mule's position in its farm argument):
.. code-block:: yaml
galaxy:
logging:
filters:
stack:
(): galaxy.web.stack.application_stack_log_filter
formatters:
stack:
(): galaxy.web.stack.application_stack_log_formatter
handlers:
console:
class: logging.StreamHandler
level: DEBUG
formatter: generic
stream: ext://sys.stderr
files:
class: logging.FileHandler
level: DEBUG
formatter: generic
filename: galaxy_default.log
filename_template: galaxy_{pool_name}_{server_id}.log
loggers:
galaxy:
handlers:
- console
- files
level: DEBUG
propagate: 0
qualname: galaxy
paste.httpserver.ThreadPool:
level: WARN
qualname: paste.httpserver.ThreadPool
routes.middleware:
level: WARN
qualname: routes.middleware
root:
handlers:
- console
- files
level: INFO
version: 1
The list of available template facts for all Galaxy application server types, and their values under the various
possible :doc:`server deployment scenarios <scaling>` are given below:
+-------------------+-----------------------------------------------------------------------------------------------+
| Fact | Application server |
+-------------------+-------------------------------+-------------------------------+-------------------------------+
| | Paste/webless | uWSGI web worker | uWSGI mule |
+===================+===============================+===============================+===============================+
| ``server_name`` | ``NAME`` for | ``main``, but can be modified with ``server_name`` in |
| | ``[server:<NAME>]`` in | ``galaxy.yml`` |
| | ``galaxy.ini`` | |
+-------------------+-------------------------------+-------------------------------+-------------------------------+
| ``server_id`` | ``None`` | 1-based worker ID | 1-based mule ID |
+-------------------+-------------------------------+-------------------------------+-------------------------------+
| ``pool_name`` | ``None`` | ``web`` | Mule's farm name |
+-------------------+-------------------------------+-------------------------------+-------------------------------+
| ``instance_id`` | ``None`` | Same as ``server_id`` | Mule's 1-based position in |
| | | | its defined farm |
+-------------------+-------------------------------+-------------------------------+-------------------------------+
| ``fqdn`` | Fully-qualified domain name of the host on which Galaxy is running |
+-------------------+-----------------------------------------------------------------------------------------------+
| ``hostname`` | "Short" hostname (with domain portion stripped) of the host on which Galaxy is running |
+-------------------+-----------------------------------------------------------------------------------------------+
INI
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
With an INI galaxy configuration, it is possible to use Python's :func:`logging.config.fileConfig` configuration method for
advanced logging configuration. For example:
.. code-block:: ini
[loggers]
keys = root, galaxy
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = INFO
handlers = console
[logger_galaxy]
level = DEBUG
handlers = console
qualname = galaxy
propagate = 0
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = DEBUG
formatter = generic
[formatter_generic]
format = %(name)s %(levelname)-5.5s %(asctime)s [p:%(process)s,w:%(worker_id)s,m:%(mule_id)s] [%(threadName)s] %(message)s
However, the ``filename_template`` Galaxy extension is not available with this method.
.. _logging levels: https://docs.python.org/2/library/logging.html#logging-levels
.. _fileConfig file format: https://docs.python.org/2/library/logging.config.html#configuration-file-format
doc/source/admin/dependency_resolvers.rst
+1 -1
View File
@@ -92,7 +92,7 @@ installed packages. This resolver can be configured either to use the version st
The Galaxy Packages dependency resolver takes a ``base_path`` argument that specifies the path under which
it starts looking for the files it requires. The default value for this ``base_path`` is the
``tool_dependency_dir`` configured in Galaxy's ``config/galaxy.ini``. Below the base path, the Galaxy Packages
``tool_dependency_dir`` configured in Galaxy's ``config/galaxy.yml``. Below the base path, the Galaxy Packages
resolver looks for directories named after tools, e.g. ``bedtools``. As mentioned before, this resolver
works in versioned and versionless mode. The default mode is versioned, where the dependency resolver looks for a
directory named after the dependency's version string. For example, if the Galaxy tool specifies that it
doc/source/admin/galaxy_options.rst
+3503
View File
File diff suppressed because it is too large Load Diff
doc/source/admin/index.rst
+6 -1
View File
@@ -6,14 +6,19 @@ This documentation is in the midst of being ported and unified based on resource
.. toctree::
:maxdepth: 2
config
config_logging
production
cluster
scaling
nginx
apache
cluster
jobs
tool_panel
authentication
dependency_resolvers
conda_faq
framework_dependencies
options
reports
useful_scripts
doc/source/admin/jobs.md
+3 -4
View File
@@ -6,7 +6,7 @@ This document is a reference for the job configuration file. [Detailed document
**The most up-to-date details of advanced job configuration features can be found in the [sample job_conf.xml](https://github.com/galaxyproject/galaxy/blob/dev/config/job_conf.xml.sample_advanced) found in the Galaxy distribution.**
Configuration of where to run jobs is performed in the `job_conf.xml` file in `$GALAXY_ROOT/config/`. The path to the config file can be overridden by setting the value of `job_config_file` in `config/galaxy.ini`. Sample configurations are provided at `galaxy-dist/config/job_conf.xml.sample_basic` and `galaxy-dist/config/job_conf.xml.sample_advanced`. The job configuration file is not required - if it does not exist, a default configuration that runs jobs on the local system (with a maximum of 4 concurrent jobs) will be used. `job_conf.xml.sample_basic` provides a configuration identical to the default configuration if no `job_conf.xml` exists.
Configuration of where to run jobs is performed in the `job_conf.xml` file in `$GALAXY_ROOT/config/`. The path to the config file can be overridden by setting the value of `job_config_file` in `config/galaxy.yml`. Sample configurations are provided at `galaxy-dist/config/job_conf.xml.sample_basic` and `galaxy-dist/config/job_conf.xml.sample_advanced`. The job configuration file is not required - if it does not exist, a default configuration that runs jobs on the local system (with a maximum of 4 concurrent jobs) will be used. `job_conf.xml.sample_basic` provides a configuration identical to the default configuration if no `job_conf.xml` exists.
## job_conf.xml Syntax
@@ -46,8 +46,7 @@ The collection contains `<handler>` elements.
```eval_rst
id
A server name (e.g. a ``[server:<name>]``</code> in <code>config/galaxy.ini</code> such as ``[server:main]``) that should be used to run jobs. </td>
<td> required </td>
A server name that should be used to run jobs. Server names are dependent on your application server deployment scenario and are explained in the :ref:`configuration section of the scaling documentation <scaling-configuration>`.
tags
A comma-separated set of strings that optional define tags to which this handler belongs.
@@ -237,7 +236,7 @@ The above examples demonstrate that the dynamic job destination framework will p
```eval_rst
``app``
Global Galaxy application object, has attributes such as config (the configuration parameters loaded from ``config/galaxy.ini``) and ``job_config`` (Galaxy representation of the data loaded in from ``job_conf.xml``).
Global Galaxy application object, has attributes such as config (the configuration parameters loaded from ``config/galaxy.yml``) and ``job_config`` (Galaxy representation of the data loaded in from ``job_conf.xml``).
``user_email``
E-mail of user submitting this job.
doc/source/admin/nginx.md
+394
View File
@@ -0,0 +1,394 @@
```eval_rst
.. |PROXY| replace:: nginx
```
# Proxying Galaxy with NGINX
In a production environment, it is recommended to run Galaxy behind a proxy web server for performance and security
reasons. The proxy server sits between clients and your Galaxy server, relaying requests between them and offloading
some of the more menial and resource-intensive tasks.
[NGINX][nginx] is a lightweight HTTP server designed with high performance proxying in mind. The Galaxy Project's public
servers, [usegalaxy.org][main] ("Main") and [Test][test], as well as the [Docker Galaxy project][docker-galaxy] use
NGINX, rather than Apache, to proxy Galaxy. NGINX was chosen for its simple, fast load balancing and other
proxy-oriented features.
Instructions for [proxying with Apache](apache.html) are also available.
[nginx]: http://nginx.org/en/
[main]: https://galaxyproject.org/main/
[test]: https://galaxyproject.org/test/
[docker-galaxy]: https://github.com/bgruening/docker-galaxy-stable
## Prerequisites
```eval_rst
.. include:: _inc_proxy_prereq.rst
```
### NGINX Proxy Prerequisities
If you plan to use nginx to handle your file uploads, you will (most likely) not be able to use your package manager's
version of nginx. The [Receiving Files With NGINX](#receiving-files-with-nginx) section explains this in detail and
provides some options for installing *nginx + upload module* packages maintained by the Galaxy Committers Team.
Otherwise, your system package manager's version of nginx should be suitable. Under Debian, the
[nginx-light][nginx-light] package contains all the necessary modules used in this guide. On EL, the [EPEL][epel]
version of nginx is suitable.
[nginx-light]: https://packages.debian.org/search?keywords=nginx-light
[epel]: https://fedoraproject.org/wiki/EPEL
## Basic Configuration
```eval_rst
.. include:: _inc_proxy_ssl.rst
```
### Serving Galaxy at the Web Server Root
```eval_rst
.. include:: _inc_proxy_serving_root.rst
.. _Galaxy Release 17.09 Proxy Documentation: https://docs.galaxyproject.org/en/release_17.09/admin/special_topics/nginx.html
```
uWSGI protocol support is built in to nginx, so (unlike Apache) no extra modules or recompiling should be required.
The following configuration is not exhaustive, only the portions most relevant to serving Galaxy are shown, these should
be incorporated with your existing/default nginx config as is appropriate for your server. Notably, the nginx package
you installed most likely has a multi-file config layout. If you are not already familiar with that layout and where
best to place your configuration, you can learn more in the [Proxy Package Layouts](proxy_package_layouts.html)
documentation.
```nginx
http {
#...
# compress responses whenever possible
gzip on;
gzip_http_version 1.1;
gzip_vary on;
gzip_comp_level 4;
gzip_proxied any;
gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;
gzip_comp_level 6;
gzip_buffers 16 8k;
# allow up to 3 minutes for Galaxy to respond to slow requests before timing out
uwsgi_read_timeout 180;
# maximum file upload size
client_max_body_size 10g;
# allowable SSL protocols
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
# use secure ciphers
ssl_ciphers
ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA256;
ssl_dhparam /etc/nginx/ssl/dhparams.pem;
ssl_prefer_server_ciphers on;
# enable session reuse
ssl_session_cache shared:SSL:8m;
ssl_session_timeout 5m;
# cert/key
ssl_certificate /etc/nginx/ssl/server.crt;
ssl_certificate_key /etc/nginx/ssl/server.key;
# OCSP stapling
ssl_stapling on;
ssl_stapling_verify on;
ssl_trusted_certificate /etc/nginx/ssl/ca.crt;
server {
listen 80 default_server;
listen [::]:80 default_server;
server_name _;
return 301 https://$host$request_uri;
}
server {
listen 443 default_server;
listen [::]:443 default_server;
server_name _;
# use a variable for convenience
set $galaxy_root /srv/galaxy/server;
# Enable HSTS
add_header Strict-Transport-Security "max-age=15552000; includeSubdomains";
# proxy all requests not matching other locations to uWSGI
location / {
uwsgi_pass unix:///srv/galaxy/var/uwsgi.sock
uwsgi_param UWSGI_SCHEME $scheme;
include uwsgi_params;
}
# serve framework static content
location /static/style {
alias $galaxy_root/static/style/blue;
expires 24h;
}
location /static {
alias $galaxy_root/static;
expires 24h;
}
location /robots.txt {
alias $galaxy_root/static/robots.txt;
expires 24h;
}
location /favicon.ico {
alias $galaxy_root/static/favicon.ico;
expires 24h;
}
# serve visualization and interactive environment plugin static content
location ~ ^/plugins/(?<plug_type>.+?)/(?<vis_name>.+?)/static/(?<static_file>.*?)$ {
alias $galaxy_root/config/plugins/$plug_type/$vis_name/static/$static_file;
expires 24;
}
}
}
```
Be sure to set `$galaxy_root` to the path to your copy of Galaxy and modify the value of `uwsgi_pass` to match your
uWSGI socket path. With the default configuration, uWSGI will bind to a random TCP socket, so you will need to set it to
a fixed value as described in the [Scaling and Load Balancing](scaling.html) documentation. If using a UNIX domain
socket, be sure to pay particular attention to the discussion of users and permissions.
### Additional Notes
- **Do not** simply copy the SSL configuration directives and expect them to work on your server or to be secure! These
are provided as examples of some of the best practices as of the time of writing, but will not always be up to date.
Use the guides referenced in [basic configuration](#basic-configuration) section to configure SSL properly.
- If your existing nginx configuration contains a line or included config file defining a default server, be sure to
disable it by commenting its `server {}` or preventing its inclusion (under Debian this is done by removing its
symlink from `/etc/nginx/sites-enabled`).
- `uwsgi_read_timeout` can be adjusted as appropriate for your site. This is the amount of time allowed for
communication between nginx and uWSGI to block while waiting for a response from Galaxy, and is useful for holding
client (browser) connections while uWSGI is restarting Galaxy subprocesses or Galaxy is performing a slow operation.
- The parameter `client_max_body_size` specifies the maximum upload size that can be handled by POST requests through
nginx. You should set this to the largest file size that you wish to allow for upload and that could be reasonably
handled by your site. It defaults to 1MB, so it will need to be increased if you are dealing with genome sized
datasets.
- If you must serve Galaxy without SSL, you would simply replace the `listen` directives in the SSL `server {}` block
with the `listen` directives from the non-SSL `server {}` block and remove the non-SSL block and SSL directives from
the `http {}` block.
- If the proxy works but you are getting 404 errors for Galaxy's static content, be sure that the user that nginx runs
as has access to Galaxy's `static/` directory (and all its parent directories) on the filesystem. You can test this on
the command line with e.g. `sudo -u www-data ls /srv/galaxy/server/static`.
### Serving Galaxy at a URL Prefix
It may be necessary to serve Galaxy from an address other than the web server root (`https://www.example.org/galaxy`),
instead of `https://galaxy.example.org`). To do this, you need to make the following changes to the configuration in the
previous section:
1. In the nginx config, prefix all of the location directives with your prefix and redirect requests from `/prefix` to
`/prefix/` like so:
```nginx
#...
# proxy all requests not matching other locations to uWSGI
location /galaxy {
uwsgi_pass unix:///srv/galaxy/var/uwsgi.sock
uwsgi_param UWSGI_SCHEME $scheme;
include uwsgi_params;
}
# serve framework static content
location /galaxy/static/style {
alias $galaxy_root/static/style/blue;
expires 24h;
}
# additional static locations...
# redirect /prefix -> /prefix/
rewrite ^/galaxy$ /galaxy/ last;
```
2. The Galaxy application needs to be aware that it is running with a prefix (for generating URLs in dynamic pages).
This is accomplished by configuring uWSGI and Galaxy (the `uwsgi` and `galaxy` sections in `config/galaxy.yml`
respectively) like so and restarting Galaxy:
```yaml
uwsgi:
#...
socket: unix:///srv/galaxy/var/uwsgi.sock
mount: /galaxy=galaxy.webapps.galaxy.buildapp:uwsgi_app()
manage-script-name: true
# `module` MUST NOT be set when `mount` is in use
#module: galaxy.webapps.galaxy.buildapp:uwsgi_app()
galaxy:
#...
cookie_path: /galaxy
```
`cookie_path` should be set to prevent Galaxy's session cookies from clobbering each other if you are running more
than one instance of Galaxy under different URL prefixes on the same hostname.
Be sure to consult the [Scaling and Load Balancing](scaling.html) documentation, other options unrelated to proxying
should also be set in the `uwsgi` section of the config.
## Advanced Configuration Topics
### Sending Files With Nginx
Galaxy sends files (e.g. dataset downloads) by opening the file and streaming it in chunks through the proxy server.
However, this ties up the Galaxy process, which can impact the performance of other operations (see [Production Server
Configuration](production.html) for a more in-depth explanation).
Nginx can assume this task instead and as an added benefit, speed up downloads. This is accomplished through the use of
the special `X-Accel-Redirect` header. Dataset security is maintained in this configuration because nginx will still
check with Galaxy to ensure that the requesting user has permission to access the dataset before sending it.
To enable it, add the following to your Galaxy's `server {}` block:
```nginx
location /_x_accel_redirect/ {
internal;
alias /;
}
```
Next, edit `galaxy.yml` and make the following change before restarting Galaxy:
```yaml
galaxy:
#...
nginx_x_accel_redirect_base: '/_x_accel_redirect'
```
For this to work, the user under which your nginx server runs will need read access to Galaxy's `files_path` directory
(by default, `database/files/`) and its contents. This is most easily done by adding the nginx user to the Galaxy user's
primary group and setting the `umask(2)` to create files with the group read permission set. If you start Galaxy from
the command line, you can do this like so:
```shell-session
admin@server$ sudo usermod -a -G galaxy www-data # add `www-data` user to `galaxy` group
admin@server$ sudo -iu galaxy
galaxy@server$ umask 027
galaxy@server$ sh run.sh
```
If you start Galaxy from supervisord, you can set the `umask` option in the [program
section](http://supervisord.org/configuration.html#program-x-section-settings) after adding the nginx user to the Galaxy
group as shown above.
### Receiving Files With Nginx
Galaxy receives files (e.g. dataset uploads) by streaming them in chunks through the proxy server and writing the files
to disk. However, this again ties up the Galaxy process. nginx can assume this task instead and as an added benefit,
speed up uploads. This is accomplished through the use of
[nginx_upload_module](http://www.grid.net.ru/nginx/upload.en.html), a 3rd-party nginx module.
To enable it, you must first download, compile and install nginx with the upload module, since prior to NGINX 1.11.5,
nginx did not support shared modules, and the upload module is not yet shared-compatible. Because this is a tedious
and complicated process, the Galaxy Committers team maintains (for some platforms) versions of nginx modified from their
upstream package sources (APT, EPEL, etc.) to include the upload module:
- [Ubuntu (PPA)](https://launchpad.net/~galaxyproject/+archive/ubuntu/nginx)
- [Enterprise Linux](https://depot.galaxyproject.org/yum/)
To contribute support for additional platforms, please see the [Galaxy
Starforge](https://github.com/galaxyproject/starforge) project, which is used to do the repackaging.
Once nginx with the upload module is installed, create a directory in which to store uploads (ideally, for performance
reasons, on the same filesystem as Galaxy's datasets) and add the necessary directives to `nginx.conf`:
```nginx
user galaxy;
http {
#...
server {
#...
# handle file uploads via the upload module
location /_upload {
upload_store /srv/galaxy/upload_store;
upload_store_access user:rw group:rw;
upload_pass_form_field "";
upload_set_form_field "__${upload_field_name}__is_composite" "true";
upload_set_form_field "__${upload_field_name}__keys" "name path";
upload_set_form_field "${upload_field_name}_name" "$upload_file_name";
upload_set_form_field "${upload_field_name}_path" "$upload_tmp_path";
upload_pass_args on;
upload_pass /_upload_done;
}
# once upload is complete, redirect to the proper galaxy path
location /_upload_done {
set $dst /api/tools;
if ($args ~ nginx_redir=([^&]+)) {
set $dst $1;
}
rewrite "" $dst;
}
}
```
Note the `user` directive at the top, outside of the `http {}` block. To ensure that Galaxy has write permission on the
uploaded files, nginx's workers will need to run as the same user as Galaxy.
When serving Galaxy at a URL prefix as described in the [Serving Galaxy at a URL
prefix](#serving-galaxy-at-a-url-prefix) section, you will need to change `set $dst /api/tools;` to `set $dst
/prefix/api/tools;` (e.g. `set $dst /galaxy/api/tools;`).
Finally, edit `galaxy.yml` and make the following change before restarting Galaxy:
```yaml
galaxy:
#...
nginx_upload_store: /srv/galaxy/upload_store
nginx_upload_path: '/_upload'
```
### Use Galaxy Authentication to Protect Custom Paths
You may find it useful to require authentication for access to certain paths on your server. For example, Galaxy can
run a separate reports app which gives useful information about your Galaxy instance. See the [Reports Configuration
documentation](reports.html) and [Peter Briggs' blog post on the
subject](http://galacticengineer.blogspot.com/2015/06/exposing-galaxy-reports-via-nginx-in.html) for more.
After succesfully following the blog post, Galaxy reports should be available at e.g. `https://galaxy.example.org/reports`.
To secure this page to only Galaxy administrators, adjust your nginx config accordingly:
**TODO:** This is not valid for the uWSGI proxy method and needs to be updated. -nate 2018-01-11
```nginx
location /reports {
#...
satisfy any; # only one auth method needs to succeed
deny all; # host-based auth is not allowed
auth_request /_auth; # forward authentication
}
location /_auth {
#internal; probably?
# The used galaxy api endpoint is only available to galaxy admins and thus limits the access
# to only logged in admins.
proxy_pass http://localhost/api/configuration/dynamic_tool_confs;
proxy_pass_request_body off;
proxy_set_header Content-Length "";
proxy_set_header X-Original-URI $request_uri;
}
```
### External User Authentication
- [Nginx for External Authentication](https://galaxyproject.org/admin/config/nginx-external-user-auth/)
- [Built-in Galaxy External Authentication](authentication.html)
doc/source/admin/options.rst
+5
View File
@@ -0,0 +1,5 @@
----------------------------
Configuration Options
----------------------------
.. include:: galaxy_options.rst
doc/source/admin/production.md
+14 -14
View File
@@ -7,7 +7,7 @@ The [basic installation instructions](https://getgalaxy.org) are suitable for de
By default, Galaxy:
* Uses [SQLite](http://www.sqlite.org/) (a serverless database), so you don't have to run/configure a database server for quick or basic development. However, while SQLite [supports concurrent access](https://sqlite.org/lockingv3.html) it does not support multiple concurrent writes, which can reduce system throughput.
* Uses a built-in HTTP server, written in Python. Much of the work performed by this server can be moved to [nginx](special_topics/nginx.html) or Apache, which will increase performance.
* Uses a built-in HTTP server, written in Python. Much of the work performed by this server can be moved to [nginx](nginx.html) or Apache, which will increase performance.
* Runs all tools locally. Moving to a [cluster](cluster.html) will greatly increase capacity.
* Runs in a single process, which is a performance problem in [CPython](http://en.wikipedia.org/wiki/CPython).
@@ -37,17 +37,17 @@ nate@weyerbacher% sh run.sh
## Basic configuration
The steps to install Galaxy mostly follow those of the [regular instructions](http://getgalaxy.org). The difference is that after performing the groundwork above, you should initialize the configuration file (`cp config/galaxy.ini.sample config/galaxy.ini`) and modify it as outlined below before starting the server. If you make any changes to this configuration file while the server is running, you will have to restart the server for the changes to take effect.
The steps to install Galaxy mostly follow those of the [regular instructions](http://getgalaxy.org). The difference is that after performing the groundwork above, you should initialize the configuration file (`cp config/galaxy.yml.sample config/galaxy.yml`) and modify it as outlined below before starting the server. If you make any changes to this configuration file while the server is running, you will have to restart the server for the changes to take effect.
### Disable the developer settings
Two options are set in the sample `config/galaxy.ini` which should not be enabled on a production server. You should set both to `False`:
Two options are set in the sample `config/galaxy.yml` which should not be enabled on a production server. You should set both to `false`:
* `debug = False` - Disable middleware that loads the entire response in memory for displaying debugging information in the page. If left enabled, the proxy server may timeout waiting for a response or your Galaxy process may run out of memory if it's serving large files.
* `use_interactive = False` - Disables displaying and live debugging of tracebacks via the web. Leaving it enabled will expose your configuration (database password, id_secret, etc.).
* Disable `filter-with = gzip`. Leaving the gzip filter enabled will cause UI failures because of the way templates are streamed once `debug` is set to `False`. You will still be able (and are encouraged) to enable gzip in the proxy server.
* `debug: false` - Disable middleware that loads the entire response in memory for displaying debugging information in the page. If left enabled, the proxy server may timeout waiting for a response or your Galaxy process may run out of memory if it's serving large files.
* `use_interactive: false` - Disables displaying and live debugging of tracebacks via the web. Leaving it enabled will expose your configuration (database password, id_secret, etc.).
* Disable `filter-with: gzip`. Leaving the gzip filter enabled will cause UI failures because of the way templates are streamed once `debug` is set to `False`. You will still be able (and are encouraged) to enable gzip in the proxy server.
During deployment, you may run into problems with failed jobs. By default, Galaxy removes files related to job execution. You can instruct Galaxy to keep files of failed jobs with: `cleanup_job = onsuccess`
During deployment, you may run into problems with failed jobs. By default, Galaxy removes files related to job execution. You can instruct Galaxy to keep files of failed jobs with: `cleanup_job: onsuccess`
### Switching to a database server
@@ -61,7 +61,7 @@ To use an external database, you'll need to set one up. That process is outside
Once installed, create a new database user and new database which the new user is the owner of. No further setup is required, since Galaxy manages its own schema. If you are using a UNIX socket to connect the application to the database (this is the standard case if Galaxy and the database are on the same system), you'll want to name the database user the same as the system user under which you run the Galaxy process.
To configure Galaxy, set `database_connection` in Galaxy's config file, `config/galaxy.ini`. The syntax for a database URL is explained in the [SQLAlchemy documentation](http://docs.sqlalchemy.org/en/latest/core/engines.html).
To configure Galaxy, set `database_connection` in Galaxy's config file, `config/galaxy.yml`. The syntax for a database URL is explained in the [SQLAlchemy documentation](http://docs.sqlalchemy.org/en/latest/core/engines.html).
Here follow two example database URLs with username and password:
@@ -81,7 +81,7 @@ mysql:///mydatabase?unix_socket=/var/run/mysqld/mysqld.sock
For more hints on available options for the database URL, see the [SQLAlchemy documentation](http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls).
If you are using [MySQL](http://dev.mysql.com/) and encounter the "MySQL server has gone away" error, please note the `database_engine_option_pool_recycle` option in `config/galaxy.ini`. If this does not solve your problem, see [this post](http://gmod.827538.n3.nabble.com/template/NamlServlet.jtp?macro=print_post&node=2354941) on the Galaxy Development [mailing list](/src/mailing-lists/index.md).
If you are using [MySQL](http://dev.mysql.com/) and encounter the "MySQL server has gone away" error, please note the `database_engine_option_pool_recycle` option in `config/galaxy.yml`. If this does not solve your problem, see [this post](http://gmod.827538.n3.nabble.com/template/NamlServlet.jtp?macro=print_post&node=2354941) on the Galaxy Development [mailing list](/src/mailing-lists/index.md).
If you are using [MySQL](http://dev.mysql.com/) please make sure the database output is in UTF-8, otherwise you may encounter python TypeErrors.
@@ -95,8 +95,8 @@ Downloading and uploading data can also be moved to the proxy server. This is e
Virtually any server that proxies HTTP should work, although we provide configuration examples for:
* [Apache](special_topics/apache.html), and
* [nginx](special_topics/nginx.html), a high performance reverse proxy, used by our public Galaxy sites
* [Apache](apache.html), and
* [nginx](nginx.html), a high performance reverse proxy, used by our public Galaxy sites
### Using a compute cluster
@@ -155,12 +155,12 @@ For those readers who've already been running Galaxy on a cluster, a bit of info
### Tune the database
[PostgreSQL](http://www.postgresql.org/) can store results more efficiently than Galaxy, and as a result, reduce Galaxy's memory footprint. When a query is made, the result will remain on the Postgres server and Galaxy can retrieve only the rows it needs. To enable this, set `database_engine_option_server_side_cursors = True` in the Galaxy config.
[PostgreSQL](http://www.postgresql.org/) can store results more efficiently than Galaxy, and as a result, reduce Galaxy's memory footprint. When a query is made, the result will remain on the Postgres server and Galaxy can retrieve only the rows it needs. To enable this, set `database_engine_option_server_side_cursors: true` in the Galaxy config.
If your server logs errors about the database connection pool size, you may need to increase the default minimum and maximum number of pool connections, 5 and 10. These config file options are `database_engine_option_pool_size` and `database_engine_option_max_overflow`.
Finally, if you are using Galaxy <= release_2014.06.02, we recommend that you instruct Galaxy to use one database connection per thread, to avoid connection overhead and overuse. This can be enabled with `database_engine_option_strategy = threadlocal`.
Finally, if you are using Galaxy <= release_2014.06.02, we recommend that you instruct Galaxy to use one database connection per thread, to avoid connection overhead and overuse. This can be enabled with `database_engine_option_strategy: threadlocal`.
### Make the proxy handle uploads and downloads
By default, Galaxy receives file uploads as a stream from the proxy server and then writes this file to disk. Likewise, it sends files as a stream to the proxy server. This occupies the GIL in that Galaxy process and will decrease responsiveness for other operations in that process. To solve this problem, you can configure your proxy server to serve downloads directly, involving Galaxy only for the task of authorizing that the user has permission to read the dataset. If using nginx as the proxy, you can configure it to receive uploaded files and write them to disk itself, only notifying Galaxy of the upload once it's completed. All the details on how to configure these can be found on the [Apache](special_topics/apache.html) and [nginx](special_topics/nginx.html) proxy instruction pages.
By default, Galaxy receives file uploads as a stream from the proxy server and then writes this file to disk. Likewise, it sends files as a stream to the proxy server. This occupies the GIL in that Galaxy process and will decrease responsiveness for other operations in that process. To solve this problem, you can configure your proxy server to serve downloads directly, involving Galaxy only for the task of authorizing that the user has permission to read the dataset. If using nginx as the proxy, you can configure it to receive uploaded files and write them to disk itself, only notifying Galaxy of the upload once it's completed. All the details on how to configure these can be found on the [Apache](apache.html) and [nginx](nginx.html) proxy instruction pages.
doc/source/admin/proxy_package_layout.rst
+172
View File
@@ -0,0 +1,172 @@
Proxy Package Layouts
========================================
Every software package has a suggested filesystem layout, and proxy servers like Apache and NGINX are no exception.
However, Linux distribution package maintainers often have their own opinions about layout, especially with respect to
following a layout standard or scheme employed by their particular distribution.
Thus, although you can configure the proxy server entirely within its primary configuration file, if you have installed
the proxy via your system package manager, this may not be the best idea. The primary config file in the cases of both Apache
and nginx under both Debian-based distributions and Enterprise Linux-based distributions contains various *include*
directives designed to allow you to place your configuration in files that are entirely controlled by you. This helps to
avoid conflicts in the primary config file when the package is upgraded.
.. hint::
Primary configuration files can be found at:
- nginx (both EL and Debian): ``/etc/nginx/nginx.conf``
- Apache:
- EL: ``/etc/httpd/conf/httpd.conf``
- Debian: ``/etc/apache2/apache2.conf``
Package Layout Overviews
---------------------------
Debian
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Debian uses a very similar directory scheme for both nginx and Apache, where ``<server>`` is ``apache2`` or ``nginx``
and ``<component>`` is some configuration subcomponent:
- ``/etc/<server>/<component>-available`` for files containing configuration snippets for various instances of the given
component
- ``/etc/<server>/<component>-enabled`` for symbolic links to files in ``/etc/<server>/<component>-available`` for each
snippet that the administrator wishes to enable
``<component>-available`` is effectively a "repository" for configurations which are enabled by symbolic links in
``<component>-enabled``.
Both nginx and Apache use the component ``sites``, intended for individual website configurations. Apache additionally
uses the components ``mods`` and ``conf`` for module loading/configuration and general global configuration statements,
respectively. Nginx, lacking the ``mods`` and ``conf`` components, provides ``/etc/nginx/conf.d``
.. tip::
On Debian, the paths most relevant to our purposes are:
- nginx:
- ``/etc/nginx/conf.d/*.conf`` for general directives that belong in the ``http {}`` block
- ``/etc/nginx/sites-available/*`` for individual site configs
- ``/etc/nginx/sites-enabled/*`` to enable sites
- Apache:
- ``/etc/apache2/conf-available/*`` for general directives that belong in the ``http {}`` block
- ``/etc/apache2/conf-enabled/*`` to enable configs
- ``/etc/apache2/sites-available/*`` for individual site configs
- ``/etc/apache2/sites-enabled/*`` to enable sites
Enterprise Linux
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
EL's layout is simpler: both nginx and Apache provide a single configuration inclusion directory, where ``<server>`` is
``httpd`` or ``nginx``:
- ``/etc/<server>/conf.d/``
Apache additionally has a module configuration directory at ``/etc/httpd/conf.modules.d``
.. tip::
On EL, the paths most relevant to our purposes are:
- nginx: ``/etc/nginx/conf.d/*.conf``
- Apache: ``/etc/httpd/conf.d/*.conf``
NGINX
---------------------------
**Global option configuration:**
On both Debian and EL, you could create ``/etc/nginx/conf.d/galaxy_options.conf`` for global options intended for the
``http {}`` block. Because this file is included from within the ``http {}`` block, you would simply define directives
without enclosing them in any sort of block:
.. code-block:: nginx
uwsgi_read_timeout 180;
client_max_body_size 10g;
ssl_certificate /etc/nginx/ssl/server.crt;
ssl_certificate_key /etc/nginx/ssl/server.key;
#...
**Site configuration:**
For the site configurations, you could create:
- ``/etc/nginx/sites-available/galaxy`` on Debian
- ``/etc/nginx/conf.d/galaxy_site.conf`` on EL
These files contain ``server {}`` blocks (again, not enclosed in an ``http {}``):
.. code-block:: nginx
server {
listen 80 default_server;
listen [::]:80 default_server;
server_name _;
#...
}
server {
listen 443 default_server;
listen [::]:443 default_server;
server_name _;
#...
}
On Debian, you'd then need to symlink the config with:
.. code-block:: shell-session
# ln -s /etc/nginx/sites-available/galaxy /etc/nginx/sites-enabled/galaxy
Apache
---------------------------
**Global option configuration:**
For the global options, you could create:
- ``/etc/apache2/confs-available/galaxy.conf`` on Debian
- ``/etc/httpd/conf.d/galaxy_options.conf`` on EL
With the global configuration directives:
.. code-block:: apache
SSLProtocol all -SSLv3
SSLCipherSuite ...
#...
**Site configuration:**
For the site configurations, you could create:
- ``/etc/apache2/sites-available/galaxy.conf`` on Debian
- ``/etc/httpd/conf.d/galaxy_site.conf`` on EL
With the ``<VirtualHost>`` blocks:
.. code-block:: apache
<VirtualHost _default_:80>
Redirect permanent / https://galaxy.example.org
</VirtualHost>
<VirtualHost _default_:443>
SSLEngine on
SSLCertificateFile /etc/apache2/ssl/server.crt
SSLCertificateKeyFile /etc/apache2/ssl/server.key
#...
</VirtualHost>
On Debian you'd then need to symlink the configs with (or do it by hand with `ln -s`):
.. code-block:: shell-session
# a2enconf galaxy
# a2ensite galaxy
doc/source/admin/reports.md
+3 -3
View File
@@ -4,12 +4,12 @@ Galaxy includes a report tool that is separate from the main process but which g
## Setup on localhost
The report tool takes its configuration settings from a file called reports.ini, which is located in the config subdirectory of the Galaxy distribution.
The report tool takes its configuration settings from a file called `reports.yml`, which is located in the ``config/` subdirectory of the Galaxy distribution.
Configuring the reports for your local setup is a case of:
* Making a copy of `reports.ini.sample` called `reports.ini`.
* Editing the `database_connection` parameter to match the one in your `galaxy.ini` file
* Making a copy of ``reports.yml.sample`` called `reports.yml`.
* Editing the `database_connection` parameter to match the one in your `galaxy.yml` file
* Optionally, editing the `port` parameter (by default the tool uses port 9001)
You should also set the 'salt' parameter `session_secret` if you intend to expose the reports via the web proxy (see below).
doc/source/admin/reports.rst
+1 -1
View File
@@ -41,7 +41,7 @@ following (and more):
Configuration
----------------------------
- Configure ``config/reports.yml`` in the same manner as your main galaxy instance (i.e., same database connection, but different port). This is a uwsgi YAML configuration file and should contain a reports section with app-specific configuration (options described below).
- Configure ``config/reports.yml`` in the same manner as your main galaxy instance (i.e., same database connection, but different web server port). This is a uWSGI YAML configuration file and should contain a ``reports`` section with app-specific configuration (options described below).
- The default port for the reports application is ``9001``, and like Galaxy it only binds to localhost by default.
- ``database_connection`` should match the value used in your Galaxy configuration
doc/source/admin/scaling.md
+482 -213
View File
@@ -1,190 +1,544 @@
# Scaling and Load Balancing
The Galaxy framework is written in Python and makes extensive use of threads. However, one of the drawbacks of Python is the [Global Interpreter Lock](http://docs.python.org/c-api/init.html#thread-state-and-the-global-interpreter-lock), which prevents more than one thread from being on CPU at a time. Because of this, having a multi-core system will not improve the Galaxy framework's performance out of the box since Galaxy can use (at most) one core at a time. However, Galaxy can easily run in multiple separate processes, which solves this problem. For a more thorough explanation of this problem and why you will almost surely want to switch to the load balanced configuration if running for more than a small handful of users, see the [production configuration](production.html) page.
The Galaxy framework is written in Python and makes extensive use of threads. However, one of the drawbacks of Python
is the [Global Interpreter Lock](http://docs.python.org/c-api/init.html#thread-state-and-the-global-interpreter-lock),
which prevents more than one thread from being on CPU at a time. Because of this, having a multi-core system will not
improve the Galaxy framework's performance out of the box since Galaxy can use (at most) one core at a time in its
default configuration. However, Galaxy can easily run in multiple separate processes, which solves this problem. For a
more thorough explanation of this problem and why you will almost surely want to switch to the multiprocess
configuration if running for more than a small handful of users, see the [production configuration](production.html)
page.
Just to be clear: increasing the values of `threadpool_workers` in `galaxy.ini` or the number of plugin workers in `job_conf.xml` will not make you Galaxy server much more responsive. The key to scaling Galaxy is the ability to run *multiple* Galaxy servers which co-operatively work on the same database.
Just to be clear: increasing the values of `threadpool_workers` in `galaxy.yml` or the number of plugin workers in
`job_conf.xml` will not make you Galaxy server much more responsive. The key to scaling Galaxy is the ability to run
*multiple* Galaxy servers which co-operatively work on the same database.
A simple configuration:
* 1 "job handler" process - responsible for starting and monitoring jobs, submitting jobs to a cluster (if configured), and for setting metadata (externally or internally).
* 1 "web server" process - responsible for servicing web pages to users.
## Terminology
An advanced configuration:
* Multiple "job handler" processes.
* Multiple "web server" processes, proxied through a load-balancing capable web server (e.g. nginx or apache).
* **web worker** - Galaxy server process responsible for servicing web requests for the UI/API
* **job handler** - Galaxy server process responsible for setting up, starting, and monitoring jobs, submitting jobs to
a cluster (if configured), for setting metadata (if not set on the cluster), and cleaning up after jobs
* **[uWSGI][uwsgi]** - Powerful application server written in C that implements the HTTP and Python WSGI protocols
* **[Mules][uwsgi-mules]** - uWSGI processes started after the main application (Galaxy) that can run separate code
and receive messages from uWSGI web workers
* **[Zerg Mode][uwsgi-zerg-mode]** - uWSGI configuration where multiple copies of the same application can be started
simultaneously in order to maintain availability during application restarts
* **Webless Galaxy application** - The Galaxy application run as a standalone Python application with no web/WSGI server
* **[Paste][paste]** - Application server written in pure Python that implements the HTTP and Python WSGI protocols
### Web Server(s)
[uwsgi]: http://uwsgi-docs.readthedocs.io/
[uwsgi-mules]: http://uwsgi-docs.readthedocs.io/en/latest/Mules.html
[uwsgi-zerg-mode]: http://uwsgi-docs.readthedocs.io/en/latest/Zerg.html
[paste]: http://paste.readthedocs.io/
There are a few different ways you can run multiple web server processes:
## Application Servers
**Standalone Paste-based processes:**
* Pros:
* Simplest setup, especially if only using a single web server process
* No additional dependencies
* Proxy not required if only using a single web server process
* Cons:
* Not as resilient to failure
* Load balancing typically round-robin regardless of individual process load
* No dynamic scaling
It is possible to run the Galaxy server in many different ways, including under different web application frameworks, or
as a standalone server with no web stack. For most of its modern life, prior to the 18.01 release, Galaxy (by default)
used the [Python Paste][paste] web stack, and ran in a single process.
**uWSGI:**
* Pros:
* Higher performance server than Paste
* Better scalability and fault tolerance
* Easier process management and Galaxy server restartability
* Cons:
* Requires uWSGI
Beginning with Galaxy release 18.01, the default application server for new installations of Galaxy is [uWSGI][uwsgi].
Prior to 18.01, it was possible (and indeed, recommended for production Galaxy servers) to run Galaxy under uWSGI, but
it was necessary to install and configure uWSGI separately from Galaxy. uWSGI is now provided with Galaxy as a Python
Wheel and installed in to its virtualenv, as described in detail in the [Framework
Dependencies](framework_dependencies.html) documentation.
Using uWSGI for production servers is recommended by the Galaxy team.
uWSGI has numerous benefits over Python Paste for our purposes:
#### Standalone Paste-based processes
* Written in C and designed to be high performance
* Easily runs multiple processes by increasing `processes` config option
* Load balances multiple processes internally rather than requiring load balancing in the proxy server
* Offload engine for serving static content
* Speaks high performance native protocol between uWSGI and proxy server
* Can speak HTTP and HTTPS protocols without proxy server
* Incredibly featureful, supports a wide array of deployment scenarios
* Supports WebSockets, which enable Galaxy Interactive Environments out-of-the-box without a proxy server or Node.js
In `galaxy.ini`, define one or more `[server:...]` sections:
## Deployment Options
```ini
[server:web0]
use = egg:Paste#http
port = 8080
host = 127.0.0.1
use_threadpool = true
threadpool_workers = 7
There are multiple deployment strategies for the Galaxy application that you can choose from. The right one depends on
the configuration of the infrastructure on which you are deploying. In all cases, all Galaxy job features such as
[running on a cluster](cluster.html) are supported.
[server:web1]
use = egg:Paste#http
port = 8081
host = 127.0.0.1
use_threadpool = true
threadpool_workers = 7
Although uWSGI implements nearly all the features that were previously the responsibility of an upstream proxy server,
at this time, it is still recomended to place a proxy server in front of uWSGI and utilize it for all of its traditional
roles (serving static content, serving dataset downloads, etc.) as described in the [production
configuration](production.html) documentation.
When using uWSGI with a proxy server, it is recommended that you use the native high performance uWSGI protocol
(supported by both [Apache](apache.html) and [nginx](nginx.html)) between uWSGI and the
proxy server, rather than HTTP.
### uWSGI with jobs handled by web workers (default configuration)
Referred to in this documentation as the **uWSGI all-in-one** strategy.
* Job handlers and web workers are the same processes and cannot be separated
* The web worker that receives the job request from the UI/API will be the job handler for that job
Under this strategy, jobs will be handled by uWSGI web workers. Having web processes handle jobs will negatively impact
UI/API performance.
This is the default out-of-the-box configuration as of Galaxy Release 18.01.
### uWSGI for web serving with Mules as job handlers
Referred to in this documentation as the **uWSGI + Mules** strategy.
* Job handlers run as children of the uWSGI process
* Jobs are dispatched from web workers to job handlers via native *mule messaging*
* Jobs can only be dispatched to mules on the same host
* Trivially easy to enable (disabled by default for simplicity reasons)
Under this strategy, job handling is offloaded to dedicated non-web-serving processes that are started and stopped
directly by the master uWSGI process. As a benefit of using mule messaging, only job handlers that are alive will be
selected to run jobs.
This is the recommended deployment strategy for Galaxy servers that run web servers and job handlers **on the same
host**.
### uWSGI for web serving and Webless Galaxy applications as job handlers
Referred to in this documentation as the **uWSGI + Webless** strategy.
* Job handlers are started as standalone Python applications with no web stack
* Jobs are dispatched from web workers to job handlers via the Galaxy database
* Jobs can be dispatched to job handlers running on any host
* The recommended deployment strategy for production Galaxy instances prior to 18.01
Like mules, under this strategy, job handling is offloaded to dedicated non-web-serving processes, but those processes
are [managed by the administrator](#starting-and-stopping). Because the handler is randomly assigned by the web worker
when the job is submitted via the UI/API, jobs may be assigned to dead handlers.
This is the recommended deployment strategy for Galaxy servers that run web servers and job handlers **on different
hosts**.
## Legacy Deployment Options
Certain deployment strategies were commonly used prior to the introduction of new features described above. These are
still possible but should no longer be used.
### uWSGI for web serving with Paste Galaxy applications as job handlers
This is essentially the same as **uWSGI + Webless** but needlessly starts handlers with a web stack. This was
recommended before the Webless method existed.
### Paste for web serving with Paste or Webless job handlers
Unlike uWSGI, Paste cannot start multiple server processes on its own. Prior to uWSGI support, this was the only way to
run multiple Galaxy processes, but each web worker and job handler process had to be configured and managed separately.
### Paste web serving and job handling in a single process (default configuration, releases prior to 18.01)
This was the default configuration prior to the 18.01 Galaxy release and offered the simplest out-of-the-box setup at
the expense of performance and scalability.
```eval_rst
.. _scaling-configuration:
```
## Configuration
Two are shown, you should create as many as are suitable for your usage and hardware. On our eight-core server, I run six web server processes. You may find you only need one, which is a slightly simpler configuration.
### uWSGI
#### uWSGI
Although this document goes in to significant detail about uWSGI configuration, many more options are available, as well
as additional documentation about options described here. Consult the uWSGI documentation for more:
In `galaxy.ini`, define a `[uwsgi]` section:
* [Configuring uWSGI](https://uwsgi-docs.readthedocs.io/en/latest/Configuration.html)
* [uWSGI Options](https://uwsgi-docs.readthedocs.io/en/latest/Options.html)
* [Quickstart for Python/WSGI applications](https://uwsgi-docs.readthedocs.io/en/latest/WSGIquickstart.html)
Configuration is performed in the `uwsgi` section of `galaxy.yml`. You will find that the default, if copied from
`galaxy.yml.sample`, is commented out. The default configuration options are provided to uWSGI on the command line by
Galaxy's `run.sh` script.
Galaxy releases prior to 18.01 (or upgraded-to-18.01+ servers which have not migrated their configuration to the YAML
format) used an INI-format configuration file, `galaxy.ini`.
Note that uWSGI's YAML parser is hand-coded and not actually conformant to the YAML standard. Specifically:
* Multiple identical keys with unique values can exist in the same dictionary/hash, as with `hook-master-start` in the
example below.
* Quoting values (with single or double quotes) is unncessary since the parser treats all values as strings. The parser
does not correctly handle these quote characters, resulting in invalid values.
If using `galaxy.ini`, the option names and values are the same but in INI format, for example:
```ini
[uwsgi]
processes = 8
stats = 127.0.0.1:9191
processes = 4
socket = 127.0.0.1:4001
pythonpath = lib
threads = 4
logto = /path/to/uwsgi.log
master = True
...
```
#### Configuration common to all uWSGI deployment styles
Port numbers for `stats` and `socket` can be adjusted as desired. Moreover, in the `[app:main]` section, you must set:
In `galaxy.yml`, define a `uwsgi` section. Shown below are the options common to all deployment scenarios:
```ini
static_enabled = False
track_jobs_in_database = True
```yaml
uwsgi:
# required in order to start the galaxy application
module: galaxy.webapps.galaxy.buildapp:uwsgi_app()
virtualenv: .venv
pythonpath: lib
# performance options
master: true
enable-threads: true
processes: 2
threads: 4
offload-threads: 1
# fix up signal handling
die-on-term: true
hook-master-start: unix_signal:2 gracefully_kill_them_all
hook-master-start: unix_signal:5 gracefully_kill_them_all
# listening options
# job handling options
```
Some of these options warrant explanation:
You will also need to have uWSGI installed. There are a variety of ways to do this. It can be installed system-wide by installing from your system's package manager (on Debian and Ubuntu systems, the `uwsgi` and `uwsgi-plugin-python` provide the necessary components), or with the `easy_install` or `pip` commands (which will install it to the system's Python `site-packages` directory). Alternatively, if you are already running Galaxy from a Python virtualenv, you can use `pip install uwsgi` with that virtualenv's copy of `pip` to install to that virtualenv as your unprivileged Galaxy user.
* `master`: Instructs uWSGI to first start a master process manager and then fork web workers, mules, http servers (if
enabled), and any others from the master. This is required for certain operational modes such as daemonization, but
can interfere with the use of `<CTRL>+<C>` to shut down Galaxy when running in the foreground on the command line, and
so is not enabled by default (except when `run.sh --daemon` is used). Its use is strongly recommended for all
production deployments.
* `processes`: Controls the number of Galaxy application processes uWSGI will spawn. Increased web performance can be
attained by increasing this value.
* `threads`: Controls the number of web worker threads each application process will spawn.
* `offload-threads`: uWSGI can use a dedicated threadpool for serving static content and handling internal routing,
setting this value automatically enables such offloading.
Also, make sure you have installed PasteDeploy, you can follow the same ways from above.
Additional options are explained in the [uWSGI Minutiae](#uwsgi-minutiae) below.
The web processes can then be started under uWSGI using:
Note that the performance option values given above are just examples and should be tuned per your specific needs.
However, as given, they are a good place to start.
```console
% cd /path/to/galaxy-dist
% PYTHONPATH=eggs/PasteDeploy-1.5.0-py2.7.egg uwsgi --ini-paste config/galaxy.ini
Due to the Python GIL, increasing the value of `threads` has diminishing returns on web performance while increasing the
memory footprint of each application process. Increasing it is most useful on servers experiencing a high amount of IO
waiting, but the greatest performance gain comes from increasing `processes` as appropriate for the hardware on which
Galaxy is running.
#### Listening and proxy options
**With a proxy server:**
To use the native uWSGI protocol, set the `socket` option:
```yaml
# listening options
socket: unix:///srv/galaxy/var/uwsgi.sock
```
Here we've used a UNIX domain socket because there's less overhead than a TCP socket and it can be secured by filesystem
permissions, but you can also listen on a port:
The `--daemonize` option can be used to start in the background. uWSGI has an astounding number of options, see [its documentation](http://uwsgi.readthedocs.org/) for help.
Once started, a proxy server (typically Apache or nginx) must be configured to proxy requests to uWSGI (using uWSGI's native protocol). Configuration details for these can be found below.
### Job Handler(s)
In `galaxy.ini`, define one or more additional `[server:...]` sections:
```ini
[server:handler0]
use = egg:Paste#http
port = 8090
host = 127.0.0.1
use_threadpool = true
threadpool_workers = 5
[server:handler1]
use = egg:Paste#http
port = 8091
host = 127.0.0.1
use_threadpool = true
threadpool_workers = 5
```yaml
# listening options
socket: 127.0.0.1:4001
```
The choice of port 4001 is arbitrary, but in both cases, the socket location must match whatever socket the proxy server
is configured to communicate with. If using a UNIX domain socket, be sure that the proxy server's user has read/write
permission on the socket. Because Galaxy and the proxy server most likely run as different users, this is not likely to
be the case by default. One common solution is to add the proxy server's user to the Galaxy user's primary group.
uWSGI's `chmod-socket` option can also help here.
Using web processes as handlers is possible, but it is not recommended since handler operations can impact web UI performance.
You can consult the Galaxy documentation for [Apache](apache.html) or [nginx](nginx.html)
for help with the proxy-side configuration.
### Remaining configuration options
By setting the `socket` option, `run.sh` will no longer automatically serve Galaxy via HTTP (since it is assumed that
you are setting a socket to serve Galaxy via a proxy server). If you wish to continue serving HTTP directly with uWSGI
while `socket` is set, you can use the `http` option as shown in the directions below.
If you do not have a `job_conf.xml` file, you will need to create one. There are samples for a basic configuration and an advanced configuration provided in the distribution. Please note that creating `job_conf.xml` overrides any legacy job running settings in `galaxy.ini`. See the [jobs configuration documentation](jobs.html) for more detail on job configuration.
**Without a proxy server** or with a proxy server that does not speak the uWSGI native protocol:
In `job_conf.xml`, create `<handler>` tags with `id` attributes that match the handler server names you defined in `galaxy.ini`. For example, using the configuration above, the `<handlers>` section of `job_conf.xml` would look like:
uWSGI can be configured to serve HTTP and/or HTTPS directly:
```yaml
# listening options
http: :8080
https: :8443,server.crt,server.key
static-map: /static/style=static/style/blue
static-map: /static=static
```
To bind to ports < 1024 (e.g. if you want to bind to the standard HTTP/HTTPS ports 80/443), you must bind as the `root`
user and drop privileges to the Galaxy user with a configuration such as:
```yaml
# listening options
shared-socket: :80
shared-socket: :443,server.crt,server.key
http: =0
https: =1
uid: galaxy
gid: galaxy
static-map: /static/style=static/style/blue
static-map: /static=static
```
To redirect HTTP traffic to the HTTPS port rather than serving Galaxy over HTTP, change `http: =0` in the above example
to `http-to-https: =0`.
Because `run.sh` performs setup steps, **it should not be run as `root`**. Instead, you can run uWSGI directly as root
with:
```sh-session
# cd /srv/galaxy/server
# ./.venv/bin/uwsgi --yaml config/galaxy.yml
```
You can run the startup-time setup steps as the galaxy user after upgrading Galaxy with `sh
./scripts/common_startup.sh`.
#### uWSGI all-in-one job handling
Ensure that no `<handlers>` section exists in your `job_conf.xml` (or no `job_conf.xml` exists at all) and start Galaxy
normally. No additional configuration is required. To increase the number of web workers/job handlers, increase the
value of `processes`.
By default, a job will be handled by the web worker that receives the job setup request (via the UI/API). Jobs can be
explicitly mapped to specific workers as described in the [Job configuration documentation](jobs.html) by using the
handler IDs `main.web.N`, where `N` is the web worker ID, starting at 1 and incrementing for each process defined by the
value of `processes`. Each worker that you wish to explicitly map jobs to should be defined in the `<handlers>` section
of `job_conf.xml`. *Do not* define a default handler.
For example, to have the 3rd web worker handle the `test1` tool, you would set the following in `job_conf.xml`
(irrelevant options are not shown):
```xml
<handlers default="handlers">
<handler id="handler0" tags="handlers"/>
<handler id="handler1" tags="handlers"/>
</handlers>
<job_conf>
<handlers>
<handler id="main.web.3" />
</handlers>
<tools>
<tool id="test1" handler="main.web.3" />
</tools>
</job_conf>
```
#### uWSGI + Mule job handling
Any tool not set to an explicit job destination will then be serviced by one of the handlers with the `handlers` tag. It is possible to dedicate handlers to specific destinations or tools. For details on how to do this, please see the [job configuration documentation](jobs.html).
Ensure that no `<handlers>` section exists in your `job_conf.xml` (or no `job_conf.xml` exists at all) and add the
following to the `uwsgi` section of `galaxy.yml` to start a single job handler mule:
```yaml
# job handling options
mule: lib/galaxy/main.py
farm: job-handlers:1
```
Then start Galaxy normally. To add additional mule handlers, add additional `mule` options and add their ID(s), comma
separated, to the `job-handlers` farm. For example, 3 handlers are defined like so:
```yaml
# job handling options
mule: lib/galaxy/main.py
mule: lib/galaxy/main.py
mule: lib/galaxy/main.py
farm: job-handlers:1,2,3
```
By default, a job will be handled by whatever mule currently has the lock on the mule message queue. After receiving a
message, it will release the lock, giving other mules a chance to handle future jobs. Jobs can be explicitly mapped to
specific mules as described in the [Job configuration documentation](jobs.html) by using the handler IDs
`main.job-handlers.N`, where `N` is the mule's position in the farm, starting at 1 and incrementing for each mule in the
farm (this is not necessarily the mule ID, but it will be if you only define one farm and you add mules to that farm in
sequential order). Each worker that you wish to explicitly map jobs to should be defined in the `<handlers>` section
of `job_conf.xml`. *Do not* define a default handler.
For example, to have the 2nd mule in the three-mule job-handlers farm shown above handle the `test1` tool, you would set
the following in `job_conf.xml` (irrelevant options are not shown):
```xml
<job_conf>
<handlers>
<handler id="main.job-handlers.2" />
</handlers>
<tools>
<tool id="test1" handler="main.job-handlers.2" />
</tools>
</job_conf>
```
#### uWSGI + Webless job handling
Define a `<handlers>` section in `job_conf.xml` defining the webless handlers you plan to start. In this case, unlike
the uWSGI job handling strategies, you will need to define a default:
```xml
<job_conf>
<handlers default="handlers">
<handler id="handler1" tags="handlers" />
<handler id="handler2" tags="handlers" />
<handler id="handler3" />
</handlers>
<tools>
<tool id="test1" handler="handler3" />
</tools>
</job_conf>
```
The definition of a default handler prevents uWSGI web workers from starting the Galaxy job handling code. `run.sh` will
start the uWSGI process(es), but you will need to start the webless handler processes yourself. This is done on the
command line like so:
```sh-session
$ cd /srv/galaxy/server
$ ./scripts/galaxy-main -c config/galaxy.yml --server-name handler0 --daemonize
$ ./scripts/galaxy-main -c config/galaxy.yml --server-name handler1 --daemonize
$ ./scripts/galaxy-main -c config/galaxy.yml --server-name handler2 --daemonize
```
However, a better option to managing processes by hand is to use a process manager as documented in the [Starting and
Stopping](#starting-and-stopping) section.
#### uWSGI Minutiea
**Threads**
Although `enable-threads` was explicitly set in our example, in reality, as long as any value is set for the `threads`
option, `enable-threads` is set implicitly. This option enables the Python GIL and application threads (threads started
by Galaxy itself for various non-web tasks), which Galaxy uses extensively. Setting it explicitly, however, is harmless
and can prevent strange difficult-to-debug situations if `threads` is accidentally unset.
**Signals**
The signal handling options (`die-on-term` and `hook-master-start` with `unix_signal` values) are not required but, if
set, will override [uWSGI's unconventional signal handling](https://uwsgi-docs.readthedocs.io/en/latest/Management.html)
and cause `SIGTERM` to kill the server rather than restart it, and the uWSGI master process to gracefully shut down its
web workers and job handler mules (i.e. the various Galaxy application processes) when it receives a `SIGINT` (signal 2)
or `SIGTERM` (signal 15) signal (e.g. from `kill(1)` or `<CTRL>+<C>`). When shutting down gracefully, uWSGI will wait 60
seconds (by default, but this can be changed with the `reload-mercy`, `worker-reload-mercy` and `mule-reload-mercy`
options) for child processes to die before forcefully killing them with `SIGKILL` (signal 9). Alternatively, you may
prefer to have it shut down gracefully on `SIGTERM` but forcefully on `SIGINT` (forceful shutdown by uWSGI is still
slightly cleaner than `kill -9` of the master process since it can attempt to release sockets cleanly) or vice-versa,
which you can do by setting one of the signals to call `kill_them_all` rather than `gracefully_kill_them_all`:
```yaml
# fix up signal handling
die-on-term: true
hook-master-start: unix_signal:2 kill_them_all
hook-master-start: unix_signal:15 gracefully_kill_them_all
```
More details on the `unix_signal` hook can be found in [uWSGI Issue #849](https://github.com/unbit/uwsgi/issues/849).
**Logging and daemonization**
It's possible to configure uWSGI to log to a file with the `logto` or `logto2` options (when running in the foreground,
the default), but more advanced logging options that split log files for each process are possible and described in the
Galaxy [Logging Configuration documentation](config_logging.html)
When running as a daemon with `run.sh --daemon`, output is logged to `galaxy.log` and the pid is written to
`galaxy.pid`. These can be controlled with the `daemonize` and `pidfile` arguments (their `daemonize2` and `pidfile2`
counterparts wait until after the application successfully loads to open and write the files). If you set a `daemonize*`
option, you should not use the `--daemon` argument to `run.sh` (or not use `run.sh` at all, and start Galaxy directly
with `uwsgi` on the command line).
**External uwsgi binary:**
It is still possible to run Galaxy using an external copy of uWSGI (for example, installed from APT under
Debian/Ubuntu). This was the recommended installation method in the past. To use an external uWSGI, you'll need simply
need to start with uWSGI directly, rather than using the `run.sh` script. Once you have configured Galaxy/uWSGI, you can
start it with:
```sh-session
$ cd /srv/galaxy/server
$ uwsgi --yaml config/galaxy.yml
```
When installing uWSGI, be sure to install the Python plugin, as this is not always contained in the same package (such
as when installing from APT under Debian/Ubuntu). With the APT packages, you will also need to add `--plugin python` to
the command line (or `plugin: python` to the `uwsgi` section of `galaxy.yml`).
**Other options**
The `py-call-osafterfork` option may only be needed when mule messaging is in use, but it [seems to have no negative
effect](https://github.com/unbit/uwsgi/issues/643) and may solve other situations with starting a complex threaded
Python application.
The `chdir` option is useful if you are not using `run.sh`, to be able to call `uwsgi` from anywhere without having to
`cd` to the Galaxy directory first.
**Monitoring**
The [uwsgitop](https://github.com/xrmx/uwsgitop) tool uses uWSGI's stats server (the `stats` option, which is configured
to listen on a socket or port in the same manner as the `socket` option) to report on the health and performance of the
web workers. It can be installed with `pip install uwsgitop`. This tool can be useful for determing whether a worker is
stuck, or seeing the throughput of traffic on your site.
## Starting and Stopping
Since you need to run multiple processes, the typical `run.sh` method for starting and stopping Galaxy won't work. The current recommended way to manage these multiple processes is with [Supervisord](http://supervisord.org/). You can use a supervisord config file like the following or be inspired by [this example](https://github.com/galaxyproject/galaxy/blob/dev/contrib/galaxy_supervisor.conf). Be sure to `supervisord restart` or `supervisord reread && supervisord update` whenever you make configuration changes.
If you are using the **uWSGI + Webless job handlers** deployment strategy or want to run your Galaxy server as a
persistent service, you can control it through a process manager. The current recommended process manager is
[Supervisord](http://supervisord.org/). If you are comfortable with systemd and are running a relatively modern Linux
distribution, you can also configure Galaxy as a service directly in systemd.
### Supervisord
You can use a supervisord config file like the following or be inspired by [this
example](https://github.com/galaxyproject/galaxy/blob/dev/contrib/galaxy_supervisor.conf). Be sure to `supervisord
update` or `supervisord reread && supervisord restart` whenever you make configuration changes.
```ini
[program:galaxy_uwsgi]
command = /usr/bin/uwsgi --plugin python --ini-paste /path/to/galaxy/config/galaxy.ini
directory = /path/to/galaxy
[program:web]
command = /srv/galaxy/venv/bin/uwsgi --yaml /srv/galaxy/config/galaxy.yml
directory = /srv/galaxy/server
umask = 022
autostart = true
autorestart = true
startsecs = 10
user = gxprod
environment = PATH=/path/to/galaxy/venv:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin,PYTHON_EGG_CACHE=/path/to/galaxy/.python-eggs,PYTHONPATH=/path/to/galaxy/eggs/PasteDeploy-1.5.0-py2.7.egg
user = galaxy
numprocs = 1
stopsignal = INT
```
This configuration defines a "program" named `web` which represents our Galaxy uWSGI frontend. You'll notice that we've
set a command, a directory, a umask, all of which you should be familiar with. Additionally we've specified that the
process should `autostart` on boot, and `autorestart` if it ever crashes. We specify `startsecs` to say "the process
must stay up for this long before we consider it OK. If the process crashes sooner than that (e.g. bad changes you've
made to your local installation) supervisord will try again a couple of times to restart the process before giving up
and marking it as failed. This is one of the many ways supervisord is much friendly for managing these sorts of tasks.
This configuration defines a "program" named "galaxy_uwsgi" which represents our galaxy uWSGI frontend. You'll notice that we've set a command, a directory, a umask, all of which you should be familiar with. Additionally we've specified that the process should **autostart** on boot, and **autorestart** if it ever crashes. We specify **startsecs** to say "the process must stay up for this long before we consider it OK. If the process crashes sooner than that (e.g. bad changes you've made to your local installation) supervisord will try again a couple of times to restart the process before giving up and marking it as failed. This is one of the many ways supervisord is much friendly for managing these sorts of tasks.
Next, we set up our job handlers:
If using the **uWSGI + Webless** scenario, you'll need to addtionally define job handlers to start:
```ini
[program:handler]
command = /path/to/galaxy/venv/bin/python ./scripts/paster.py serve config/galaxy.ini --server-name=handler%(process_num)s --pid-file=/path/to/galaxy/handler%(process_num)s.pid --log-file=/path/to/galaxy/handler%(process_num)s.log
directory = /path/to/galaxy
command = python ./scripts/galaxy-main -c /srv/galaxy/config/galaxy.yml --server-name=handler%(process_num)s --pid-file=/srv/galaxy/var/handler%(process_num)s.pid --log-file=/srv/galaxy/log/handler%(process_num)s.log
directory = /srv/galaxy/server
process_name = handler%(process_num)s
numprocs = 2
numprocs = 3
umask = 022
autostart = true
autorestart = true
startsecs = 15
user = gxprod
environment = PYTHON_EGG_CACHE=/path/to/galaxy/.python-eggs,SGE_ROOT=/var/lib/gridengine
user = galaxy
```
This is similar to the "web" definition above, however, you'll notice that we use `%(process_num)s`. That's a variable
substitution in the `command` and `process_name` fields. We've set `numprocs = 3`, which says to launch three handler
processes. Supervisord will loop over `0..numprocs` and launch `handler0`, `handler1`, and `handler2` processes
automatically for us, templating out the command string so each handler receives a different log file and name.
Nearly all of this is the same as above, however, you'll notice that we use `$(process_num)s`. That's a variable substitution in the command and process_name fields. We've set **numproces=2** which says to launch two handler processes. Supervisord will launch loop over `0..numprocs` and launch a `handler0` and `handler1` process automatically for us, templating out the command string so each handler receives a different log file and name.
Lastly, we collect the two tasks above into a single group:
Lastly, collect the tasks defined above into a single group. If you are not using webless handlers this is as simple as:
```ini
[group:galaxy]
programs = handler, galaxy_uwsgi
programs = web
```
With webless handlers, it is:
```ini
[group:galaxy]
programs = web, handler
```
This will let us manage these tasks more globally with the `supervisorctl` command line tool:
@@ -192,123 +546,38 @@ This will let us manage these tasks more globally with the `supervisorctl` comma
# supervisorctl status
galaxy:handler0 RUNNING pid 7275, uptime 16:32:17
galaxy:handler1 RUNNING pid 7276, uptime 16:32:17
galaxy:uwsgi RUNNING pid 7299, uptime 16:32:16
galaxy:handler2 RUNNING pid 7277, uptime 16:32:17
galaxy:web RUNNING pid 7299, uptime 16:32:16
```
This command shows us the status of our jobs, and we can easily restart all of the processes at once by naming the group. Familiar commands like start and stop are also available.
This command shows us the status of our jobs, and we can easily restart all of the processes at once by naming the
group. Familiar commands like start and stop are also available.
```console
# supervisorctl restart galaxy:
galaxy:handler0: stopped
galaxy:handler1: stopped
galaxy:uwsgi: stopped
galaxy:uwsgi: started
galaxy:handler2: stopped
galaxy:web: stopped
galaxy:web: started
galaxy:handler0: started
galaxy:handler1: started
galaxy:handler2: started
```
### Transparent restart - Zerg Mode
### Systemd
TODO: write this section.
### Transparent Restart - Zerg Mode
The standard uWSGI operation mode allows you to restart the Galaxy application while blocking client connections. Zerg Mode does away with the waiting by running a special Zerg Pool process, and connecting Zergling workers (aka Galaxy application processes) to the pool. As long as at least one is connected, requests can be served.
See the [GCC2017 Admin Training session](https://github.com/galaxyproject/dagobah-training/blob/2017-montpellier/sessions/10-uwsgi/ex2-zerg-mode.md) on how to set this up.
## Proxy Server
## Notes on Legacy Configurations
If using only one web process, you can proxy as per the normal instructions for a [production configuration page](production.html). Otherwise, you'll need to set up load balancing.
If you have specified a separate job runner and you want to use the "Manage jobs" interface as administrator you also have to define a proxy for the job runner as shown [below](#manage-jobs).
### Apache
Be sure to consult the [Apache proxy documentation](special_topics/apache.html) for additional features such as proxying static content and accelerated downloads.
#### Standalone Paste-based processes
To balance on Apache, you'll need to enable `mod_proxy_balancer` in addition to `mod_proxy`, which is available in Apache 2.2 (but not older versions such as 1.3 or 2.0). Add the following to your Apache configuration to set up balancing for the two example web servers defined above:
```apache
<Proxy balancer://galaxy>
BalancerMember http://localhost:8080
BalancerMember http://localhost:8081
</Proxy>
```
And replace the following line from the [regular proxy configuration](special_topics/apache.html):
```apache
RewriteRule ^(.*) http://localhost:8080$1 [P]
```
With:
```apache
RewriteRule ^(.*) balancer://galaxy$1 [P]
```
#### uWSGI
mod_uwsgi is available in apache2.4 and later. This means you *must* be on Ubuntu 14.04 or later. There are ways to do this on older systems, which is outside the scope of this documentation. You'll need to enable `mod_uwsgi`, and then add the following to your Apache configuration:
```apache
<Location "/galaxy">
Sethandler uwsgi-handler
uWSGISocket 127.0.0.1:4001
uWSGImaxVars 512
</Location>
```
### nginx
Be sure to consult the [nginx proxy documentation](special_topics/nginx.html) for additional features such as proxying static content and accelerated downloads.
#### Standalone Paste-based processes
To proxy with nginx, you'll simply need to add all of the web applications to the `upstream` section, [which already exists](special_topics/nginx.html). The relevant parts of the configuration would look like this:
```nginx
http {
upstream galaxy_app {
server localhost:8080;
server localhost:8081;
}
server {
location / {
proxy_pass http://galaxy_app;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
}
}
```
#### uWSGI
uWSGI support is built in to nginx, so no extra modules or recompiling should be required. To proxy to Galaxy, use the following configuration:
```nginx
uwsgi_read_timeout 180;
location / {
uwsgi_pass 127.0.0.1:4001;
uwsgi_param UWSGI_SCHEME $scheme;
include uwsgi_params;
}
```
`uwsgi_read_timeout` can be adjusted as appropriate for your site. This is the amount of time connections will block between while nginx waits for a response from uWSGI and is useful for holding client (browser) connections while uWSGI is restarting Galaxy subprocesses.
## Notes on legacy configurations
Previously it was necessary to create two separate Galaxy config files to use multiple processes. This is no longer necessary, and if you have multiple config files in your existing installation, it is suggested that you merge them in to a single file.
Galaxy previously used a single "job manager" process to assign jobs to handlers. This is no longer necessary as handlers are selected by the web processes at the time of job creation.
The `track_jobs_in_database` option in `galaxy.ini` can still be set but should be unnecessary. If there are more than one `[server:...]` sections in the file, database job tracking will be enabled automatically.
The `track_jobs_in_database` option in the Galaxy config file can still be set, but doing so is unnecessary as it now
defaults to "enabled" under all scenarios. Galaxy's in-memory job tracking can still be used when using the **uWSGI
all-in-one** deployment strategy by setting the option to `false`. In-memory tracking can be slightly more responsive
and thus can be useful in development, but should not be used in production.
doc/source/admin/special_topics/apache.md
-227
View File
@@ -1,227 +0,0 @@
# Proxying Galaxy with Apache
For various reasons (performance, authentication, etc.) in a production environment, it's recommended to run Galaxy behind a web server proxy. Although any proxy could work, Apache is a common choice. Alternatively, we use [nginx](http://nginx.net/) for our public sites and open source infrastructure, and [details are available](./nginx.html) for it, too.
Currently the only recommended way to run Galaxy with Apache is using `mod_rewrite` and `mod_proxy`. Either `mod_proxy_uwsgi` or `mod_proxy_http` may be used from there.
To support proxying, the `mod_proxy`, `mod_http_proxy` and `mod_rewrite` modules must be enabled in the Apache config. The main proxy directives, `ProxyRequests` and `ProxyVia` do **not** need to be enabled.
## Prerequisites
Make sure that inbound (and outbound) traffic to the TCP protocol HTTP on port 80 (and HTTPS on port 443 if using SSL) is permitted by your server's firewall/security.
```eval_rst
.. warning:: Please note that Galaxy should *never* be located on disk inside Apache's `DocumentRoot`. By default, this would expose all of Galaxy (including datasets) to anyone on the web.
```
## Basic configuration
### Allow Encoded Slashes in URLs
Some Galaxy URLs contain encoded slashes (%2F) in the path and Apache will not serve these URLs by default. To configure Apache to serve URLs with encoded slashes in the path, add the following line to your Apache configuration file:
```apache
AllowEncodedSlashes NoDecode
```
**Note**: The `NoDecode` setting was added in httpd 2.2.18, and CentOS 6 (as RHEL 6 does) only has 2.2.15. The [CentOS SCLo SIG Repo](https://wiki.centos.org/SpecialInterestGroup/SCLo/CollectionsList) has an httpd24 package.
### Serving Galaxy at the web server root (/)
For a default Galaxy configuration running on [http://localhost:8080/](http://localhost:8080/), the following lines in the Apache configuration will proxy requests to the Galaxy application:
```apache
# Rewrite
RewriteEngine on
RewriteRule ^(.*) http://localhost:8080$1 [P]
```
Or, if using `mod_proxy` with HTTP transport
```apache
ProxyPass / http://127.0.0.1:8080/
```
Or, if using `mod_proxy` with uWSGI transport
```apache
ProxyPass / uwsgi://127.0.0.1:4001/
```
Thus, all requests on your server are now redirected to Galaxy. Because this example uses the "root" of your web server, you may want to use a [VirtualHost](http://httpd.apache.org/docs/2.2/vhosts/) to be able to run other sites from this same server.
If your Apache server is set up to use `mod_security`, you may need to modify the value of the `SecRequestBodyLimit`. The default value on some systems will limit uploads to only a few kilobytes.
Since Apache is more efficient at serving static content, it is best to serve it directly, reducing the load on the Galaxy process and allowing for more effective compression (if enabled), caching, and pipelining. To do so, your configuration will now include the following, where `$GALAXY_ROOT` should be replaced with the filesystem path to your Galaxy installation
```apache
RewriteEngine on
RewriteRule ^/static/style/(.*) $GALAXY_ROOT/static/june_2007_style/blue/$1 [L]
RewriteRule ^/static/scripts/(.*) $GALAXY_ROOT/static/scripts/packed/$1 [L]
RewriteRule ^/static/(.*) $GALAXY_ROOT/static/$1 [L]
RewriteRule ^/favicon.ico $GALAXY_ROOT/static/favicon.ico [L]
RewriteRule ^/robots.txt $GALAXY_ROOT/static/robots.txt [L]
```
You will need to ensure that filesystem permissions are set such that the user running your Apache server has access to the Galaxy static/ directory.
### Serving Galaxy at a sub directory (such as /galaxy)
It may be necessary to house Galaxy at an address other than the web server root (`http://www.example.org/galaxy`), instead of `http://www.example.org`). To do this, you need to make the following changes:
Two changes are necessary:
1. In the apache config, prefix all of the location directives with your prefix, like so:
```apache
RewriteEngine on
RewriteRule ^/galaxy/static/style/(.*) $GALAXY_ROOT/static/june_2007_style/blue/$1 [L]
RewriteRule ^/galaxy/static/scripts/(.*) $GALAXY_ROOT/static/scripts/packed/$1 [L]
RewriteRule ^/galaxy/static/(.*) $GALAXY_ROOT/static/$1 [L]
RewriteRule ^/galaxy/favicon.ico $GALAXY_ROOT/static/favicon.ico [L]
RewriteRule ^/galaxy/robots.txt $GALAXY_ROOT/static/robots.txt [L]
```
Note the first rewrite rule deals with the missing trailing slash problem. If left out, [http://www.example.org/galaxy](http://www.example.org/galaxy) will result in a 404 error.
If you are using `mod_rewrite` for serving:
```
RewriteRule ^/galaxy$ /galaxy/ [R]
RewriteRule ^/galaxy(.*) http://localhost:8080$1 [P]
```
Or for `proxy_http`/`proxy_uwsgi`:
```
ProxyPass /galaxy http://127.0.0.1:8080/galaxy
# or
ProxyPass /galaxy uwsgi://127.0.0.1:4001/
```
2. The Galaxy application needs to be aware that it is running with a prefix (for generating URLs in dynamic pages). This is accomplished by configuring a Paste proxy-prefix filter in the `[app:main]` section of `config/galaxy.ini` and restarting Galaxy:
```ini
[filter:proxy-prefix]
use = egg:PasteDeploy#prefix
prefix = /galaxy
[app:main]
filter-with = proxy-prefix
cookie_path = /galaxy
```
`cookie_prefix` should be set to prevent Galaxy's session cookies from clobbering each other if running more than one instance of Galaxy in different subdirectories on the same hostname.
### SSL
If you place Galaxy behind a proxy address that uses SSL (i.e., `https://` URLs), edit your galaxy location block (e.g. `location /` when served at the root, or something else like `location /galaxy` when served under a prefix)
```apache
<Location "/">
...
RequestHeader set X-URL-SCHEME https
...
</Location>
```
Setting `X-URL-SCHEME` makes Galaxy aware of what type of URL it should generate for external sites like Biomart. This should be added to the existing `<Location />` block if you already have one, and adjusted accordingly if you're serving Galaxy from a subdirectory.
## Advanced Configuration Topics
### Compression and caching
All of Galaxy's static content can be cached on the client side, and everything (including dynamic content) can be compressed on the fly. This will decrease download and page load times for your clients, as well as decrease server load and bandwidth usage. To enable, you'll need to load `mod_deflate` and `mod_expires` in your Apache configuration, and then set:
```apache
<Location "/">
...
# Compress all uncompressed content.
SetOutputFilter DEFLATE
SetEnvIfNoCase Request_URI \.(?:gif|jpe?g|png)$ no-gzip dont-vary
SetEnvIfNoCase Request_URI \.(?:t?gz|zip|bz2)$ no-gzip dont-vary
SetEnvIfNoCase Request_URI /history/export_archive no-gzip dont-vary
</Location>
<Location "/static">
# Allow browsers to cache everything from /static for 6 hours
ExpiresActive On
ExpiresDefault "access plus 6 hours"
...
</Location>
```
The contents above should be added to the existing `<Location "/">` block if you already have one, and adjusted accordingly if you're serving Galaxy from a subdirectory.
### Sending files using Apache
Galaxy sends files (e.g. dataset downloads) by opening the file and streaming it in chunks through the proxy server. However, this ties up the Galaxy process, which can impact the performance of other operations (see [production configuration](../production.html) for a more in-depth explanation).
Apache can assume this task instead and as an added benefit, speed up downloads. This is accomplished through the use of `mod_xsendfile`, a 3rd-party Apache module. Dataset security is maintained in this configuration because Apache will still check with Galaxy to ensure that the requesting user has permission to access the dataset before sending it.
To enable it, you must first install `mod_xsendfile`, this is usually available via your OS's repositories. Once done, add the appropriate `LoadModule` directive to your Apache configuration to load the xsendfile module and the `XSendFile` directives to your proxy configuration:
```apache
<Location "/">
XSendFile on
XSendFilePath /
</Location>
```
Finally edit your `$GALAXY_ROOT/config/galaxy.ini` and make the following change before restarting Galaxy:
```ini
[app:main]
apache_xsendfile = True
```
For this to work, the user under which your nginx server runs will need read access to Galaxy's `$GALAXY_ROOT/database/files/` directory and its contents.
### External user authentication
- [Apache for External Authentication](https://galaxyproject.org/admin/config/apache-external-user-auth/)
- [Built-in Galaxy External Authentication](../authentication.html)
### Display Sites
Display sites such as UCSC work not by sending data directly from Galaxy to UCSC via the client's browser, but by sending UCSC a URL to the data in Galaxy that the UCSC server will retrieve data from. Since enabling authentication will place **all** of Galaxy behind authentication, such display sites will no longer be able to access data via that URL. If `display_servers` is set to a non-empty value in `$GALAXY_ROOT/config/galaxy.ini`, this tells Galaxy it should allow the named servers access to data in Galaxy. However, you still need to configure Apache to allow access to the datasets. An example config is provided here that allows the UCSC Main/Test backends:
```apache
<Location "/root/display_as">
Satisfy Any
Order deny,allow
Deny from all
Allow from hgw1.cse.ucsc.edu
Allow from hgw2.cse.ucsc.edu
Allow from hgw3.cse.ucsc.edu
Allow from hgw4.cse.ucsc.edu
Allow from hgw5.cse.ucsc.edu
Allow from hgw6.cse.ucsc.edu
Allow from hgw7.cse.ucsc.edu
Allow from hgw8.cse.ucsc.edu
</Location>
```
**PLEASE NOTE that this introduces a security hole** , the impact of which depends on whether you have restricted access to the dataset via Galaxy's [internal dataset permissions](https://galaxyproject.org/learn/security-features/).
- By default, data in Galaxy is public. Normally with a Galaxy server behind authentication in a proxy server this is of little concern since only clients who've authenticated can access Galaxy. However, if display site exceptions are made as shown above, anyone could use those public sites to bypass authentication and view any **public** dataset on your Galaxy server. If you have not changed from the default and most of your datasets are public, you should consider running your own display sites that are also behind authentication rather than using the public ones.
- For datasets for which access has been restricted to one or more roles (i.e. it is no longer "public"), access for reading via external browsers is only allowed for a brief period, when someone with access permission clicks the "display at..." link. During this period, anyone who has the dataset ID would then be able to use the browser to view this dataset. Although such a scenario is unlikely, it is technically possible.
### Proxying multiple galaxy worker threads
If you've configured multiple threads for galaxy in the `config/galaxy.ini` file, you will need a `ProxyBalancer` to manage sending requests to each of the threads. You can do that with apache configuration as follows:
```apache
<Proxy balancer://galaxy>
BalancerMember http://localhost:8400
BalancerMember http://localhost:8401
</Proxy>
# Replace the following line from the regular proxy configuration:
# RewriteRule ^(.*) http://localhost:8080$1 [P]
# With:
RewriteRule ^(.*) balancer://galaxy$1 [P]
```
doc/source/admin/special_topics/bug_reports.rst
+1 -1
View File
@@ -46,7 +46,7 @@ information about the job and links to the precise locations within Galaxy.
As a plugin, nothing has changed to this functionality, though future updates
may add features. It currently takes all of its configuration from your
``$GALAXY/config/galaxy.ini``, e.g. the variables ``error_email_to`` and
``$GALAXY/config/galaxy.yml``, e.g. the variables ``error_email_to`` and
``email_from``.
JSON
doc/source/admin/special_topics/chat.rst
+7 -7
View File
@@ -23,7 +23,7 @@ This service will need to be deployed on a host that has:
- Access to the Galaxy database
- A port exposed
- A non-default ``id_secret`` set in your ``galaxy.ini``
- A non-default ``id_secret`` set in your ``galaxy.yml``
We say "access to galaxy database", this service could run on an NFS mounted
copy of Galaxy, if the host it is deployed on can access the database of the
@@ -36,16 +36,16 @@ Deploying the Server
The chat server is a very new feature in Galaxy and has been disabled by
default at a number of levels, as not all users may wish to use this feature.
Configuration is managed in ``galaxy.ini``:
Configuration is managed in ``galaxy.yml``:
.. code-block:: ini
.. code-block:: yaml
# Galaxy real time communication server settings
enable_communication_server = True
communication_server_host = http://localhost
communication_server_port = 7070
enable_communication_server: True
communication_server_host: http://localhost
communication_server_port: 7070
# persistent_communication_rooms is a comma-separated list of rooms that should be always available.
persistent_communication_rooms = NGS,RNA-Seq,Genome Annotation
persistent_communication_rooms: NGS,RNA-Seq,Genome Annotation
As you can see in this configuration, it is pointing at a service running on
the same host as Galaxy, talking to port 7070.
doc/source/admin/special_topics/ftp.md
+2 -2
View File
@@ -10,7 +10,7 @@ Although there is no specific required server, we use [ProFTPD](http://proftpd.o
## Configure Galaxy
The first step is to choose a directory into which your users will upload files. Preferably this will be on the same filesystem as Galaxy's datasets (by default, `galaxy_dist/database/files/`). The FTP server will create subdirectories inside of this directory which match the user's email address. Likewise, Galaxy will expect to find email-named subdirectories at that path. This directory should be set in the config file (`galaxy.ini`) as `ftp_upload_dir`.
The first step is to choose a directory into which your users will upload files. Preferably this will be on the same filesystem as Galaxy's datasets (by default, `galaxy_dist/database/files/`). The FTP server will create subdirectories inside of this directory which match the user's email address. Likewise, Galaxy will expect to find email-named subdirectories at that path. This directory should be set in the config file (`galaxy.yml`) as `ftp_upload_dir`.
In the config file, you'll also want to set `ftp_upload_site` to the hostname your users should connect to via FTP. This will be provided in the help text on the Upload File form.
@@ -38,7 +38,7 @@ GRANT
## Configuring ProFTPD
By default, Galaxy stores passwords using [PBKDF2](http://en.wikipedia.org/wiki/PBKDF2). It's possible to disable this using the `use_pbkdf2 = False` setting in `galaxy.ini`. Once disabled, any new passwords created will be stored in an older hex-encoded SHA1 format. Because of this, it's possible to have both PBKDF2 and SHA1 passwords in your database (especially if your server has been around since before PBKDF2 support was added). Although this is fine (Galaxy can read passwords in either format), ProFTPD will expect them in one format or the other (although with some amount of hackery it could probably be made to read both).
By default, Galaxy stores passwords using [PBKDF2](http://en.wikipedia.org/wiki/PBKDF2). It's possible to disable this using the `use_pbkdf2: false` setting in the `galaxy` section of `galaxy.yml`. Once disabled, any new passwords created will be stored in an older hex-encoded SHA1 format. Because of this, it's possible to have both PBKDF2 and SHA1 passwords in your database (especially if your server has been around since before PBKDF2 support was added). Although this is fine (Galaxy can read passwords in either format), ProFTPD will expect them in one format or the other (although with some amount of hackery it could probably be made to read both).
Because of this, you'll need to choose one or the other in your Galaxy config (PBKDF2 is more secure and therefore preferred) and configure ProFTPD accordingly. If users cannot log in because their password is stored in the wrong format, they can simply use Galaxy's password change form to set their password, which will rewrite their password using the currently configured algorithm.
doc/source/admin/special_topics/index.rst
-2
View File
@@ -5,8 +5,6 @@ Special Topics
.. toctree::
:maxdepth: 2
nginx
apache
ftp
interactive_environments
mulled_containers
doc/source/admin/special_topics/interactive_environments.rst
+13 -13
View File
@@ -102,26 +102,26 @@ Galaxy runs as.
Configuring the Proxy
^^^^^^^^^^^^^^^^^^^^^
Configuration is all managed in ``galaxy.ini``. The default arguments used
Configuration is all managed in ``galaxy.yml``. The default arguments used
for the proxy are:
.. code-block:: ini
.. code-block:: yaml
dynamic_proxy_manage=True
dynamic_proxy_session_map=database/session_map.sqlite
dynamic_proxy_bind_port=8800
dynamic_proxy_bind_ip=0.0.0.0
dynamic_proxy_debug=True
dynamic_proxy_manage: true
dynamic_proxy_session_map: database/session_map.sqlite
dynamic_proxy_bind_port: 8800
dynamic_proxy_bind_ip: 0.0.0.0
dynamic_proxy_debug: true
As you can see most of these variables map directly to the command line
arguments to the NodeJS script. There are a few extra parameters which will
be needed if you run Galaxy behind an upstream proxy like nginx or
Apache:
.. code-block:: ini
.. code-block:: yaml
dynamic_proxy_external_proxy=True
dynamic_proxy_prefix=gie_proxy
dynamic_proxy_external_proxy: true
dynamic_proxy_prefix: gie_proxy
The first option says that you have Galaxy and the Galaxy NodeJS proxy wrapped
in an upstream proxy like Apache or NGINX. This will cause Galaxy to connect
@@ -139,7 +139,7 @@ second (``dynamic_proxy_prefix``) option sets the URL path that's used to
differentiate requests that should go through the proxy to those that should go
to Galaxy. You will need to add special upstream proxy configuration to handle
this, and you'll need to use the same ``dynamic_proxy_prefix`` in your
``galaxy.ini`` that you use in your URL routes.
``galaxy.yml`` that you use in your URL routes.
In the examples below, we assume that your Galaxy installation is available
at a URL such as ``https://f.q.d.n/galaxy``. If instead it is available at a
@@ -296,10 +296,10 @@ configuration file:
swarm_mode = True
You can also enable swarm mode for *all* GIE plugins by setting
``interactive_environment_swarm_mode`` in ``galaxy.ini`` to ``True``. If using
``interactive_environment_swarm_mode`` in ``galaxy.yml`` to ``True``. If using
this setting, you must still set ``docker_connect_port`` in each GIE plugin's
ini configuration file. The ``swarm_mode`` setting in individual GIE plugin
config files will override the value set in ``galaxy.ini``.
config files will override the value set in ``galaxy.yml``.
Note that your Galaxy server does not need to be a member of the swarm itself.
It can use the method outlined above in the `Docker on Another Host`_ section
doc/source/admin/special_topics/nginx.md
-307
View File
@@ -1,307 +0,0 @@
# Proxying Galaxy with NGINX
[NGINX](http://nginx.org/en/) is a lightweight http server designed with high performance proxying in mind. The public Galaxy sites ([Main](https://galaxyproject.org/main/) and [Test](https://galaxyproject.org/test/)) as well as the [Docker Galaxy project](https://github.com/bgruening/docker-galaxy-stable) use nginx to proxy rather than Apache for its simple, fast load balancing and other features.
Galaxy should _never_ be located on disk inside nginx's `root`. By default, this would expose all of Galaxy (including datasets) to anyone on the web.
## Prerequisites
Make sure that inbound (and outbound) traffic to the TCP protocol HTTP on port 80 (and HTTPS on port 443 if using SSL) is permitted by your server's firewall/security.
```eval_rst
.. warning:: Please note that Galaxy should *never* be located on disk inside Nginx's document root. By default, this would expose all of Galaxy (including datasets) to anyone on the web.
```
## Basic configuration
### Serving Galaxy at the web server root (/)
For a default Galaxy configuration running on [http://localhost:8080/](http://localhost:8080/) (see [SSL](https://github.com/VJalili/galaxy-site/blob/patch-1/src/admin/config/nginxProxy/index.md#ssl) section for HTTPS), the following lines in the nginx configuration will proxy requests to the Galaxy application:
```nginx
http {
...
upstream galaxy_app {
server localhost:8080;
}
proxy_next_upstream off;
server {
client_max_body_size 10G;
# ... other server stuff ...
location / {
proxy_pass http://galaxy_app;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
# serve static content for visualization and interactive environment plugins
location ~ ^/plugins/(?<plug_type>.+?)/(?<vis_name>.+?)/static/(?<static_file>.*?)$ {
alias $GALAXY_ROOT/config/plugins/$plug_type/$vis_name/static/$static_file;
}
}
}
```
**Notes:**
Make sure that you either comment out or modify line containing default configuration for enabled sites (if present, e.g. on Debian/Ubuntu systems).
```
include /etc/nginx/sites-enabled/*;
```
- The `proxy_next_upstream off;` disables nginx's round-robin scheme to prevent it from submitting POST requests more than once. This is unsafe, and is useful when using more than one upstream.
- Replace `$GALAXY_ROOT` with the path to your copy of Galaxy.
- The parameter `client_max_body_size` specifies the maximum upload size that can be handled by POST requests through nginx. You should set this to the largest file size that could be reasonable handled by your network. It defaults to 1M files, so will probably need to be increased if you are dealing with genome sized datasets.
Since nginx is more efficient at serving static content, it is best to serve it directly, reducing the load on the Galaxy process and allowing for more effective compression (if enabled), caching, and pipelining. To do so, add the following to your existing `server { }` block, replacing `$GALAXY_ROOT` with the correct path.:
```nginx
http {
server {
location /static {
alias $GALAXY_ROOT/static;
}
location /static/style {
alias $GALAXY_ROOT/static/style/blue;
}
location /static/scripts {
alias $GALAXY_ROOT/static/scripts;
}
location /favicon.ico {
alias $GALAXY_ROOT/static/favicon.ico;
}
location /robots.txt {
alias $GALAXY_ROOT/static/robots.txt;
}
}
}
```
You'll need to ensure that filesystem permissions are set such that the user running your nginx server has access to the Galaxy static/ directory.
### Serving Galaxy at a sub directory (such as /galaxy)
It may be necessary to house Galaxy at an address other than the web server root (`http://www.example.org/galaxy`), instead of `http://www.example.org`). To do this, you need to make the following changes:
1. In the nginx config, prefix all of the location directives with your prefix, like so:
```nginx
http {
server {
...
location /galaxy {
proxy_pass http://galaxy_app;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
location /galaxy/static {
alias $GALAXY_ROOT/static;
}
...
}
}
```
2. The Galaxy application needs to be aware that it is running with a prefix (for generating URLs in dynamic pages). This is accomplished by configuring a Paste proxy-prefix filter in the `[app:main]` section of `config/galaxy.ini` and restarting Galaxy:
```ini
[filter:proxy-prefix]
use = egg:PasteDeploy#prefix
prefix = /galaxy
[app:main]
filter-with = proxy-prefix
cookie_path = /galaxy
```
`cookie_prefix` should be set to prevent Galaxy's session cookies from clobbering each other if running more than one instance of Galaxy in different subdirectories on the same hostname.
### SSL
If you place Galaxy behind a proxy address that uses SSL (i.e., `https://` URLs), edit your galaxy location block (e.g. `location /` when served at the root, or something else like `location /galaxy` when served under a prefix)
```nginx
location / {
...
proxy_set_header X-URL-SCHEME https;
...
}
```
Setting `X-URL-SCHEME` makes Galaxy aware of what type of URL it should generate for external sites like Biomart. This should be added to the existing `location / { } ` block if you already have one, and adjusted accordingly if you're serving Galaxy from a subdirectory.
## Advanced Configuration Topics
### Compression and caching
All of Galaxy's static content can be cached on the client side, and everything (including dynamic content) can be compressed on the fly. This will decrease download and page load times for your clients, as well as decrease server load and bandwidth usage. To enable, you'll need nginx gzip support (which is standard unless compiled with `--without-http_gzip_module`), and the following in your `nginx.conf`:
```nginx
http {
...
gzip on;
gzip_http_version 1.1;
gzip_vary on;
gzip_comp_level 4;
gzip_proxied any;
gzip_types text/plain text/css application/x-javascript text/xml application/xml text/javascript application/json application/javascript;
gzip_buffers 16 8k;
gzip_disable "MSIE [1-6].(?!.*SV1)";
...
}
```
For caching, you'll need to add an `expires` directive to the `location /static { }` blocks:
```nginx
http {
server {
location /static {
alias $GALAXY_ROOT/static;
expires 24h;
}
location /static/style {
alias $GALAXY_ROOT/static/style/blue;
expires 24h;
}
}
}
```
The contents of `location /static { }` should be adjusted accordingly if you're serving Galaxy from a subdirectory.
### Sending files using Nginx
Galaxy sends files (e.g. dataset downloads) by opening the file and streaming it in chunks through the proxy server. However, this ties up the Galaxy process, which can impact the performance of other operations (see [Production Server Configuration](/src/admin/config/performance/production-server/index.md) for a more in-depth explanation).
Nginx can assume this task instead and as an added benefit, speed up downloads. This is accomplished through the use of the special `X-Accel-Redirect` header. Dataset security is maintained in this configuration because nginx will still check with Galaxy to ensure that the requesting user has permission to access the dataset before sending it.
To enable it, add the following to your `nginx.conf`:
```nginx
http {
server {
location /_x_accel_redirect/ {
internal;
alias /;
}
}
}
```
Finally edit your `$GALAXY_ROOT/config/galaxy.ini` and make the following change before restarting Galaxy:
```ini
[app:main]
nginx_x_accel_redirect_base = /_x_accel_redirect
```
For this to work, the user under which your nginx server runs will need read access to Galaxy's `$GALAXY_ROOT/database/files/` directory and its contents.
### Receiving files using nginx
Galaxy receives files (e.g. dataset uploads) by streaming them in chunks through the proxy server and writing the files to disk. However, this again ties up the Galaxy process. nginx can assume this task instead and as an added benefit, speed up uploads. This is accomplished through the use of `nginx_upload_module`, a 3rd-party nginx module.
To enable it, you must first [download](http://www.grid.net.ru/nginx/upload.en.html), compile and install `nginx_upload_module`. This means recompiling nginx. Once done, add the necessary directives to `nginx.conf`:
```nginx
user galaxy;
http {
server {
...
location /_upload {
upload_store $GALAXY_ROOT/database/tmp/upload_store;
upload_pass_form_field "";
upload_set_form_field " __${upload_field_name}__ is_composite" "true";
upload_set_form_field " __${upload_field_name}__ keys" "name path";
upload_set_form_field "${upload_field_name}_name" "$upload_file_name";
upload_set_form_field "${upload_field_name}_path" "$upload_tmp_path";
upload_pass_args on;
upload_pass /_upload_done;
}
location /_upload_done {
set $dst /api/tools;
if ($args ~ nginx_redir=([^&]+)) {
set $dst $1;
}
rewrite "" $dst;
}
...
}
}
```
Note the `user` directive. To ensure that Galaxy has write permission on the uploaded files, nginx's workers will need to run as the same user as Galaxy.
Finally edit your `$GALAXY_ROOT/config/galaxy.ini` and make the following change before restarting Galaxy:
```ini
[app:main]
nginx_upload_store = database/tmp/upload_store
nginx_upload_path = /_upload
```
When serving Galaxy with a prefix, as described in the serving Galaxy in a sub-directory section above, you will need to change one line in the `\_upload\_done` section. If your galaxy instance is available from `/galaxy`, then the first line should include this prefix:
```nginx
set $dst /galaxy/api/tools;
```
### Protect Galaxy Reports
Galaxy can run a separate reports app which gives useful information about your Galaxy instance.
To setup this reports app, have a look here https://docs.galaxyproject.org/en/master/admin/reports.html and here http://galacticengineer.blogspot.de/2015/06/exposing-galaxy-reports-via-nginx-in.html
After succesfully following the blogpost you will have your galaxy reports available at e.g. http://yourgalaxy/reports
To secure this page to only galaxy administrators, adjust your nginx config with the following snippets:
```nginx
# Add these snippets to your galaxy nginx configuration to make the reports
# daemon running on the default port 9001 available under the same address
# as your galaxy instance. In addition, the reports app will only be available
# to admins logged in to the galaxy instance.
upstream reports {
server localhost:9001;
}
server { # This you should already have.
listen 80;
(..) # The rest of your nginx configuration for galaxy
location /reports { # the section to make reports available
proxy_pass http://reports; # on the same host as your galaxy at e.g. http://galaxy/reports
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
satisfy any; # Restrict access
deny all;
auth_request /auth;
}
location /auth {
# The used galaxy api endpoint is only available to galaxy admins and thus limits the access
# to only logged in admins.
proxy_pass http://localhost/api/configuration/dynamic_tool_confs;
proxy_pass_request_body off;
proxy_set_header Content-Length "";
proxy_set_header X-Original-URI $request_uri;
}
(..)
}
```
### External User Authentication
- [Nginx for External Authentication](https://galaxyproject.org/admin/config/nginx-external-user-auth/)
- [Built-in Galaxy External Authentication](../authentication.html)
doc/source/admin/special_topics/performance_tracking.rst
+22 -18
View File
@@ -8,21 +8,21 @@ Most performance tracking requires sending metrics to a metrics collection serve
uWSGI
-----
As you have certainly switched to uWSGI from the default paste server, there is some built-in uWSGI support for performance logging. You can send uWSGI's internal metrics to a carbon (Graphite) server by setting the carbon option in your galaxy.ini:
As you have certainly switched to uWSGI from the default paste server, there is some built-in uWSGI support for performance logging. You can send uWSGI's internal metrics to a carbon (Graphite) server by setting the carbon option in your galaxy.yml:
.. code-block:: ini
.. code-block:: yaml
[uwsgi]
socket = ...
carbon = 127.0.0.1:2003
uwsgi:
socket: ...
carbon: 127.0.0.1:2003
Or a StatsD server via:
.. code-block:: ini
.. code-block:: yaml
[uwsgi]
socket = ...
statsd-push = 127.0.0.1:8125
wsgi:
socket: ...
statsd-push: 127.0.0.1:8125
The `official documentation <http://uwsgi-docs.readthedocs.io/en/latest/Metrics.html#stats-pushers>`__ contains further information on uWSGI and stats servers. In the `uWSGI Stats Server <http://uwsgi-docs.readthedocs.io/en/latest/StatsServer.html>` documentation, you can see an example of the sort of information that you will be able to collect.
@@ -31,21 +31,25 @@ API / Route Timing Statistics
Galaxy provides middleware to automatically log the amount of time controllers take to execute and to send that data to a stats server. Using the stats server of your choice, you can calculate the relevant statistics to ensure that your Galaxy server is performing as expected.
The statsD configuration requires setting the following:
The statsD configuration requires setting the following options in the ``galaxy`` section of ``config/galaxy.yml``:
.. code-block:: ini
.. code-block:: yaml
statsd_host=127.0.0.1
statsd_port=8125
statsd_prefix=galaxy
galaxy:
#...
statsd_host: 127.0.0.1
statsd_port: 8125
statsd_prefix: galaxy
And the graphite configuration is very similar:
.. code-block:: ini
.. code-block:: yaml
graphite_host=127.0.0.1
graphite_port=2003
graphite_prefix=galaxy
galaxy:
# ...
graphite_host: 127.0.0.1
graphite_port: 2003
graphite_prefix: galaxy
Most people visualize the statistics using something like `Grafana <https://grafana.com/>`__:
doc/source/admin/special_topics/webhooks.rst
+1 -1
View File
@@ -14,7 +14,7 @@ Plugin activation
-----------------
All webhooks that are included in the main Galaxy distribution are located in the ``config/plugins/webhooks/demo`` folder
and are deactivated by default.
To activate these demo webhooks make sure this path is added to ``webhooks_dir`` in your ``galaxy.ini``. You can add as many
To activate these demo webhooks make sure this path is added to ``webhooks_dir`` in your ``galaxy.yml``. You can add as many
webhook folders as you like as a comma separated list.
Webhooks supports one additional layer of activating/deactivating by changing the ``activate: true`` in each config of each webhook.
doc/source/admin/tool_panel.rst
+2 -2
View File
@@ -9,7 +9,7 @@ The contents of the tool panel are defined by the following configuration files.
Local tools
~~~~~~~~~~~
In the past, the file named by your ``tool_config_file`` configuration setting in your ``config/galaxy.ini`` file was the only file used to populate your Galaxy tool panel. The default name for this file is ``tool_conf.xml``. Since this was the only file involved in populating your Galaxy tool panel, it defined the items (tools, workflows, sections and labels) that would be displayed and the way in which they would be arranged.
In the past, the file named by your ``tool_config_file`` configuration setting in your ``config/galaxy.yml`` file was the only file used to populate your Galaxy tool panel. The default name for this file is ``tool_conf.xml``. Since this was the only file involved in populating your Galaxy tool panel, it defined the items (tools, workflows, sections and labels) that would be displayed and the way in which they would be arranged.
Tool Shed tools
~~~~~~~~~~~~~~~
@@ -23,7 +23,7 @@ Layout
------
The 3 or more files described in the previous section (``tool_conf.xml``, one or more ``shed_tool_conf.xml`` files, and ``migrated_tools_conf.xml``) are all used to load tool panel items (tools, sections, labels and workflows). A file named ``integrated_tool_panel.xml`` defines the arrangement for displaying these loaded items in your Galaxy tool panel.
If this file does not exist in your Galaxy installation directory, it will be automatically created and populated when you start your Galaxy server. It is initially populated based on the order in which the tool panel items are loaded. The items are loaded as each tool panel configuration file is parsed and its items are loaded. The order in which these configuration files are parsed is the order of the comma-separated list of files defined in your ``tool_config_file`` setting in your ``config/galaxy.ini`` configuration file. The ``migrated_tools_conf.xml`` file is always parsed and loaded last. Let's look at an example to help clarify how this works.
If this file does not exist in your Galaxy installation directory, it will be automatically created and populated when you start your Galaxy server. It is initially populated based on the order in which the tool panel items are loaded. The items are loaded as each tool panel configuration file is parsed and its items are loaded. The order in which these configuration files are parsed is the order of the comma-separated list of files defined in your ``tool_config_file`` setting in your ``config/galaxy.yml`` configuration file. The ``migrated_tools_conf.xml`` file is always parsed and loaded last. Let's look at an example to help clarify how this works.
If you uninstall a repository that contains tools, entries for those tools will automatically be removed from the shed-related tool panel config file and the integrated_tool_panel.xml file.
doc/source/conf.py
+3 -1
View File
@@ -46,7 +46,7 @@ sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pa
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc']
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx']
if not SKIP_SOURCE:
extensions += ['sphinx.ext.doctest', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.viewcode']
@@ -132,6 +132,8 @@ pygments_style = 'sphinx'
# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []
# Intersphinx mapping to Python 2.7 documentation
intersphinx_mapping = {'python': ('https://docs.python.org/2.7', None)}
# -- Options for HTML output ---------------------------------------------------
lib/galaxy/config.py
+5 -10
View File
@@ -120,6 +120,7 @@ LOGGING_CONFIG_DEFAULT = {
},
},
}
"""Default value for logging configuration, passed to :func:`logging.config.dictConfig`"""
def resolve_path(path, root):
@@ -187,10 +188,12 @@ class Configuration(object):
# Where dataset files are stored
self.file_path = resolve_path(kwargs.get("file_path", "database/files"), self.root)
# new_file_path and legacy_home_dir can be overridden per destination in job_conf.
self.new_file_path = resolve_path(kwargs.get("new_file_path", "database/tmp"), self.root)
override_tempdir = string_as_bool(kwargs.get("override_tempdir", "True"))
if override_tempdir:
tempfile.tempdir = self.new_file_path
self.shared_home_dir = kwargs.get("shared_home_dir", None)
self.openid_consumer_cache_path = resolve_path(kwargs.get("openid_consumer_cache_path", "database/openid_consumer_cache"), self.root)
self.cookie_path = kwargs.get("cookie_path", "/")
# Galaxy OpenID settings
@@ -481,7 +484,7 @@ class Configuration(object):
involucro_path = kwargs.get('involucro_path', None)
if involucro_path is None:
involucro_path = os.path.join(tool_dependency_dir, "involucro")
involucro_path = os.path.join(tool_dependency_dir or "database", "involucro")
self.involucro_path = resolve_path(involucro_path, self.root)
self.involucro_auto_init = string_as_bool(kwargs.get('involucro_auto_init', True))
@@ -530,7 +533,7 @@ class Configuration(object):
global_conf_parser = configparser.ConfigParser()
self.config_file = None
self.global_conf_parser = global_conf_parser
if global_conf and "__file__" in global_conf:
if global_conf and "__file__" in global_conf and ".yml" not in global_conf["__file__"]:
self.config_file = global_conf['__file__']
global_conf_parser.read(global_conf['__file__'])
# Heartbeat log file name override
@@ -580,14 +583,6 @@ class Configuration(object):
# Store advanced job management config
self.job_handlers = [x.strip() for x in kwargs.get('job_handlers', self.server_name).split(',')]
self.default_job_handlers = [x.strip() for x in kwargs.get('default_job_handlers', ','.join(self.job_handlers)).split(',')]
# Galaxy messaging (AMQP) configuration options
self.amqp = {}
try:
amqp_config = global_conf_parser.items("galaxy_amqp")
except configparser.NoSectionError:
amqp_config = {}
for k, v in amqp_config:
self.amqp[k] = v
# Galaxy internal control queue configuration.
# If specified in universe, use it, otherwise we use whatever 'real'
# database is specified. Lastly, we create and use new sqlite database
lib/galaxy/datatypes/binary.py
+103 -79
View File
@@ -187,14 +187,11 @@ class GenericAsn1Binary(Binary):
edam_data = "data_0849"
@dataproviders.decorators.has_dataproviders
class Bam(Binary):
"""Class describing a BAM binary file"""
class BamNative(Binary):
"""Class describing a BAM binary file that is not necessarily sorted"""
edam_format = "format_2572"
edam_data = "data_0863"
file_ext = "bam"
track_type = "ReadTrack"
data_sources = {"data": "bai", "index": "bigwig"}
file_ext = "bam_native"
MetadataElement(name="bam_index", desc="BAM Index File", param=metadata.FileParameter, file_ext="bai", readonly=True, no_value=None, visible=False, optional=True)
MetadataElement(name="bam_version", default=None, desc="BAM Version", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value=None)
@@ -217,80 +214,9 @@ class Bam(Binary):
"""
pysam.merge('-O', 'BAM', output_file, *split_files)
def dataset_content_needs_grooming(self, file_name):
"""
Check if file_name is a coordinate-sorted BAM file
"""
# The best way to ensure that BAM files are coordinate-sorted and indexable
# is to actually index them.
index_name = tempfile.NamedTemporaryFile(prefix="bam_index").name
try:
# If pysam fails to index a file it will write to stderr,
# and this causes the set_meta script to fail. So instead
# we start another process and discard stderr.
cmd = ['python', '-c', "import pysam; pysam.index('%s', '%s')" % (file_name, index_name)]
with open(os.devnull, 'w') as devnull:
subprocess.check_call(cmd, stderr=devnull, shell=False)
needs_sorting = False
except subprocess.CalledProcessError:
needs_sorting = True
try:
os.unlink(index_name)
except Exception:
pass
return needs_sorting
def groom_dataset_content(self, file_name):
"""
Ensures that the BAM file contents are sorted. This function is called
on an output dataset after the content is initially generated.
"""
# Use pysam to sort the BAM file
# This command may also creates temporary files <out.prefix>.%d.bam when the
# whole alignment cannot fit into memory.
# do this in a unique temp directory, because of possible <out.prefix>.%d.bam temp files
if not self.dataset_content_needs_grooming(file_name):
# Don't re-sort if already sorted
return
tmp_dir = tempfile.mkdtemp()
tmp_sorted_dataset_file_name_prefix = os.path.join(tmp_dir, 'sorted')
sorted_file_name = "%s.bam" % tmp_sorted_dataset_file_name_prefix
slots = os.environ.get('GALAXY_SLOTS', 1)
try:
pysam.sort("-@%s" % slots, file_name, '-T', tmp_sorted_dataset_file_name_prefix, '-O', 'BAM', '-o', sorted_file_name)
except Exception:
shutil.rmtree(tmp_dir, ignore_errors=True)
raise
# Move samtools_created_sorted_file_name to our output dataset location
shutil.move(sorted_file_name, file_name)
# Remove temp file and empty temporary directory
os.rmdir(tmp_dir)
def init_meta(self, dataset, copy_from=None):
Binary.init_meta(self, dataset, copy_from=copy_from)
def set_meta(self, dataset, overwrite=True, **kwd):
# These metadata values are not accessible by users, always overwrite
index_file = dataset.metadata.bam_index
if not index_file:
index_file = dataset.metadata.spec['bam_index'].param.new_file(dataset=dataset)
pysam.index(dataset.file_name, index_file.file_name)
dataset.metadata.bam_index = index_file
# Now use pysam with BAI index to determine additional metadata
try:
bam_file = pysam.AlignmentFile(dataset.file_name, mode='rb', index_filename=index_file.file_name)
# TODO: Reference names, lengths, read_groups and headers can become very large, truncate when necessary
dataset.metadata.reference_names = list(bam_file.references)
dataset.metadata.reference_lengths = list(bam_file.lengths)
dataset.metadata.bam_header = bam_file.header
dataset.metadata.read_groups = [read_group['ID'] for read_group in dataset.metadata.bam_header.get('RG', []) if 'ID' in read_group]
dataset.metadata.sort_order = bam_file.header.get('HD', {}).get('SO', None)
dataset.metadata.bam_version = bam_file.header.get('HD', {}).get('VN', None)
except Exception:
# Per Dan, don't log here because doing so will cause datasets that
# fail metadata to end in the error state
pass
def sniff(self, filename):
# BAM is compressed in the BGZF format, and must not be uncompressed in Galaxy.
# The first 4 bytes of any bam file is 'BAM\1', and the file is binary.
@@ -302,6 +228,21 @@ class Bam(Binary):
except Exception:
return False
def set_meta(self, dataset, overwrite=True, **kwd):
try:
bam_file = pysam.AlignmentFile(dataset.file_name, mode='rb')
# TODO: Reference names, lengths, read_groups and headers can become very large, truncate when necessary
dataset.metadata.reference_names = list(bam_file.references)
dataset.metadata.reference_lengths = list(bam_file.lengths)
dataset.metadata.bam_header = bam_file.header
dataset.metadata.read_groups = [read_group['ID'] for read_group in dataset.metadata.bam_header.get('RG', []) if 'ID' in read_group]
dataset.metadata.sort_order = bam_file.header.get('HD', {}).get('SO', None)
dataset.metadata.bam_version = bam_file.header.get('HD', {}).get('VN', None)
except Exception:
# Per Dan, don't log here because doing so will cause datasets that
# fail metadata to end in the error state
pass
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
dataset.peek = "Binary bam alignments file"
@@ -326,10 +267,9 @@ class Bam(Binary):
return zip(file_paths, rel_paths)
def get_chunk(self, trans, dataset, offset=0, ck_size=None):
index_file = dataset.metadata.bam_index
if not offset == -1:
try:
with pysam.AlignmentFile(dataset.file_name, "rb", index_filename=index_file.file_name) as bamfile:
with pysam.AlignmentFile(dataset.file_name, "rb") as bamfile:
ck_size = 300 # 300 lines
ck_data = ""
header_line_count = 0
@@ -382,6 +322,90 @@ class Bam(Binary):
column_names=column_names,
column_types=column_types)
@dataproviders.decorators.has_dataproviders
class Bam(BamNative):
"""Class describing a BAM binary file"""
edam_format = "format_2572"
edam_data = "data_0863"
file_ext = "bam"
track_type = "ReadTrack"
data_sources = {"data": "bai", "index": "bigwig"}
def dataset_content_needs_grooming(self, file_name):
"""
Check if file_name is a coordinate-sorted BAM file
"""
# The best way to ensure that BAM files are coordinate-sorted and indexable
# is to actually index them.
index_name = tempfile.NamedTemporaryFile(prefix="bam_index").name
try:
# If pysam fails to index a file it will write to stderr,
# and this causes the set_meta script to fail. So instead
# we start another process and discard stderr.
cmd = ['python', '-c', "import pysam; pysam.index('%s', '%s')" % (file_name, index_name)]
with open(os.devnull, 'w') as devnull:
subprocess.check_call(cmd, stderr=devnull, shell=False)
needs_sorting = False
except subprocess.CalledProcessError:
needs_sorting = True
try:
os.unlink(index_name)
except Exception:
pass
return needs_sorting
def groom_dataset_content(self, file_name):
"""
Ensures that the BAM file contents are sorted. This function is called
on an output dataset after the content is initially generated.
"""
# Use pysam to sort the BAM file
# This command may also creates temporary files <out.prefix>.%d.bam when the
# whole alignment cannot fit into memory.
# do this in a unique temp directory, because of possible <out.prefix>.%d.bam temp files
if not self.dataset_content_needs_grooming(file_name):
# Don't re-sort if already sorted
return
tmp_dir = tempfile.mkdtemp()
tmp_sorted_dataset_file_name_prefix = os.path.join(tmp_dir, 'sorted')
sorted_file_name = "%s.bam" % tmp_sorted_dataset_file_name_prefix
slots = os.environ.get('GALAXY_SLOTS', 1)
try:
pysam.sort("-@%s" % slots, file_name, '-T', tmp_sorted_dataset_file_name_prefix, '-O', 'BAM', '-o', sorted_file_name)
except Exception:
shutil.rmtree(tmp_dir, ignore_errors=True)
raise
# Move samtools_created_sorted_file_name to our output dataset location
shutil.move(sorted_file_name, file_name)
# Remove temp file and empty temporary directory
os.rmdir(tmp_dir)
def set_meta(self, dataset, overwrite=True, **kwd):
# These metadata values are not accessible by users, always overwrite
index_file = dataset.metadata.bam_index
if not index_file:
index_file = dataset.metadata.spec['bam_index'].param.new_file(dataset=dataset)
pysam.index(dataset.file_name, index_file.file_name)
dataset.metadata.bam_index = index_file
# Now use pysam with BAI index to determine additional metadata
try:
bam_file = pysam.AlignmentFile(dataset.file_name, mode='rb', index_filename=index_file.file_name)
# TODO: Reference names, lengths, read_groups and headers can become very large, truncate when necessary
dataset.metadata.reference_names = list(bam_file.references)
dataset.metadata.reference_lengths = list(bam_file.lengths)
dataset.metadata.bam_header = bam_file.header
dataset.metadata.read_groups = [read_group['ID'] for read_group in dataset.metadata.bam_header.get('RG', []) if 'ID' in read_group]
dataset.metadata.sort_order = bam_file.header.get('HD', {}).get('SO', None)
dataset.metadata.bam_version = bam_file.header.get('HD', {}).get('VN', None)
except Exception:
# Per Dan, don't log here because doing so will cause datasets that
# fail metadata to end in the error state
pass
def sniff(self, file_name):
return super(Bam, self).sniff(file_name) and not self.dataset_content_needs_grooming(file_name)
# ------------- Dataproviders
# pipe through samtools view
# ALSO: (as Sam)
lib/galaxy/datatypes/converters/bam_native_to_bam_converter.xml
+23
View File
@@ -0,0 +1,23 @@
<tool id="CONVERTER_bam_native_to_bam" name="Convert BAM native to BAM" version="1.0.0" hidden="true">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<requirements>
<requirement type="package" version="1.6">samtools</requirement>
</requirements>
<command><![CDATA[
samtools sort
-@ \${GALAXY_SLOTS:-1}
-o '${output}'
-O bam
-T dataset
'${input}'
]]>
</command>
<inputs>
<param format="bam_native" name="input" type="data" label="Choose a BAM native file"/>
</inputs>
<outputs>
<data format="bam" name="output"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/converters/bam_to_bigwig_converter.xml
+1 -1
View File
@@ -17,7 +17,7 @@
> temp.bg && bedGraphToBigWig temp.bg '$chromInfo' '$output']]>
</command>
<inputs>
<param format="bam" name="input" type="data" label="Choose BAM file"/>
<param format="bam,bam_native" name="input" type="data" label="Choose BAM file"/>
</inputs>
<outputs>
<data format="bigwig" name="output"/>
lib/galaxy/datatypes/converters/sam_to_bam_native.xml
+23
View File
@@ -0,0 +1,23 @@
<tool id="CONVERTER_sam_to_bam_native" name="Convert SAM to BAM native - without sorting" version="1.0.0">
<!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
<requirements>
<requirement type="package" version="1.6">samtools</requirement>
</requirements>
<command><![CDATA[
samtools view
-b
-h
-@ \${GALAXY_SLOTS:-2}
-o '${output}'
'$input'
]]>
</command>
<inputs>
<param name="input" type="data" format="sam" label="SAM file"/>
</inputs>
<outputs>
<data name="output" format="bam_native"/>
</outputs>
<help>
</help>
</tool>
lib/galaxy/datatypes/sniff.py
+1 -1
View File
@@ -337,7 +337,7 @@ def guess_ext(fname, sniff_order):
'bam'
>>> fname = get_test_fname('3unsorted.bam')
>>> guess_ext(fname, sniff_order)
'bam'
'bam_native'
>>> fname = get_test_fname('test.idpDB')
>>> guess_ext(fname, sniff_order)
'idpdb'
lib/galaxy/jobs/__init__.py
+58 -2
View File
@@ -1110,6 +1110,15 @@ class JobWrapper(object, HasResourceParameters):
if flush:
self.sa_session.flush()
@property
def home_target(self):
home_target = self.tool.home_target
return home_target
@property
def tmp_target(self):
return self.tool.tmp_target
def get_destination_configuration(self, key, default=None):
""" Get a destination parameter that can be defaulted back
in app.config if it needs to be applied globally.
@@ -1251,7 +1260,7 @@ class JobWrapper(object, HasResourceParameters):
if job.states.ERROR == final_job_state:
dataset.blurb = "error"
dataset.mark_unhidden()
elif not purged and dataset.has_data():
elif not purged:
# If the tool was expected to set the extension, attempt to retrieve it
if dataset.ext == 'auto':
dataset.extension = context.get('ext', 'data')
@@ -1299,7 +1308,7 @@ class JobWrapper(object, HasResourceParameters):
except Exception:
dataset.set_peek()
else:
# Handle an empty dataset.
# Handle purged datasets.
dataset.blurb = "empty"
if dataset.ext == 'auto':
dataset.extension = context.get('ext', 'txt')
@@ -1603,6 +1612,39 @@ class JobWrapper(object, HasResourceParameters):
return dp.dataset_id
return None
@property
def tmp_dir_creation_statement(self):
tmp_dir = self.get_destination_configuration("tmp_dir", None)
if not tmp_dir or tmp_dir.lower() == "true":
working_directory = self.working_directory
return '''$([ ! -e '{0}/tmp' ] || mv '{0}/tmp' '{0}'/tmp.$(date +%Y%m%d-%H%M%S) ; mkdir '{0}/tmp'; echo '{0}/tmp')'''.format(working_directory)
else:
return tmp_dir
def home_directory(self):
home_target = self.home_target
return self._target_to_directory(home_target)
def tmp_directory(self):
tmp_target = self.tmp_target
return self._target_to_directory(tmp_target)
def _target_to_directory(self, target):
working_directory = self.working_directory
tmp_dir = self.get_destination_configuration("tmp_dir", None)
if target is None or (target == "job_tmp_if_explicit" and tmp_dir is None):
return None
elif target in ["job_tmp", "job_tmp_if_explicit"]:
return "$_GALAXY_JOB_TMP_DIR"
elif target == "shared_home":
return self.get_destination_configuration("shared_home_dir", None)
elif target == "job_home":
return "$_GALAXY_JOB_HOME_DIR"
elif target == "pwd":
return os.path.join(working_directory, "working")
else:
raise Exception("Unknown target type [%s]" % target)
def get_tool_provided_job_metadata(self):
if self.tool_provided_job_metadata is not None:
return self.tool_provided_job_metadata
@@ -2027,6 +2069,14 @@ class ComputeEnvironment(object):
be rewritten.)
"""
@abstractmethod
def home_directory(self):
"""Home directory of target job - none if HOME should not be set."""
@abstractmethod
def tmp_directory(self):
"""Temp directory of target job - none if HOME should not be set."""
class SimpleComputeEnvironment(object):
@@ -2072,6 +2122,12 @@ class SharedComputeEnvironment(SimpleComputeEnvironment):
def tool_directory(self):
return os.path.abspath(self.job_wrapper.tool.tool_dir)
def home_directory(self):
return self.job_wrapper.home_directory()
def tmp_directory(self):
return self.job_wrapper.tmp_directory()
class NoopQueue(object):
"""
lib/galaxy/jobs/rule_helper.py
+1 -1
View File
@@ -44,7 +44,7 @@ class RuleHelper(object):
tool = self.app.toolbox.get_tool(job_or_tool.tool_id, tool_version=job_or_tool.tool_version)
# Can't import at top because circular import between galaxy.tools and galaxy.jobs.
import galaxy.tools.deps.containers
tool_info = galaxy.tools.deps.containers.ToolInfo(tool.containers, tool.requirements, tool.requires_galaxy_python_environment)
tool_info = galaxy.tools.deps.containers.ToolInfo(tool.containers, tool.requirements, tool.requires_galaxy_python_environment, tool.docker_env_pass_through)
container_description = self.app.container_finder.find_best_container_description(["docker"], tool_info)
return container_description is not None
lib/galaxy/jobs/runners/__init__.py
+8 -1
View File
@@ -327,7 +327,9 @@ class BaseJobRunner(object):
for env in envs:
env_setup_commands.append(env_to_statement(env))
command_line = job_wrapper.runner_command_line
tmp_dir_creation_statement = job_wrapper.tmp_dir_creation_statement
options = dict(
tmp_dir_creation_statement=tmp_dir_creation_statement,
job_instrumenter=job_instrumenter,
galaxy_lib=job_wrapper.galaxy_lib_dir,
galaxy_virtual_env=job_wrapper.galaxy_virtual_env,
@@ -353,6 +355,7 @@ class BaseJobRunner(object):
compute_working_directory=None,
compute_tool_directory=None,
compute_job_directory=None,
compute_tmp_directory=None,
):
job_directory_type = "galaxy" if compute_working_directory is None else "pulsar"
if not compute_working_directory:
@@ -364,13 +367,17 @@ class BaseJobRunner(object):
if not compute_tool_directory:
compute_tool_directory = job_wrapper.tool.tool_dir
if not compute_tmp_directory:
compute_tmp_directory = job_wrapper.tmp_directory()
tool = job_wrapper.tool
from galaxy.tools.deps import containers
tool_info = containers.ToolInfo(tool.containers, tool.requirements, tool.requires_galaxy_python_environment)
tool_info = containers.ToolInfo(tool.containers, tool.requirements, tool.requires_galaxy_python_environment, tool.docker_env_pass_through)
job_info = containers.JobInfo(
compute_working_directory,
compute_tool_directory,
compute_job_directory,
compute_tmp_directory,
job_directory_type,
)
lib/galaxy/jobs/runners/pulsar.py
+10
View File
@@ -871,6 +871,16 @@ class PulsarComputeEnvironment(ComputeEnvironment):
def tool_directory(self):
return self._tool_dir
def home_directory(self):
# TODO: revisit and implement this, won't break anything working in the
# meantime.
return None
def tmp_directory(self):
# TODO: revisit and implement this, won't break anything working in the
# meantime.
return None
class UnsupportedPulsarException(Exception):
lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh
+5
View File
@@ -12,7 +12,12 @@ _galaxy_setup_environment() {
fi
export PYTHONPATH
fi
_GALAXY_JOB_HOME_DIR="$working_directory/home"
_GALAXY_JOB_TMP_DIR=$tmp_dir_creation_statement
$env_setup_commands
# These don't get cleaned on a re-run but may in the future.
[ -z "$_GALAXY_JOB_TMP_DIR" -a ! -f "$_GALAXY_JOB_TMP_DIR" ] || mkdir -p "$_GALAXY_JOB_TMP_DIR"
[ -z "$_GALAXY_JOB_HOME_DIR" -a ! -f "$_GALAXY_JOB_HOME_DIR" ] || mkdir -p "$_GALAXY_JOB_HOME_DIR"
if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" \
-a "`command -v python`" != "$GALAXY_VIRTUAL_ENV/bin/python" ]; then
. "$GALAXY_VIRTUAL_ENV/bin/activate"
lib/galaxy/jobs/runners/util/job_script/__init__.py
+1
View File
@@ -52,6 +52,7 @@ OPTIONAL_TEMPLATE_PARAMS = {
'integrity_injection': INTEGRITY_INJECTION,
'shell': DEFAULT_SHELL,
'preserve_python_environment': True,
'tmp_dir_creation_statement': '""',
}
lib/galaxy/managers/library_datasets.py
+5 -1
View File
@@ -78,6 +78,10 @@ class LibraryDatasetsManager(object):
if new_misc_info is not None and new_misc_info != ldda.info:
ldda.info = new_misc_info
changed = True
new_message = new_data.get('message', None)
if new_message is not None and new_message != ldda.message:
ldda.message = new_message
changed = True
new_file_ext = new_data.get('file_ext', None)
if new_file_ext is not None and new_file_ext != ldda.extension:
ldda.extension = new_file_ext
@@ -103,7 +107,7 @@ class LibraryDatasetsManager(object):
raise RequestParameterInvalidException('%s must have at least length of %s' % (key, MINIMUM_STRING_LENGTH))
val = validation.validate_and_sanitize_basestring(key, val)
validated_payload[key] = val
if key in ('misc_info'):
if key in ('misc_info', 'message'):
val = validation.validate_and_sanitize_basestring(key, val)
validated_payload[key] = val
if key in ('file_ext'):
lib/galaxy/objectstore/__init__.py
+10 -2
View File
@@ -10,6 +10,7 @@ import os
import random
import shutil
import threading
import time
from xml.etree import ElementTree
try:
@@ -358,7 +359,7 @@ class DiskObjectStore(ObjectStore):
def empty(self, obj, **kwargs):
"""Override `ObjectStore`'s stub by checking file size on disk."""
return os.path.getsize(self.get_filename(obj, **kwargs)) == 0
return self.size(obj, **kwargs) == 0
def size(self, obj, **kwargs):
"""Override `ObjectStore`'s stub by return file size on disk.
@@ -367,7 +368,14 @@ class DiskObjectStore(ObjectStore):
"""
if self.exists(obj, **kwargs):
try:
return os.path.getsize(self.get_filename(obj, **kwargs))
filepath = self.get_filename(obj, **kwargs)
for _ in range(0, 2):
size = os.path.getsize(filepath)
if size != 0:
break
# May be legitimately 0, or there may be an issue with the FS / kernel, so we try again
time.sleep(0.01)
return size
except OSError:
return 0
else:
lib/galaxy/tools/__init__.py
+21 -1
View File
@@ -623,6 +623,23 @@ class Tool(object, Dictifiable):
self.parse_command(tool_source)
self.environment_variables = self.parse_environment_variables(tool_source)
self.tmp_directory_vars = tool_source.parse_tmp_directory_vars()
home_target = tool_source.parse_home_target()
tmp_target = tool_source.parse_tmp_target()
# If a tool explicitly sets one of these variables just respect that and turn off
# explicit processing by Galaxy.
for environment_variable in self.environment_variables:
if environment_variable.get("name") == "HOME":
home_target = None
continue
for tmp_directory_var in self.tmp_directory_vars:
if environment_variable.get("name") == tmp_directory_var:
tmp_target = None
break
self.home_target = home_target
self.tmp_target = tmp_target
self.docker_env_pass_through = tool_source.parse_docker_env_pass_through()
# Parameters used to build URL for redirection to external app
redirect_url_params = tool_source.parse_redirect_url_params_elem()
@@ -1890,6 +1907,9 @@ class Tool(object, Dictifiable):
try:
if [hda.dependent_jobs for hda in [jtod.dataset for jtod in job.output_datasets] if hda.dependent_jobs]:
return True
elif job.output_dataset_collection_instances:
# We'll want to replace this item
return 'job_produced_collection_elements'
except Exception as exception:
log.error(str(exception))
pass
@@ -2517,7 +2537,7 @@ class SortTool(DatabaseOperationTool):
try:
sorted_elements = [old_elements_dict[line.strip()] for line in open(hda.file_name)]
except KeyError:
hdca_history_name = "%s: %s" (hdca.hid, hdca.name)
hdca_history_name = "%s: %s" % (hdca.hid, hdca.name)
message = "List of element identifiers does not match element identifiers in collection '%s'" % hdca_history_name
raise Exception(message)
lib/galaxy/tools/actions/__init__.py
+34 -26
View File
@@ -534,43 +534,51 @@ class DefaultToolAction(object):
for p in old_job.parameters:
if p.name.endswith('|__identifier__'):
current_job.parameters.append(p.copy())
remapped_hdas = {}
input_hdcas = set()
remapped_hdas = self.__remap_data_inputs(old_job=old_job, current_job=current_job)
for jtod in old_job.output_datasets:
for (job_to_remap, jtid) in [(jtid.job, jtid) for jtid in jtod.dataset.dependent_jobs]:
if (trans.user is not None and job_to_remap.user_id == trans.user.id) or (
trans.user is None and job_to_remap.session_id == galaxy_session.id):
if job_to_remap.state == job_to_remap.states.PAUSED:
job_to_remap.state = job_to_remap.states.NEW
for hda in [dep_jtod.dataset for dep_jtod in job_to_remap.output_datasets]:
if hda.state == hda.states.PAUSED:
hda.state = hda.states.NEW
hda.info = None
input_values = dict([(p.name, json.loads(p.value)) for p in job_to_remap.parameters])
remapped_hdas[jtod.dataset] = out_data[jtod.name]
for jtidca in job_to_remap.input_dataset_collections:
input_hdcas.add(jtidca.dataset_collection)
old_dataset_id = jtod.dataset_id
new_dataset_id = out_data[jtod.name].id
input_values = update_dataset_ids(input_values, {old_dataset_id: new_dataset_id}, src='hda')
for p in job_to_remap.parameters:
p.value = json.dumps(input_values[p.name])
jtid.dataset = out_data[jtod.name]
jtid.dataset.hid = jtod.dataset.hid
log.info('Job %s input HDA %s remapped to new HDA %s' % (job_to_remap.id, jtod.dataset.id, jtid.dataset.id))
self.__remap_parameters(job_to_remap, jtid, jtod, out_data)
trans.sa_session.add(job_to_remap)
trans.sa_session.add(jtid)
for hdca in input_hdcas:
hdca.collection.replace_failed_elements(remapped_hdas)
if hdca.implicit_collection_jobs:
for job in hdca.implicit_collection_jobs.jobs:
if job.job_id == old_job.id:
job.job_id = current_job.id
jtod.dataset.visible = False
trans.sa_session.add(jtod)
for jtodc in old_job.output_dataset_collection_instances:
hdca = jtodc.dataset_collection_instance
hdca.collection.replace_failed_elements(remapped_hdas)
if hdca.implicit_collection_jobs:
for job in hdca.implicit_collection_jobs.jobs:
if job.job_id == old_job.id:
job.job_id = current_job.id
except Exception:
log.exception('Cannot remap rerun dependencies.')
def __remap_data_inputs(self, old_job, current_job):
"""Record output datasets from old_job and build a dictionary that maps the old output HDAs to the new output HDAs."""
remapped_hdas = {}
old_output_datasets = {jtod.name: jtod.dataset for jtod in old_job.output_datasets}
for jtod in current_job.output_datasets:
remapped_hdas[old_output_datasets[jtod.name]] = jtod.dataset
return remapped_hdas
def __remap_parameters(self, job_to_remap, jtid, jtod, out_data):
if job_to_remap.state == job_to_remap.states.PAUSED:
job_to_remap.state = job_to_remap.states.NEW
for hda in [dep_jtod.dataset for dep_jtod in job_to_remap.output_datasets]:
if hda.state == hda.states.PAUSED:
hda.state = hda.states.NEW
hda.info = None
input_values = dict([(p.name, json.loads(p.value)) for p in job_to_remap.parameters])
old_dataset_id = jtod.dataset_id
new_dataset_id = out_data[jtod.name].id
input_values = update_dataset_ids(input_values, {old_dataset_id: new_dataset_id}, src='hda')
for p in job_to_remap.parameters:
p.value = json.dumps(input_values[p.name])
jtid.dataset = out_data[jtod.name]
jtid.dataset.hid = jtod.dataset.hid
log.info('Job %s input HDA %s remapped to new HDA %s' % (job_to_remap.id, jtod.dataset.id, jtid.dataset.id))
def _wrapped_params(self, trans, tool, incoming, input_datasets=None):
wrapped_params = WrappedParameters(trans, tool, incoming, input_datasets=input_datasets)
return wrapped_params
lib/galaxy/tools/data_manager/manager.py
+7 -7
View File
@@ -317,10 +317,10 @@ class DataManager(object):
self.process_move(data_table_name, name, output_ref_values[name].extra_files_path, **data_table_value)
data_table_value[name] = self.process_value_translation(data_table_name, name, **data_table_value)
data_table.add_entry(data_table_value, persist=True, entry_source=self)
send_control_task(self.data_managers.app,
'reload_tool_data_tables',
noop_self=True,
kwargs={'table_name': data_table_name})
send_control_task(self.data_managers.app,
'reload_tool_data_tables',
noop_self=True,
kwargs={'table_name': data_table_name})
if self.undeclared_tables and data_tables_dict:
# We handle the data move, by just moving all the data out of the extra files path
# moving a directory and the target already exists, we move the contents instead
@@ -338,9 +338,9 @@ class DataManager(object):
if name in path_column_names:
data_table_value[name] = os.path.abspath(os.path.join(self.data_managers.app.config.galaxy_data_manager_data_path, value))
data_table.add_entry(data_table_value, persist=True, entry_source=self)
send_control_task(self.data_managers.app, 'reload_tool_data_tables',
noop_self=True,
kwargs={'table_name': data_table_name})
send_control_task(self.data_managers.app, 'reload_tool_data_tables',
noop_self=True,
kwargs={'table_name': data_table_name})
else:
for data_table_name, data_table_values in data_tables_dict.items():
# tool returned extra data table entries, but data table was not declared in data manager
lib/galaxy/tools/deps/containers.py
+14 -3
View File
@@ -283,21 +283,24 @@ class ToolInfo(object):
# variables they can consume (e.g. JVM options, license keys, etc..)
# and add these to env_path_through
def __init__(self, container_descriptions=[], requirements=[], requires_galaxy_python_environment=False):
def __init__(self, container_descriptions=[], requirements=[], requires_galaxy_python_environment=False, env_pass_through=["GALAXY_SLOTS"]):
self.container_descriptions = container_descriptions
self.requirements = requirements
self.requires_galaxy_python_environment = requires_galaxy_python_environment
self.env_pass_through = ["GALAXY_SLOTS"]
self.env_pass_through = env_pass_through
class JobInfo(object):
def __init__(self, working_directory, tool_directory, job_directory, job_directory_type):
def __init__(
self, working_directory, tool_directory, job_directory, tmp_directory, job_directory_type
):
self.working_directory = working_directory
self.job_directory = job_directory
# Tool files may be remote staged - so this is unintuitively a property
# of the job not of the tool.
self.tool_directory = tool_directory
self.tmp_directory = tmp_directory
self.job_directory_type = job_directory_type # "galaxy" or "pulsar"
@@ -394,6 +397,7 @@ class HasDockerLikeVolumes:
variables[name] = os.path.abspath(value)
add_var("working_directory", self.job_info.working_directory)
add_var("tmp_directory", self.job_info.tmp_directory)
add_var("job_directory", self.job_info.job_directory)
add_var("tool_directory", self.job_info.tool_directory)
add_var("galaxy_root", self.app_info.galaxy_root_dir)
@@ -408,6 +412,8 @@ class HasDockerLikeVolumes:
defaults = "$galaxy_root:default_ro,$tool_directory:default_ro"
if self.job_info.job_directory:
defaults += ",$job_directory:default_ro"
if self.job_info.tmp_directory is not None:
defaults += ",$tmp_directory:rw"
if self.app_info.outputs_to_working_directory:
# Should need default_file_path (which is of course an estimate given
# object stores anyway).
@@ -454,6 +460,11 @@ class DockerContainer(Container, HasDockerLikeVolumes):
preprocessed_volumes_str = preprocess_volumes(volumes_raw, self.container_type)
# TODO: Remove redundant volumes...
volumes = docker_util.DockerVolume.volumes_from_str(preprocessed_volumes_str)
# If a tool definitely has a temp directory available set it to /tmp in container for compat.
# with CWL. This is part of that spec and should make it easier to share containers between CWL
# and Galaxy.
if self.job_info.tmp_directory is not None:
volumes.append(docker_util.DockerVolume.volume_from_str("%s:/tmp:rw" % self.job_info.tmp_directory))
volumes_from = self.destination_info.get("docker_volumes_from", docker_util.DEFAULT_VOLUMES_FROM)
docker_host_props = dict(
lib/galaxy/tools/deps/docker_util.py
+10 -3
View File
@@ -155,7 +155,9 @@ def build_docker_run_command(
if terminal:
command_parts.append("-t")
for env_directive in env_directives:
command_parts.extend(["-e", shlex_quote(env_directive)])
# e.g. -e "GALAXY_SLOTS=$GALAXY_SLOTS"
# These are environment variable expansions so we don't quote these.
command_parts.extend(["-e", env_directive])
for volume in volumes:
command_parts.extend(["-v", shlex_quote(str(volume))])
if volumes_from:
@@ -175,8 +177,13 @@ def build_docker_run_command(
if set_user:
user = set_user
if set_user == DEFAULT_SET_USER:
user = str(os.geteuid())
command_parts.extend(["-u", user])
# If future-us is ever in here and fixing this for docker-machine just
# use cwltool.docker_id - it takes care of this default nicely.
euid = os.geteuid()
egid = os.getgid()
user = "%d:%d" % (euid, egid)
command_parts.extend(["--user", user])
full_image = image
if tag:
full_image = "%s:%s" % (full_image, tag)
lib/galaxy/tools/error_reports/plugins/sentry.py
+12 -7
View File
@@ -73,6 +73,17 @@ class SentryPlugin(ErrorPlugin):
error_message = ERROR_TEMPLATE.format(**extra)
# Update context with user information in a sentry-specific manner
# Getting the url allows us to link to the dataset info page in case
# anything is missing from this report.
try:
url = web.url_for(controller="dataset",
action="show_params",
dataset_id=self.app.security.encode_id(dataset.id),
qualified=True)
except AttributeError:
# The above does not work when handlers are separate from the web handlers
url = None
self.sentry_client.context.merge({
# User information here also places email links + allows seeing
# a list of affected users in the tags/filtering.
@@ -80,14 +91,8 @@ class SentryPlugin(ErrorPlugin):
'name': user.username,
'email': user.email,
},
# This allows us to link to the dataset info page in case
# anything is missing from this report.
'request': {
'url': web.url_for(
controller="dataset", action="show_params",
dataset_id=self.app.security.encode_id(dataset.id),
qualified=True
)
'url': url
}
})
lib/galaxy/tools/evaluation.py
+9
View File
@@ -521,6 +521,15 @@ class ToolEvaluator(object):
environment_variable["raw"] = True
environment_variables.append(environment_variable)
home_dir = self.compute_environment.home_directory()
tmp_dir = self.compute_environment.tmp_directory()
if home_dir:
environment_variable = dict(name="HOME", value='"%s"' % home_dir, raw=True)
environment_variables.append(environment_variable)
if tmp_dir:
for tmp_directory_var in self.tool.tmp_directory_vars:
environment_variable = dict(name=tmp_directory_var, value='"%s"' % tmp_dir, raw=True)
environment_variables.append(environment_variable)
self.environment_variables = environment_variables
return environment_variables
lib/galaxy/tools/parameters/output_collect.py
+31 -19
View File
@@ -474,43 +474,54 @@ def discover_files(output_name, tool_provided_metadata, extra_file_collectors, j
# just load entries from tool provided metadata...
assert len(extra_file_collectors) == 1
extra_file_collector = extra_file_collectors[0]
target_directory = discover_target_directory(extra_file_collector, job_working_directory)
target_directory = discover_target_directory(extra_file_collector.directory, job_working_directory)
for dataset in tool_provided_metadata.get_new_datasets(output_name):
filename = dataset["filename"]
path = os.path.join(target_directory, filename)
yield DiscoveredFile(path, extra_file_collector, JsonCollectedDatasetMatch(dataset, extra_file_collector, filename, path=path))
else:
for (match, collector) in walk_over_extra_files(extra_file_collectors, job_working_directory, matchable):
for (match, collector) in walk_over_file_collectors(extra_file_collectors, job_working_directory, matchable):
yield DiscoveredFile(match.path, collector, match)
def discover_target_directory(extra_file_collector, job_working_directory):
directory = job_working_directory
if extra_file_collector.directory:
directory = os.path.join(directory, extra_file_collector.directory)
def discover_target_directory(dir_name, job_working_directory):
if dir_name:
directory = os.path.join(job_working_directory, dir_name)
if not util.in_directory(directory, job_working_directory):
raise Exception("Problem with tool configuration, attempting to pull in datasets from outside working directory.")
return directory
return directory
else:
return job_working_directory
def walk_over_extra_files(extra_file_collectors, job_working_directory, matchable):
def walk_over_file_collectors(extra_file_collectors, job_working_directory, matchable):
for extra_file_collector in extra_file_collectors:
assert extra_file_collector.discover_via == "pattern"
matches = []
directory = discover_target_directory(extra_file_collector, job_working_directory)
if not os.path.isdir(directory):
continue
for filename in os.listdir(directory):
path = os.path.join(directory, filename)
if not os.path.isfile(path):
continue
for match in walk_over_extra_files(extra_file_collector.directory, extra_file_collector, job_working_directory, matchable):
yield match, extra_file_collector
def walk_over_extra_files(target_dir, extra_file_collector, job_working_directory, matchable):
"""
Walks through all files in a given directory, and returns all files that
match the given collector's match criteria. If the collector has the
recurse flag enabled, will also recursively descend into child folders.
"""
matches = []
directory = discover_target_directory(target_dir, job_working_directory)
for filename in os.listdir(directory):
path = os.path.join(directory, filename)
if os.path.isdir(path) and extra_file_collector.recurse:
# The current directory is already validated, so use that as the next job_working_directory when recursing
for match in walk_over_extra_files(filename, extra_file_collector, directory, matchable):
yield match
else:
match = extra_file_collector.match(matchable, filename, path=path)
if match:
matches.append(match)
for match in extra_file_collector.sort(matches):
yield match, extra_file_collector
for match in extra_file_collector.sort(matches):
yield match
def dataset_collector(dataset_collection_description):
@@ -551,6 +562,7 @@ class DatasetCollector(object):
self.default_visible = dataset_collection_description.default_visible
self.directory = dataset_collection_description.directory
self.assign_primary_output = dataset_collection_description.assign_primary_output
self.recurse = dataset_collection_description.recurse
def _pattern_for_dataset(self, dataset_instance=None):
token_replacement = r'\d+'
lib/galaxy/tools/parser/interface.py
+17
View File
@@ -91,6 +91,23 @@ class ToolSource(object):
""" Return environment variable templates to expose.
"""
def parse_home_target(self):
"""Should be "job_home", "shared_home", "job_tmp", "pwd", or None.
"""
return "pwd"
def parse_tmp_target(self):
"""Should be "pwd", "shared_home", "job_tmp", "job_tmp_if_explicit", or None.
"""
return "job_tmp"
def parse_tmp_directory_vars(self):
"""Directories to override if a tmp_target is not None."""
return ["TMPDIR", "TMP", "TEMP"]
def parse_docker_env_pass_through(self):
return ["GALAXY_SLOTS", "HOME"] + self.parse_tmp_directory_vars()
@abstractmethod
def parse_interpreter(self):
""" Return string containing the interpreter to prepend to the command
lib/galaxy/tools/parser/output_collection_def.py
+2
View File
@@ -63,6 +63,7 @@ class DatasetCollectionDescription(object):
self.default_visible = asbool(kwargs.get("visible", None))
self.assign_primary_output = asbool(kwargs.get('assign_primary_output', False))
self.directory = kwargs.get("directory", None)
self.recurse = False
class ToolProvidedMetadataDatasetCollection(DatasetCollectionDescription):
@@ -77,6 +78,7 @@ class FilePatternDatasetCollectionDescription(DatasetCollectionDescription):
def __init__(self, **kwargs):
super(FilePatternDatasetCollectionDescription, self).__init__(**kwargs)
pattern = kwargs.get("pattern", "__default__")
self.recurse = asbool(kwargs.get("recurse", False))
if pattern in NAMED_PATTERNS:
pattern = NAMED_PATTERNS.get(pattern)
self.pattern = pattern
lib/galaxy/tools/parser/xml.py
+21
View File
@@ -126,6 +126,27 @@ class XmlToolSource(ToolSource):
)
return environment_variables
def parse_home_target(self):
target = "job_home" if self.parse_profile() >= "18.01" else "shared_home"
command_el = self._command_el
command_legacy = (command_el is not None) and command_el.get("use_shared_home", None)
if command_legacy is not None:
target = "shared_home" if string_as_bool(command_legacy) else "job_home"
return target
def parse_tmp_target(self):
# Default to not touching TMPDIR et. al. but if job_tmp is set
# in job_conf then do. This is a very conservative approach that shouldn't
# break or modify any configurations by default.
return "job_tmp_if_explicit"
def parse_docker_env_pass_through(self):
if self.parse_profile() < "18.01":
return ["GALAXY_SLOTS"]
else:
# Pass home, etc...
return super(XmlToolSource, self).parse_docker_env_pass_through()
def parse_interpreter(self):
interpreter = None
command_el = self._command_el
lib/galaxy/tools/toolbox/filters/__init__.py
+1 -1
View File
@@ -73,7 +73,7 @@ class FilterFactory(object):
else:
# No module found, just load a function from this file or
# one that has be explicitly imported.
function = getattr(globals(), filter_name.strip())
function = globals()[filter_name.strip()]
return function
def _import_filter(self, module_name, function_name):
lib/galaxy/tools/toolbox/watcher.py
+4
View File
@@ -252,6 +252,10 @@ class LocFileEventHandler(FileSystemEventHandler):
if self.loc_watcher.path_hash.get(path) == cur_hash:
return
else:
time.sleep(0.5)
if cur_hash != md5_hash_file(path):
# We're still modifying the file, it'll be picked up later
return
self.loc_watcher.path_hash[path] = cur_hash
self.loc_watcher.tool_data_tables.reload_tables(path=path)
lib/galaxy/tools/xsd/galaxy.xsd
+10
View File
@@ -2706,6 +2706,11 @@ deprecated and using the ``$__tool_directory__`` variable is superior.
<xs:documentation>Only used if ``detect_errors="exit_code", tells Galaxy the specified exit code indicates an out of memory error. Galaxy instances may be configured to retry such jobs on resources with more memory.</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="use_shared_home" type="xs:string">
<xs:annotation>
<xs:documentation>When running a job for this tool, do not isolate its $HOME directory within the job's directory - use either the shared_home_dir setting in Galaxy or the default $HOME specified in the job's default environment.</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="interpreter" type="xs:string" gxdocs:deprecated="true">
<xs:annotation>
<xs:documentation xml:lang="en">This attribute defines the programming language in which the tool's executable file is written. Any language can be used (tools can be written in Python, C, Perl, Java, etc.). The executable file must be in the same directory of the XML file. If instead this attribute is not specified, the tag content should be a Bash command calling executable(s) available in the $PATH. </xs:documentation>
@@ -3935,6 +3940,11 @@ More information can be found on Planemo's documentation for
<xs:documentation xml:lang="en">Directory (relative to working directory) to search for files.</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="recurse" type="xs:boolean" use="optional">
<xs:annotation>
<xs:documentation xml:lang="en">Indicates that the specified directory should be searched recursively for matching files.</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="format" type="xs:string" use="optional">
<xs:annotation>
<xs:documentation xml:lang="en">Format (or datatype) of discovered datasets (an alias with ``ext``).</xs:documentation>
lib/galaxy/util/path/__init__.py
+3 -1
View File
@@ -293,7 +293,9 @@ def __splitext_ignore(path, ignore=None):
ignore = map(__ext_strip_sep, __listify(ignore))
root, ext = __splitext_no_sep(path)
if ext in ignore:
root, ext = __splitext_no_sep(path)
new_path = path[0:(-len(ext) - 1)]
root, ext = __splitext_no_sep(new_path)
return (root, ext)
lib/galaxy/web/stack/__init__.py
+38 -11
View File
@@ -3,6 +3,7 @@
from __future__ import absolute_import
import inspect
import json
import logging
import os
@@ -14,10 +15,12 @@ try:
except ImportError:
uwsgi = None
import yaml
from six import string_types
from galaxy.util.bunch import Bunch
from galaxy.util.facts import get_facts
from galaxy.util.path import has_ext
from galaxy.util.properties import nice_config_parser
from .message import ApplicationStackMessage, ApplicationStackMessageDispatcher
from .transport import ApplicationStackTransport, UWSGIFarmMessageTransport
@@ -174,6 +177,25 @@ class UWSGIApplicationStack(MessageApplicationStack):
postfork_functions = []
@staticmethod
def _get_config_file(confs, loader, section):
"""uWSGI allows config merging, in which case the corresponding config file option will be a list.
"""
conf = None
if isinstance(confs, list):
gconfs = filter(lambda x: os.path.exists(x) and section in loader(open(x)), confs)
if len(gconfs) == 1:
conf = gconfs[0]
elif len(gconfs) == 0:
log.warning('Could not locate a config file containing a Galaxy config from: %s',
', '.join(confs))
else:
log.warning('Multiple config files contain Galaxy configs, merging is not supported: %s',
', '.join(gconfs))
else:
conf = confs
return conf
@classmethod
def get_app_kwds(cls, config_section, app_name=None):
kwds = {
@@ -181,21 +203,26 @@ class UWSGIApplicationStack(MessageApplicationStack):
'config_section': config_section,
}
uwsgi_opt = uwsgi.opt
# check for --yaml or --json uWSGI config options first
config_file = uwsgi_opt.get("yaml") or uwsgi_opt.get("json")
# --ini and --ini-paste don't behave the same way, but this method will only be called by mules if the main
# application was loaded with --ini-paste, so we can make some assumptions, most notably, uWSGI does not have
# any way to set the app name when loading with paste.deploy:loadapp(), so hardcoding the alternate section
# name to `app:main` is fine.
if config_file is None and uwsgi_opt.get("ini") or uwsgi_opt.get("ini-paste"):
config_file = uwsgi_opt.get("ini") or uwsgi_opt.get("ini-paste")
parser = nice_config_parser(config_file)
if not parser.has_section(config_section) and parser.has_section('app:main'):
kwds['config_section'] = 'app:main'
config_file = None
# check for --set galaxy_config_file=<path>, this overrides whatever config file uWSGI was loaded with (which
# may not actually include a Galaxy config)
if uwsgi_opt.get("galaxy_config_file"):
config_file = uwsgi_opt.get("galaxy_config_file")
# check for --yaml or --json uWSGI config options next
if config_file is None:
config_file = (UWSGIApplicationStack._get_config_file(uwsgi_opt.get("yaml"), yaml.safe_load, config_section)
or UWSGIApplicationStack._get_config_file(uwsgi_opt.get("json"), json.load, config_section))
# --ini and --ini-paste don't behave the same way, but this method will only be called by mules if the main
# application was loaded with --ini-paste, so we can make some assumptions, most notably, uWSGI does not have
# any way to set the app name when loading with paste.deploy:loadapp(), so hardcoding the alternate section
# name to `app:main` is fine.
has_ini_config = config_file is None and uwsgi_opt.get("ini") or uwsgi_opt.get("ini-paste")
has_ini_config = has_ini_config or (config_file and has_ext(config_file, "ini", aliases=True, ignore="sample"))
if has_ini_config:
config_file = config_file or uwsgi_opt.get("ini") or uwsgi_opt.get("ini-paste")
parser = nice_config_parser(config_file)
if not parser.has_section(config_section) and parser.has_section('app:main'):
kwds['config_section'] = 'app:main'
kwds['config_file'] = config_file
return kwds
lib/galaxy/webapps/config_manage.py
+170 -30
View File
@@ -50,14 +50,84 @@ App = namedtuple(
UWSGI_OPTIONS = OrderedDict([
('http', {
'desc': """The address and port on which to listen. By default, only listen to localhost ($app_name will not be accessible over the network). Use '0.0.0.0' to listen on all available network interfaces.""",
'desc': """The address and port on which to listen. By default, only listen to localhost ($app_name will not be accessible over the network). Use ':$default_port' to listen on all available network interfaces.""",
'default': '127.0.0.1:$default_port',
'type': 'str',
}),
('threads', {
'default': 8,
('processes', {
'desc': """Number of web server (worker) processes to fork after the application has loaded.""",
'default': '1',
'type': 'int',
}),
('threads', {
'desc': """Number of threads for each web server process.""",
'default': 4,
'type': 'int',
}),
('offload-threads', {
'desc': """Number of threads for serving static content and handling internal routing requests.""",
'default': '2',
'type': 'int',
}),
('static-map.1', {
'key': 'static-map',
'desc': """Mapping to serve style content.""",
'default': '/static/style=static/style/blue',
'type': 'str',
}),
('static-map.2', {
'key': 'static-map',
'desc': """Mapping to serve the remainder of the static content.""",
'default': '/static=static',
'type': 'str',
}),
('master', {
'desc': """Enable the master process manager. Disabled by default for maximum compatibility with CTRL+C, but should be enabled for use with --daemon and/or production deployments.""",
'default': 'false',
'type': 'str',
}),
('virtualenv', {
'desc': """Path to the application's Python virtual environment.""",
'default': '.venv',
'type': 'str',
}),
('pythonpath', {
'desc': """Path to the application's Python library.""",
'default': 'lib',
'type': 'str',
}),
('module', {
'desc': """The entry point which returns the web application (e.g. Galaxy, Reports, etc.) that you are loading.""",
'default': '$uwsgi_module',
'type': 'str',
}),
('die-on-term', {
'desc': """Cause uWSGI to respect the traditional behavior of dying on SIGTERM (its default is to brutally reload workers)""",
'default': 'true',
'type': 'str',
}),
('hook-master-start.1', {
'key': 'hook-master-start',
'desc': """Cause uWSGI to gracefully reload workers and mules upon receipt of SIGINT (its default is to brutally kill workers)""",
'default': 'unix_signal:2 gracefully_kill_them_all',
'type': 'str',
}),
('hook-master-start.2', {
'key': 'hook-master-start',
'desc': """Cause uWSGI to gracefully reload workers and mules upon receipt of SIGTERM (its default is to brutally kill workers)""",
'default': 'unix_signal:15 gracefully_kill_them_all',
'type': 'str',
}),
('py-call-osafterfork', {
'desc': """Feature necessary for proper mule signal handling""",
'default': 'true',
'type': 'str',
}),
('enable-threads', {
'desc': """Ensure application threads will run if `threads` is unset.""",
'default': 'true',
'type': 'str',
}),
# ('route-uri', {
# 'default': '^/proxy/ goto:proxy'
# }),
@@ -73,16 +143,9 @@ UWSGI_OPTIONS = OrderedDict([
# ('route-run', {
# 'default': "['log:Proxy ${HTTP_HOST} to ${TARGET_HOST}', 'httpdumb:${TARGET_HOST}']",
# }),
('http-raw-body', {
'default': 'True'
}),
('offload-threads', {
'default': '8',
}),
('module', {
'default': '$uwsgi_module',
'type': 'str',
})
# ('http-raw-body', {
# 'default': 'True'
# }),
])
DROP_OPTION_VALUE = object()
@@ -97,6 +160,12 @@ class _OptionAction(object):
pass
class _DeprecatedAction(_OptionAction):
def lint(self, args, app_desc, key, value):
print("Option [%s] has been deprecated, this will likely be dropped in future releases of Galaxy." % key)
class _DeprecatedAndDroppedAction(_OptionAction):
def converted(self, args, app_desc, key, value):
@@ -119,7 +188,7 @@ class _PasteAppFactoryAction(_OptionAction):
print("Problem - unknown paste app factory encountered [%s]" % value)
class _ProductionNotReady(_OptionAction):
class _ProductionUnsafe(_OptionAction):
def __init__(self, unsafe_value):
self.unsafe_value = unsafe_value
@@ -131,6 +200,14 @@ class _ProductionNotReady(_OptionAction):
print(message)
class _ProductionPerformance(_OptionAction):
def lint(self, args, app_desc, key, value):
template = "Problem - option [%s] should not be set to [%s] in production environments - it may cause performance issues or instability."
message = template % (key, value)
print(message)
class _HandleFilterWithAction(_OptionAction):
def converted(self, args, app_desc, key, value):
@@ -138,6 +215,20 @@ class _HandleFilterWithAction(_OptionAction):
return DROP_OPTION_VALUE
class _RenameAction(_OptionAction):
def __init__(self, new_name):
self.new_name = new_name
def converted(self, args, app_desc, key, value):
return (self.new_name, value)
def lint(self, args, app_desc, key, value):
template = "Problem - option [%s] has been renamed (possibly with slightly different behavior) to [%s]."
message = template % (key, self.new_name)
print(message)
OPTION_ACTIONS = {
'use_beaker_session': _DeprecatedAndDroppedAction(),
'session_type': _DeprecatedAndDroppedAction(),
@@ -146,12 +237,33 @@ OPTION_ACTIONS = {
'session_secret': _DeprecatedAndDroppedAction(),
'paste.app_factory': _PasteAppFactoryAction(),
'filter-with': _HandleFilterWithAction(),
'debug': _ProductionNotReady(True),
'serve_xss_vulnerable_mimetypes': _ProductionNotReady(True),
'use_printdebug': _ProductionNotReady(True),
'use_interactive': _ProductionNotReady(True),
'id_secret': _ProductionNotReady('USING THE DEFAULT IS NOT SECURE!'),
'master_api_key': _ProductionNotReady('changethis'),
'debug': _ProductionUnsafe(True),
'serve_xss_vulnerable_mimetypes': _ProductionUnsafe(True),
'use_printdebug': _ProductionUnsafe(True),
'use_interactive': _ProductionUnsafe(True),
'id_secret': _ProductionUnsafe('USING THE DEFAULT IS NOT SECURE!'),
'master_api_key': _ProductionUnsafe('changethis'),
'external_service_type_config_file': _DeprecatedAndDroppedAction(),
'external_service_type_path': _DeprecatedAndDroppedAction(),
'enable_sequencer_communication': _DeprecatedAndDroppedAction(),
'run_workflow_toolform_upgrade': _DeprecatedAndDroppedAction(),
# Next 4 were from library search which is no longer available.
'enable_lucene_library_search': _DeprecatedAndDroppedAction(),
'fulltext_max_size': _DeprecatedAndDroppedAction(),
'fulltext_noindex_filetypes': _DeprecatedAndDroppedAction(),
'fulltext_url': _DeprecatedAndDroppedAction(),
'enable_legacy_sample_tracking_api': _DeprecatedAction(),
'enable_new_user_preferences': _DeprecatedAndDroppedAction(),
'force_beta_workflow_scheduled_for_collections': _DeprecatedAction(),
'force_beta_workflow_scheduled_min_steps': _DeprecatedAction(),
'history_local_serial_workflow_scheduling': _ProductionPerformance(),
'allow_library_path_paste': _RenameAction("allow_path_paste"),
'trust_ipython_notebook_conversion': _RenameAction("trust_jupyter_notebook_conversion"),
'enable_beta_tool_command_isolation': _DeprecatedAndDroppedAction(),
'single_user': _ProductionUnsafe(True),
'tool_submission_burst_threads': _ProductionPerformance(),
'tool_submission_burst_at': _ProductionPerformance(),
'toolform_upgrade': _DeprecatedAndDroppedAction(),
}
@@ -214,6 +326,7 @@ class AppSchema(Schema):
self.raw_schema = config_all
app_schema = config_all["mapping"][app_name]
super(AppSchema, self).__init__(app_schema["mapping"])
self.description = config_all.get("desc", None)
def get_app_option(self, name):
try:
@@ -283,6 +396,10 @@ def _to_rst(args, app_desc, heading_level="~"):
schema = app_desc.schema
for key, value in schema.app_schema.items():
default = None if "default" not in value else value["default"]
if default is True:
default = "true"
elif default is False:
default = "false"
option = schema.get_app_option(key)
option_value = OptionValue(key, default, option)
_write_option_rst(args, rst, key, heading_level, option_value)
@@ -300,7 +417,11 @@ def _write_option_rst(args, rst, key, heading_level, option_value):
rst.write("\n")
type = option.get("type", None)
default = option.get("default", "*null*")
rst.write(":Default: %s\n" % default)
if default is True:
default = "true"
elif default is False:
default = "false"
rst.write(":Default: ``%s``\n" % default)
if type:
rst.write(":Type: %s\n" % type)
rst.write("\n\n")
@@ -498,7 +619,12 @@ def _run_conversion(args, app_desc):
if key in OPTION_ACTIONS:
option_action = OPTION_ACTIONS.get(key)
value = option_action.converted(args, app_desc, key, value)
new_value = option_action.converted(args, app_desc, key, value)
if new_value:
if isinstance(new_value, tuple):
key, value = new_value
else:
value = new_value
if value is DROP_OPTION_VALUE:
continue
@@ -511,7 +637,7 @@ def _run_conversion(args, app_desc):
app_dict[key] = option_value
f = StringIO()
_write_section(args, f, "uwsgi", uwsgi_dict)
_write_section(args, f, "uwsgi", uwsgi_dict, uwsgi_hack=True)
_write_section(args, f, app_desc.app_name, app_dict)
destination = os.path.join(args.galaxy_root, app_desc.destination)
_replace_file(args, f, app_desc, ini_config, destination)
@@ -549,7 +675,12 @@ def _build_sample_yaml(args, app_desc):
'uwsgi_module': app_desc.uwsgi_module,
})
value[field] = new_field_value
_write_sample_section(args, f, 'uwsgi', Schema(options), as_comment=False)
description = getattr(schema, "description", None)
if description:
description = description.lstrip()
as_comment = "\n".join(["# %s" % l for l in description.split("\n")]) + "\n"
f.write(as_comment)
_write_sample_section(args, f, 'uwsgi', Schema(options), as_comment=False, uwsgi_hack=True)
_write_sample_section(args, f, app_desc.app_name, schema)
destination = os.path.join(args.galaxy_root, app_desc.sample_destination)
_write_to_file(args, f, destination)
@@ -579,26 +710,28 @@ def _order_load_path(path):
return raw_config
def _write_sample_section(args, f, section_header, schema, as_comment=True):
def _write_sample_section(args, f, section_header, schema, as_comment=True, uwsgi_hack=False):
_write_header(f, section_header)
for key, value in schema.app_schema.items():
default = None if "default" not in value else value["default"]
option = schema.get_app_option(key)
option_value = OptionValue(key, default, option)
_write_option(args, f, key, option_value, as_comment=as_comment)
# support uWSGI "dumb yaml parser" (unbit/uwsgi#863)
key = option.get('key', key)
_write_option(args, f, key, option_value, as_comment=as_comment, uwsgi_hack=uwsgi_hack)
def _write_section(args, f, section_header, section_dict):
def _write_section(args, f, section_header, section_dict, uwsgi_hack=False):
_write_header(f, section_header)
for key, option_value in section_dict.items():
_write_option(args, f, key, option_value)
_write_option(args, f, key, option_value, uwsgi_hack=uwsgi_hack)
def _write_header(f, section_header):
f.write("%s:\n\n" % section_header)
def _write_option(args, f, key, option_value, as_comment=False):
def _write_option(args, f, key, option_value, as_comment=False, uwsgi_hack=False):
option, value = _parse_option_value(option_value)
desc = option["desc"]
comment = ""
@@ -606,7 +739,10 @@ def _write_option(args, f, key, option_value, as_comment=False):
comment = "\n".join(YAML_COMMENT_WRAPPER.wrap(desc))
comment += "\n"
as_comment_str = "#" if as_comment else ""
key_val_str = yaml.dump({key: value}).lstrip("{").rstrip("\n}")
if uwsgi_hack:
key_val_str = "%s: %s" % (key, value)
else:
key_val_str = yaml.dump({key: value}, width=float("inf")).lstrip("{").rstrip("\n}")
lines = "%s%s%s" % (comment, as_comment_str, key_val_str)
lines_idented = "\n".join([(" %s" % l) for l in lines.split("\n")])
f.write("%s\n\n" % lines_idented)
@@ -616,6 +752,10 @@ def _parse_option_value(option_value):
if isinstance(option_value, OptionValue):
option = option_value.option
value = option_value.value
option = option_value.option
# Hack to get nicer YAML values during conversion
if option.get("type", "str") == "bool":
value = str(value).lower() == "true"
else:
value = option_value
option = OPTION_DEFAULTS
lib/galaxy/webapps/galaxy/api/configuration.py
+3
View File
@@ -87,6 +87,9 @@ class ConfigurationController(BaseAPIController):
@expose_api
@require_admin
def dynamic_tool_confs(self, trans):
# WARNING: If this method is ever changed so as not to require admin privileges, update the nginx proxy
# documentation, since this path is used as an authentication-by-proxy method for securing other paths on the
# server. A dedicated endpoint should probably be added to do that instead.
confs = self.app.toolbox.dynamic_confs(include_migrated_tool_conf=True)
return list(map(_tool_conf_to_dict, confs))
lib/galaxy/webapps/galaxy/api/histories.py
+3
View File
@@ -321,6 +321,9 @@ class HistoriesController(BaseAPIController, ExportsHistoryMixin, ImportsHistory
if archive_source:
archive_type = payload.get("archive_type", "url")
elif hasattr(archive_file, "file"):
# archive_file.file is a TemporaryFile and will be deleted once it is closed.
# We prevent this by setting `delete` to `False`.
archive_file.file.delete = False
archive_source = payload["archive_file"].file.name
archive_type = "file"
else:
lib/galaxy/webapps/galaxy/config_schema.yml
+2587
View File
File diff suppressed because it is too large Load Diff
lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py
+1 -1
View File
@@ -238,7 +238,7 @@ class AdminToolshed(AdminGalaxy):
message += 'Attempting to uninstall tool dependencies resulted in errors: %s' % errors
status = max(status, statuses.index('error'))
status = statuses[status]
remove_from_disk_check_box = CheckboxField('remove_from_disk', value=remove_from_disk_checked)
remove_from_disk_check_box = CheckboxField('remove_from_disk', value=True)
return trans.fill_template('/admin/tool_shed_repository/deactivate_or_uninstall_repository.mako',
repository=tool_shed_repositories,
remove_from_disk_check_box=remove_from_disk_check_box,
lib/galaxy/webapps/galaxy/uwsgi_schema.yml
Symlink
+1
View File
@@ -0,0 +1 @@
../uwsgi_schema.yml
lib/galaxy/webapps/reports/config.py
-7
View File
@@ -4,8 +4,6 @@ import os
import re
import sys
from six.moves import configparser
from galaxy.util import string_as_bool
log = logging.getLogger(__name__)
@@ -53,11 +51,6 @@ class Configuration(object):
self.cookie_path = kwargs.get("cookie_path", "/")
# Error logging with sentry
self.sentry_dsn = kwargs.get('sentry_dsn', None)
# Parse global_conf
global_conf = kwargs.get('global_conf', None)
global_conf_parser = configparser.ConfigParser()
if global_conf and "__file__" in global_conf:
global_conf_parser.read(global_conf['__file__'])
def get(self, key, default):
return self.config_dict.get(key, default)
lib/galaxy/webapps/tool_shed/config.py
+1 -1
View File
@@ -134,7 +134,7 @@ class Configuration(object):
global_conf = kwargs.get('global_conf', None)
global_conf_parser = configparser.ConfigParser()
self.global_conf_parser = global_conf_parser
if global_conf and "__file__" in global_conf:
if global_conf and "__file__" in global_conf and ".yml" not in global_conf["__file__"]:
global_conf_parser.read(global_conf['__file__'])
self.running_functional_tests = string_as_bool(kwargs.get('running_functional_tests', False))
self.citation_cache_type = kwargs.get("citation_cache_type", "file")
scripts/common_startup.sh
+9 -6
View File
@@ -29,7 +29,7 @@ for arg in "$@"; do
[ "$arg" = "--replace-pip" ] && REPLACE_PIP=1
[ "$arg" = "--stop-daemon" ] && FETCH_WHEELS=0
[ "$arg" = "--skip-samples" ] && COPY_SAMPLE_FILES=0
[ "$arg" = "--no-client-build" ] && SKIP_CLIENT_BUILD=1
[ "$arg" = "--skip-client-build" ] && SKIP_CLIENT_BUILD=1
done
SAMPLES="
@@ -66,14 +66,14 @@ done
# Check client build state.
if [ $SKIP_CLIENT_BUILD -eq 0 ]; then
gitbranch=$(git rev-parse --abbrev-ref HEAD)
if [ $gitbranch == "dev" ]; then
if [ "$gitbranch" = "dev" ]; then
# We're on dev. This branch (only, currently) doesn't have build
# artifacts. We should probabably swap to a list of releases?
# Compare hash.
if [ -f static/client_build_hash.txt ]; then
githash=$(git rev-parse HEAD)
statichash=$(cat static/client_build_hash.txt)
if [ $githash == $statichash ]; then
if [ "$githash" = "$statichash" ]; then
SKIP_CLIENT_BUILD=1
fi
fi
@@ -83,18 +83,21 @@ if [ $SKIP_CLIENT_BUILD -eq 0 ]; then
fi
if [ $SKIP_CLIENT_BUILD -eq 0 ]; then
echo "The Galaxy client build is out of date. Please run 'make client' or your choice of client build target (client-*)."
echo "If you're sure you'd like to skip this check, you can run galaxy with the --no-client-build flag, though this is not recommended as the client and server code will potentially be out of sync."
echo "If you're sure you'd like to skip this check, you can run galaxy with the --skip-client-build flag, though this is not recommended as the client and server code will potentially be out of sync."
echo "See ./client/README.md in the Galaxy repository for more information, including how to get help if you're having trouble."
exit 1
fi
fi
: ${GALAXY_CONFIG_FILE:=config/galaxy.ini}
: ${GALAXY_CONFIG_FILE:=config/galaxy.yml}
if [ ! -f "$GALAXY_CONFIG_FILE" ]; then
GALAXY_CONFIG_FILE=config/galaxy.ini
fi
if [ ! -f "$GALAXY_CONFIG_FILE" ]; then
GALAXY_CONFIG_FILE=universe_wsgi.ini
fi
if [ ! -f "$GALAXY_CONFIG_FILE" ]; then
GALAXY_CONFIG_FILE=config/galaxy.ini.sample
GALAXY_CONFIG_FILE=config/galaxy.yml.sample
fi
: ${GALAXY_VIRTUAL_ENV:=.venv}
scripts/common_startup_functions.sh
+8 -3
View File
@@ -6,7 +6,7 @@ parse_common_args() {
while :
do
case "$1" in
--skip-eggs|--skip-wheels|--skip-samples|--dev-wheels|--no-create-venv|--no-replace-pip|--replace-pip)
--skip-eggs|--skip-wheels|--skip-samples|--dev-wheels|--no-create-venv|--no-replace-pip|--replace-pip|--skip-client-build)
common_startup_args="$common_startup_args $1"
shift
;;
@@ -34,6 +34,7 @@ parse_common_args() {
paster_args="$paster_args $1"
fi
uwsgi_args="$uwsgi_args --reload $PID_FILE"
restart_arg_set=1
daemon_or_restart_arg_set=1
shift
;;
@@ -41,7 +42,7 @@ parse_common_args() {
paster_args="$paster_args --pid-file $PID_FILE --log-file $LOG_FILE --daemon"
# --daemonize2 waits until after the application has loaded
# to daemonize, thus it stops if any errors are found
uwsgi_args="$uwsgi_args --daemonize2 $LOG_FILE --safe-pidfile $PID_FILE"
uwsgi_args="--master --daemonize2 $LOG_FILE --pidfile2 $PID_FILE $uwsgi_args"
daemon_or_restart_arg_set=1
shift
;;
@@ -109,7 +110,11 @@ find_server() {
[ "$server_config" != "none" ] && arg_getter_args="-c $server_config"
[ -n "$server_app" ] && arg_getter_args="--app $server_app"
run_server="$UWSGI"
server_args="$(python ./scripts/get_uwsgi_args.py $arg_getter_args) $uwsgi_args"
server_args=
if [ -z "$stop_daemon_arg_set" -a -z "$restart_arg_set" ]; then
server_args="$(python ./scripts/get_uwsgi_args.py $arg_getter_args)"
fi
server_args="$server_args $uwsgi_args"
else
run_server="python"
server_args="./scripts/paster.py serve $server_config $paster_args"
scripts/common_startup_variables.sh
Executable
+38
View File
@@ -0,0 +1,38 @@
#!/usr/bin/env bash
# This file is sourced in the maintenance.sh script and could be sourced in ./run.sh or ./run_reports.sh
SCRIPTLOCATION="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# If there is a file that defines a shell environment specific to this
# instance of Galaxy, source the file.
if [ -z $GALAXY_LOCAL_ENV_FILE ];
then
GALAXY_LOCAL_ENV_FILE="$SCRIPTLOCATION/../config/local_env.sh"
fi
if [ -f $GALAXY_LOCAL_ENV_FILE ];
then
echo "Activating local env file: $GALAXY_LOCAL_ENV_FILE"
. $GALAXY_LOCAL_ENV_FILE
fi
# If there is a .venv/ directory, assume it contains a virtualenv that we
# should run this instance in.
GALAXY_VIRTUAL_ENV="${GALAXY_VIRTUAL_ENV:-$SCRIPTLOCATION/../.venv}"
if [ -d "$GALAXY_VIRTUAL_ENV" -a -z "$skip_venv" ];
then
[ -n "$PYTHONPATH" ] && { echo 'Unsetting $PYTHONPATH'; unset PYTHONPATH; }
echo "Activating virtualenv at $GALAXY_VIRTUAL_ENV"
. "$GALAXY_VIRTUAL_ENV/bin/activate"
fi
if [ -z "$GALAXY_CONFIG_FILE" ]; then
if [ -f "$SCRIPTLOCATION/../universe_wsgi.ini" ]; then
GALAXY_CONFIG_FILE=universe_wsgi.ini
elif [ -f "$SCRIPTLOCATION/../config/galaxy.ini" ]; then
GALAXY_CONFIG_FILE=config/galaxy.ini
else
GALAXY_CONFIG_FILE=config/galaxy.ini.sample
fi
export GALAXY_CONFIG_FILE
fi
echo "GALAXY_CONFIG_FILE: ${GALAXY_CONFIG_FILE}"
scripts/functional_tests.py
+2
View File
@@ -71,6 +71,8 @@ class FrameworkToolsGalaxyTestDriver(DefaultGalaxyTestDriver):
"""Galaxy-style nose TestDriver for testing framework Galaxy tools."""
framework_tool_and_types = True
conda_auto_init = True
conda_auto_install = True
class DataManagersGalaxyTestDriver(driver_util.GalaxyTestDriver):
scripts/galaxy-main
+1 -1
View File
@@ -52,7 +52,7 @@ GALAXY_ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(real_file), os.pa
GALAXY_LIB_DIR = os.path.join(GALAXY_ROOT_DIR, "lib")
DEFAULT_INI_APP = "main"
DEFAULT_CONFIG_SECTION = "galaxy"
DEFAULT_INIS = ["config/galaxy.ini", "universe_wsgi.ini", "config/galaxy.ini.sample"]
DEFAULT_INIS = ["config/galaxy.yml", "config/galaxy.ini", "universe_wsgi.ini", "config/galaxy.yml.sample"]
DEFAULT_PID = "galaxy.pid"
DEFAULT_VERBOSE = True
scripts/get_uwsgi_args.py
+5 -4
View File
@@ -20,9 +20,10 @@ ALIASES = {
'virtualenv': ('home', 'venv', 'pyhome'),
'pythonpath': ('python-path', 'pp'),
'http': ('httprouter', 'socket', 'uwsgi-socket', 'suwsgi-socket', 'ssl-socket'),
'module': ('mount',), # mount is not actually an alias for module, but we don't want to set module if mount is set
}
DEFAULT_ARGS = {
'_all_': ('virtualenv', 'pythonpath', 'master', 'threads', 'http', 'static-map', 'die-on-term', 'hook-master-start', 'enable-threads'),
'_all_': ('virtualenv', 'pythonpath', 'threads', 'http', 'static-map', 'die-on-term', 'hook-master-start', 'enable-threads'),
'galaxy': ('py-call-osafterfork',),
'reports': (),
'tool_shed': (),
@@ -77,7 +78,6 @@ def __add_config_file_arg(args, config_file, app):
__add_arg(args, ext, config_file)
if has_logging:
__add_arg(args, 'paste-logger', True)
__add_arg(args, 'module', 'galaxy.webapps.{app}.buildapp:uwsgi_app()'.format(app=app))
def _get_uwsgi_args(cliargs, kwargs):
@@ -85,11 +85,9 @@ def _get_uwsgi_args(cliargs, kwargs):
config_file = cliargs.config_file or kwargs.get('__file__')
uwsgi_kwargs = load_app_properties(config_file=config_file, config_section='uwsgi')
args = []
__add_config_file_arg(args, config_file, cliargs.app)
defaults = {
'virtualenv': os.environ.get('VIRTUAL_ENV', './.venv'),
'pythonpath': 'lib',
'master': True,
'threads': '4',
'http': 'localhost:{port}'.format(port=DEFAULT_PORTS[cliargs.app]),
'static-map': ('/static/style={here}/static/style/blue'.format(here=os.getcwd()),
@@ -100,6 +98,9 @@ def _get_uwsgi_args(cliargs, kwargs):
'unix_signal:15 gracefully_kill_them_all'),
'py-call-osafterfork': True,
}
__add_config_file_arg(args, config_file, cliargs.app)
if not __arg_set('module', uwsgi_kwargs):
__add_arg(args, 'module', 'galaxy.webapps.{app}.buildapp:uwsgi_app()'.format(app=cliargs.app))
for arg in DEFAULT_ARGS['_all_'] + DEFAULT_ARGS[cliargs.app]:
if not __arg_set(arg, uwsgi_kwargs):
__add_arg(args, arg, defaults[arg])
scripts/maintenance.sh
Executable
+91
View File
@@ -0,0 +1,91 @@
#!/usr/bin/env bash
# Configure this script in a cron job to run some common cleanup operations
# Set the Environment variable GALAXY_CONFIG_FILE to use a custom galaxy config file.
set -e
display_help(){
scriptname=`basename $0`
printf "./$scriptname [--help] [--no-dry-run] [--days 10]
Will run the galaxy cleanup scripts in the recommend order. By default a 'dry-run' is started. Specify --no-dry-run to do the actual cleanup.
--help Show this help
--dry-run|--no-dry-run Dry run(default), will only print out what the scripts would have done. Specify --no-dry-run to do the actual cleanup.
--days Number of days to use as a cut off; do not act on objects updated more recently than this
"
}
# number of days to use as a cut off; do not act on objects updated more recently than this
DAYS=10
DRYRUN=true
while :
do
case "$1" in
--no-dry-run)
DRYRUN=false
shift
;;
--dry-run ) # default
DRYRUN=true
shift
;;
--days )
DAYS="$2"
shift; shift
;;
-h )
display_help
exit 1
;;
--help )
display_help
exit 1
;;
"")
break
;;
*)
shift
;;
esac
done
SCRIPTLOCATION="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source ${SCRIPTLOCATION}/common_startup_variables.sh
if [ "$DRYRUN" = true ]; then
MODE="--info_only"
else
MODE="-r"
fi
MAINTENANCE_LOG="$SCRIPTLOCATION/../maintenance.log"
COMMANDS=(
"python $SCRIPTLOCATION/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --delete_userless_histories >> $MAINTENANCE_LOG"
"python $SCRIPTLOCATION/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --purge_histories >> $MAINTENANCE_LOG"
"python $SCRIPTLOCATION/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --purge_datasets >> $MAINTENANCE_LOG"
"python $SCRIPTLOCATION/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --purge_folders >> $MAINTENANCE_LOG"
"python $SCRIPTLOCATION/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --delete_datasets >> $MAINTENANCE_LOG"
"python $SCRIPTLOCATION/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --purge_datasets >> $MAINTENANCE_LOG")
printf "\nDry run: $DRYRUN\nDays: $DAYS\n\n"
echo "Will run following commands and output in $MAINTENANCE_LOG"
for (( i = 0; i < ${#COMMANDS[@]} ; i++ )); do
echo "${COMMANDS[$i]}"
done
if [ "$DRYRUN" = false ]; then
echo "python $SCRIPTLOCATION/set_user_disk_usage.py >> $MAINTENANCE_LOG"
fi
# Run the commands
cd $SCRIPTLOCATION/../
for (( i = 0; i < ${#COMMANDS[@]} ; i++ )); do
eval "${COMMANDS[$i]}"
done
if [ "$DRYRUN" = false ]; then
python $SCRIPTLOCATION/set_user_disk_usage.py >> $MAINTENANCE_LOG
fi
scripts/tool_shed/bootstrap_tool_shed/bootstrap_tool_shed.sh
+1 -1
View File
@@ -3,7 +3,7 @@
# Activate the virtualenv, if it exists.
[ -f ./.venv/bin/activate ] && . ./.venv/bin/activate
: ${TOOL_SHED_CONFIG_FILE:=config/tool_shed.ini.sample}
: ${TOOL_SHED_CONFIG_FILE:=config/tool_shed.yml.sample}
stop_err() {
echo $1
scripts/tool_shed/bootstrap_tool_shed/bootstrap_util.py
+1 -1
View File
@@ -124,7 +124,7 @@ def main(args):
parser = optparse.OptionParser()
parser.add_option('-c', '--config_file', dest='config', action='store', default='config/tool_shed.ini.sample')
parser.add_option('-c', '--config_file', dest='config', action='store', default='config/tool_shed.yml.sample')
parser.add_option('-e', '--execute', dest='method', action='store', default='check_db')
(args, options) = parser.parse_args()
templates/webapps/galaxy/history/display_structured.mako
+1 -1
View File
@@ -231,7 +231,7 @@ elif entity_name == "WorkflowInvocation":
<% id = trans.security.encode_id( hda.id ) %>
<%
folder = hda.copied_from_library_dataset_dataset_association.library_dataset.folder
folder_id = 'F' + trans.security.encode_id( folder.id )
folder_id = trans.security.encode_id( folder.id )
%>
<div class="copied-from copied-from-library">
<div class="header">
test-data/bam_native_from_sam.bam
BIN
View File
Binary file not shown.
test-data/sam_with_header.sam
+4 -4
View File
@@ -1,14 +1,14 @@
@SQ SN:ref LN:45
@SQ SN:ref2 LN:40
r003 16 ref 29 30 6H5M * 0 0 TAGGC *
r001 83 ref 37 30 9M = 7 -39 CAGCGCCAT *
x2 0 ref2 2 30 21M * 0 0 ggttttataaaacaaataatt ?????????????????????
r001 163 ref 7 30 8M4I4M1D3M = 37 39 TTAGATAAAGAGGATACTG * XX:B:S,12561,2,20,112
r002 0 ref 9 30 1S2I6M1P1I1P1I4M2I * 0 0 AAAAGATAAGGGATAAA *
r003 0 ref 9 30 5H6M * 0 0 AGCTAA *
r004 0 ref 16 30 6M14N1I5M * 0 0 ATAGCTCTCAGC *
r003 16 ref 29 30 6H5M * 0 0 TAGGC *
r001 83 ref 37 30 9M = 7 -39 CAGCGCCAT *
x1 0 ref2 1 30 20M * 0 0 aggttttataaaacaaataa ????????????????????
x2 0 ref2 2 30 21M * 0 0 ggttttataaaacaaataatt ?????????????????????
x3 0 ref2 6 30 9M4I13M * 0 0 ttataaaacAAATaattaagtctaca ??????????????????????????
x4 0 ref2 10 30 25M * 0 0 CaaaTaattaagtctacagagcaac ?????????????????????????
x5 0 ref2 12 30 24M * 0 0 aaTaattaagtctacagagcaact ????????????????????????
x1 0 ref2 1 30 20M * 0 0 aggttttataaaacaaataa ????????????????????
x6 0 ref2 14 30 23M * 0 0 Taattaagtctacagagcaacta ???????????????????????
test/base/driver_util.py
+10 -3
View File
@@ -129,8 +129,12 @@ def setup_galaxy_config(
update_integrated_tool_panel=False,
prefer_template_database=False,
log_format=None,
conda_auto_init=False,
conda_auto_install=False
):
"""Setup environment and build config for test Galaxy instance."""
# For certain docker operations this needs to be evaluated out - e.g. for cwltool.
tmpdir = os.path.realpath(tmpdir)
if not os.path.exists(tmpdir):
os.makedirs(tmpdir)
file_path = os.path.join(tmpdir, 'files')
@@ -188,7 +192,8 @@ def setup_galaxy_config(
api_allow_run_as='test@bx.psu.edu',
auto_configure_logging=logging_config_file is None,
check_migrate_tools=False,
conda_auto_init=False,
conda_auto_init=conda_auto_init,
conda_auto_install=conda_auto_install,
cleanup_job='onsuccess',
data_manager_config_file=data_manager_config_file,
enable_beta_tool_formats=True,
@@ -510,7 +515,7 @@ def build_galaxy_app(simple_kwargs):
"""
log.info("Galaxy database connection: %s", simple_kwargs["database_connection"])
simple_kwargs['global_conf'] = get_webapp_global_conf()
simple_kwargs['global_conf']['__file__'] = "config/galaxy.ini.sample"
simple_kwargs['global_conf']['__file__'] = "config/galaxy.yml.sample"
simple_kwargs = load_app_properties(
kwds=simple_kwargs
)
@@ -534,7 +539,7 @@ def build_shed_app(simple_kwargs):
"""
log.info("Tool shed database connection: %s", simple_kwargs["database_connection"])
# TODO: Simplify global_conf to match Galaxy above...
simple_kwargs['__file__'] = 'tool_shed_wsgi.ini.sample'
simple_kwargs['__file__'] = 'tool_shed_wsgi.yml.sample'
simple_kwargs['global_conf'] = get_webapp_global_conf()
app = ToolshedUniverseApplication(**simple_kwargs)
@@ -851,6 +856,8 @@ class GalaxyTestDriver(TestDriver):
datatypes_conf=datatypes_conf_override,
prefer_template_database=getattr(config_object, "prefer_template_database", False),
log_format=log_format,
conda_auto_init=getattr(config_object, "conda_auto_init", False),
conda_auto_install=getattr(config_object, "conda_auto_install", False),
)
galaxy_config = setup_galaxy_config(
galaxy_db_path,
test/base/integration_util.py
+11
View File
@@ -7,6 +7,7 @@ tessting configuration.
import os
from unittest import skip, TestCase
from galaxy.tools.deps.commands import which
from .api import UsesApiTestCaseMixin
from .driver_util import GalaxyTestDriver
@@ -21,6 +22,16 @@ def skip_if_jenkins(cls):
return cls
def skip_unless_executable(executable):
if which(executable):
return lambda func: func
return skip("PATH doesn't contain executable %s" % executable)
def skip_unless_docker():
return skip_unless_executable("docker")
class IntegrationTestCase(TestCase, UsesApiTestCaseMixin):
"""Unit test case with utilities for spinning up Galaxy."""
test/docker/base/run_test_wrapper.sh
+8
View File
@@ -83,8 +83,16 @@ else
GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION="$GALAXY_TEST_DBURI"
TOOL_SHED_CONFIG_OVERRIDE_DATABASE_CONNECTION="$TOOL_SHED_TEST_DBURI"
GALAXY_CONFIG_MASTER_API_KEY=${GALAXY_CONFIG_MASTER_API_KEY:-"testmasterapikey"}
# This is a path baked inside of Docker it seems, so we should support both ini and
# YAML for some time.
GALAXY_CONFIG_FILE=${GALAXY_CONFIG_FILE:-config/galaxy.ini.sample}
if [ ! -f "$GALAXY_CONFIG_FILE" ]; then
GALAXY_CONFIG_FILE=config/galaxy.yml.sample
fi
TOOL_SHED_CONFIG_FILE=${GALAXY_CONFIG_FILE:-config/tool_shed.ini.sample}
if [ ! -f "$TOOL_SHED_CONFIG_FILE" ]; then
TOOL_SHED_CONFIG_FILE=config/tool_shed.yml.sample
fi
GALAXY_CONFIG_CHECK_MIGRATE_TOOLS=false
GALAXY_CONFIG_JOB_CONFIG_FILE=${GALAXY_CONFIG_JOB_CONFIG_FILE:-config/job_conf.xml.sample}
GALAXY_CONFIG_FILE_PATH=${GALAXY_CONFIG_FILE_PATH:-/tmp/gx1}
test/functional/tools/job_environment_default.xml
+23
View File
@@ -0,0 +1,23 @@
<tool id="job_environment_default" name="job_environment_default" version="0.1.0" profile="18.01">
<requirements>
<container type="docker">busybox:ubuntu-14.04</container>
</requirements>
<command><![CDATA[
echo `id -u` > '$user_id';
echo `id -g` > '$group_id';
echo `pwd` > '$pwd';
echo "\$HOME" > '$home';
echo "\$TMP" > '$tmp';
]]></command>
<inputs>
</inputs>
<outputs>
<data name="user_id" format="txt" label="user_id" />
<data name="group_id" format="txt" label="group_id" />
<data name="pwd" format="txt" label="pwd" />
<data name="home" format="txt" label="home" />
<data name="tmp" format="txt" label="tmp" />
</outputs>
<help>
</help>
</tool>
test/functional/tools/job_environment_default_legacy.xml
+23
View File
@@ -0,0 +1,23 @@
<tool id="job_environment_default_legacy" name="job_environment_default_legacy" version="0.1.0">
<requirements>
<container type="docker">busybox:ubuntu-14.04</container>
</requirements>
<command><![CDATA[
echo `id -u` > '$user_id';
echo `id -g` > '$group_id';
echo `pwd` > '$pwd';
echo "\$HOME" > '$home';
echo "\$TMP" > '$tmp';
]]></command>
<inputs>
</inputs>
<outputs>
<data name="user_id" format="txt" label="user_id" />
<data name="group_id" format="txt" label="group_id" />
<data name="pwd" format="txt" label="pwd" />
<data name="home" format="txt" label="home" />
<data name="tmp" format="txt" label="tmp" />
</outputs>
<help>
</help>
</tool>
test/functional/tools/job_environment_explicit_shared_home.xml
+23
View File
@@ -0,0 +1,23 @@
<tool id="job_environment_explicit_shared_home" name="job_environment_explicit_shared_home" version="0.1.0" profile="18.01">
<requirements>
<container type="docker">busybox:ubuntu-14.04</container>
</requirements>
<command use_shared_home="true"><![CDATA[
echo `id -u` > '$user_id';
echo `id -g` > '$group_id';
echo `pwd` > '$pwd';
echo "\$HOME" > '$home';
echo "\$TMP" > '$tmp';
]]></command>
<inputs>
</inputs>
<outputs>
<data name="user_id" format="txt" label="user_id" />
<data name="group_id" format="txt" label="group_id" />
<data name="pwd" format="txt" label="pwd" />
<data name="home" format="txt" label="home" />
<data name="tmp" format="txt" label="tmp" />
</outputs>
<help>
</help>
</tool>
test/functional/tools/mulled_example_explicit.xml
+22
View File
@@ -0,0 +1,22 @@
<tool id="mulled_example_explicit" name="mulled_example_explicit" version="0.1.0">
<requirements>
<requirement type="package" version="0.7.15">bwa</requirement>
<container type="docker">quay.io/biocontainers/bwa:0.7.15--0</container>
</requirements>
<stdio>
<exit_code range="2:" />
</stdio>
<command><![CDATA[
bwa > $output_1 2>&1
]]></command>
<inputs>
</inputs>
<outputs>
<data name="output_1" />
</outputs>
<help><![CDATA[
TODO: Fill in help.
]]></help>
<tests>
</tests>
</tool>

Some files were not shown because too many files have changed in this diff Show More