diff --git a/client/galaxy/scripts/layout/masthead.js b/client/galaxy/scripts/layout/masthead.js
index 527ce3f8b46..d6b9aaad59e 100644
--- a/client/galaxy/scripts/layout/masthead.js
+++ b/client/galaxy/scripts/layout/masthead.js
@@ -1,7 +1,7 @@
-import Utils from "utils/utils";
import Menu from "layout/menu";
import Scratchbook from "layout/scratchbook";
import QuotaMeter from "mvc/user/user-quotameter";
+
/** Masthead **/
var View = Backbone.View.extend({
initialize: function(options) {
@@ -62,7 +62,9 @@ var View = Backbone.View.extend({
var text = "";
self.collection.each(model => {
var q = model.get("onbeforeunload") && model.get("onbeforeunload")();
- q && (text += `${q} `);
+ if (q) {
+ text += `${q} `;
+ }
});
if (text !== "") {
return text;
diff --git a/client/galaxy/scripts/layout/menu.js b/client/galaxy/scripts/layout/menu.js
index d848465994e..49edea8089f 100644
--- a/client/galaxy/scripts/layout/menu.js
+++ b/client/galaxy/scripts/layout/menu.js
@@ -105,15 +105,15 @@ var Collection = Backbone.Collection.extend({
//
// Webhooks
//
- Webhooks.add({
- url: "api/webhooks/masthead/all",
+ Webhooks.load({
+ type: "masthead",
callback: function(webhooks) {
$(document).ready(() => {
- $.each(webhooks.models, (index, model) => {
+ webhooks.each(model => {
var webhook = model.toJSON();
if (webhook.activate) {
var obj = {
- id: webhook.name,
+ id: webhook.id,
icon: webhook.config.icon,
url: webhook.config.url,
tooltip: webhook.config.tooltip,
diff --git a/client/galaxy/scripts/layout/page.js b/client/galaxy/scripts/layout/page.js
index 8cef58f32d2..a2ce69c1655 100644
--- a/client/galaxy/scripts/layout/page.js
+++ b/client/galaxy/scripts/layout/page.js
@@ -2,6 +2,7 @@ import Masthead from "layout/masthead";
import Panel from "layout/panel";
import Modal from "mvc/ui/ui-modal";
import Utils from "utils/utils";
+
var View = Backbone.View.extend({
el: "body",
className: "full-content",
@@ -61,11 +62,12 @@ var View = Backbone.View.extend({
this.render();
// start the router
- this.router &&
+ if (this.router) {
Backbone.history.start({
root: Galaxy.root,
pushState: true
});
+ }
},
render: function() {
diff --git a/client/galaxy/scripts/mvc/history/options-menu.js b/client/galaxy/scripts/mvc/history/options-menu.js
index e8f280ebf40..a330160c045 100644
--- a/client/galaxy/scripts/mvc/history/options-menu.js
+++ b/client/galaxy/scripts/mvc/history/options-menu.js
@@ -175,13 +175,13 @@ var menu = [
];
// Webhooks
-Webhooks.add({
- url: "api/webhooks/history-menu/all",
+Webhooks.load({
+ type: "history-menu",
async: false, // (hypothetically) slows down the performance
callback: function(webhooks) {
var webhooks_menu = [];
- $.each(webhooks.models, (index, model) => {
+ webhooks.each(model => {
var webhook = model.toJSON();
if (webhook.activate) {
webhooks_menu.push({
diff --git a/client/galaxy/scripts/mvc/tool/tool-form-base.js b/client/galaxy/scripts/mvc/tool/tool-form-base.js
index 65f8c197c37..87404c7da92 100644
--- a/client/galaxy/scripts/mvc/tool/tool-form-base.js
+++ b/client/galaxy/scripts/mvc/tool/tool-form-base.js
@@ -6,6 +6,7 @@ import Utils from "utils/utils";
import Deferred from "utils/deferred";
import Ui from "mvc/ui/ui-misc";
import FormBase from "mvc/form/form-view";
+import Webhooks from "mvc/webhooks";
import Citations from "components/Citations.vue";
import Vue from "vue";
export default FormBase.extend({
@@ -200,19 +201,23 @@ export default FormBase.extend({
}
// add tool menu webhooks
- $.getJSON("/api/webhooks/tool-menu/all", webhooks => {
- _.each(webhooks, webhook => {
- if (webhook.activate && webhook.config.function) {
- menu_button.addMenu({
- icon: webhook.config.icon,
- title: webhook.config.title,
- onclick: function() {
- var func = new Function("options", webhook.config.function);
- func(options);
- }
- });
- }
- });
+ Webhooks.load({
+ type: "tool-menu",
+ callback: function(webhooks) {
+ webhooks.each(model => {
+ var webhook = model.toJSON();
+ if (webhook.activate && webhook.config.function) {
+ menu_button.addMenu({
+ icon: webhook.config.icon,
+ title: webhook.config.title,
+ onclick: function() {
+ var func = new Function("options", webhook.config.function);
+ func(options);
+ }
+ });
+ }
+ });
+ }
});
return {
diff --git a/client/galaxy/scripts/mvc/tool/tool-form-composite.js b/client/galaxy/scripts/mvc/tool/tool-form-composite.js
index 1beea452ce9..00a5e78f845 100644
--- a/client/galaxy/scripts/mvc/tool/tool-form-composite.js
+++ b/client/galaxy/scripts/mvc/tool/tool-form-composite.js
@@ -598,7 +598,7 @@ var View = Backbone.View.extend({
if ($.isArray(response) && response.length > 0) {
self.$el.append($("
", { id: "webhook-view" }));
var WebhookApp = new Webhooks.WebhookView({
- urlRoot: `${Galaxy.root}api/webhooks/workflow`,
+ type: "workflow",
toolId: job_def.tool_id,
toolVersion: job_def.tool_version
});
diff --git a/client/galaxy/scripts/mvc/tool/tool-form.js b/client/galaxy/scripts/mvc/tool/tool-form.js
index f946fdd325a..5dec9427bff 100644
--- a/client/galaxy/scripts/mvc/tool/tool-form.js
+++ b/client/galaxy/scripts/mvc/tool/tool-form.js
@@ -227,7 +227,7 @@ var View = Backbone.View.extend({
if (response.jobs && response.jobs.length > 0) {
self.$el.append($("", { id: "webhook-view" }));
var WebhookApp = new Webhooks.WebhookView({
- urlRoot: `${Galaxy.root}api/webhooks/tool`,
+ type: "tool",
toolId: job_def.tool_id
});
}
diff --git a/client/galaxy/scripts/mvc/webhooks.js b/client/galaxy/scripts/mvc/webhooks.js
index 44c460fde3b..c1f2400fcf5 100644
--- a/client/galaxy/scripts/mvc/webhooks.js
+++ b/client/galaxy/scripts/mvc/webhooks.js
@@ -1,66 +1,86 @@
-/**
- Webhooks
-**/
+import Utils from "utils/utils";
-var WebhookModel = Backbone.Model.extend({
- defaults: {
- activate: false
+const Webhooks = Backbone.Collection.extend({
+ url: function() {
+ return `${Galaxy.root}api/webhooks`;
}
});
-var Webhooks = Backbone.Collection.extend({
- model: WebhookModel
-});
-
-var WebhookView = Backbone.View.extend({
+const WebhookView = Backbone.View.extend({
el: "#webhook-view",
initialize: function(options) {
- var me = this;
- var toolId = options.toolId || "";
- var toolVersion = options.toolVersion || "";
+ const toolId = options.toolId || "";
+ const toolVersion = options.toolVersion || "";
this.$el.attr("tool_id", toolId);
this.$el.attr("tool_version", toolVersion);
- this.model = new WebhookModel();
- this.model.urlRoot = options.urlRoot;
- this.model.fetch({
- success: function() {
- me.render();
+ const webhooks = new Webhooks();
+ webhooks.fetch({
+ success: data => {
+ if (options.type) {
+ data.reset(filterType(data, options.type));
+ }
+ if (data.length > 0) {
+ this.render(weightedRandomPick(data));
+ }
}
});
},
- render: function() {
- var webhook = this.model.toJSON();
-
- this.$el.html(``);
- if (webhook.styles)
- $("", { type: "text/css" })
- .text(webhook.styles)
- .appendTo("head");
- if (webhook.script)
- $("", { type: "text/javascript" })
- .text(webhook.script)
- .appendTo("head");
-
+ render: function(model) {
+ const webhook = model.toJSON();
+ this.$el.html(``);
+ Utils.appendScriptStyle(webhook);
return this;
}
});
-var add = options => {
- var webhooks = new Webhooks();
-
- webhooks.url = Galaxy.root + options.url;
+const load = options => {
+ const webhooks = new Webhooks();
webhooks.fetch({
- async: options.async ? options.async : true,
- success: options.callback
+ async: options.async !== undefined ? options.async : true,
+ success: data => {
+ if (options.type) {
+ data.reset(filterType(data, options.type));
+ }
+ options.callback(data);
+ }
});
};
+function filterType(data, type) {
+ return data.models.filter(item => {
+ let itype = item.get("type");
+ if (itype) {
+ return itype.indexOf(type) !== -1;
+ } else {
+ return false;
+ }
+ });
+}
+
+function weightedRandomPick(data) {
+ const weights = data.pluck("weight");
+ const sum = weights.reduce((a, b) => a + b);
+
+ const normalizedWeightsMap = new Map();
+ weights.forEach((weight, index) => {
+ normalizedWeightsMap.set(index, parseFloat((weight / sum).toFixed(2)));
+ });
+
+ const table = [];
+ for (const [index, weight] of normalizedWeightsMap) {
+ for (let i = 0; i < weight * 100; i++) {
+ table.push(index);
+ }
+ }
+
+ return data.at(table[Math.floor(Math.random() * table.length)]);
+}
+
export default {
- Webhooks: Webhooks,
WebhookView: WebhookView,
- add: add
+ load: load
};
diff --git a/client/galaxy/scripts/onload.js b/client/galaxy/scripts/onload.js
index 7af85c6fe35..232fc87c0ab 100644
--- a/client/galaxy/scripts/onload.js
+++ b/client/galaxy/scripts/onload.js
@@ -25,6 +25,8 @@ window.make_popup_menus = Popupmenu.make_popup_menus;
import init_tag_click_function from "ui/autocom_tagging";
window.init_tag_click_function = init_tag_click_function;
import Tours from "mvc/tours";
+import Webhooks from "mvc/webhooks";
+import Utils from "utils/utils";
// console.debug( 'galaxy globals loaded' );
// ============================================================================
@@ -176,23 +178,19 @@ $(document).ready(() => {
Tours.activeGalaxyTourRunner();
function onloadWebhooks() {
- // Wait until Galaxy.config is loaded.
- if (Galaxy.config) {
- if (Galaxy.config.enable_webhooks) {
- // Load all webhooks with the type 'onload'
- $.getJSON(`${Galaxy.root}api/webhooks/onload/all`, webhooks => {
- _.each(webhooks, webhook => {
+ if (Galaxy.root !== undefined) {
+ // Load all webhooks with the type 'onload'
+ Webhooks.load({
+ type: "onload",
+ callback: function(webhooks) {
+ webhooks.each(model => {
+ var webhook = model.toJSON();
if (webhook.activate && webhook.script) {
- $("", { type: "text/javascript" })
- .text(webhook.script)
- .appendTo("head");
- $("", { type: "text/css" })
- .text(webhook.styles)
- .appendTo("head");
+ Utils.appendScriptStyle(webhook);
}
});
- });
- }
+ }
+ });
} else {
setTimeout(onloadWebhooks, 100);
}
diff --git a/client/galaxy/scripts/qunit/test-app.js b/client/galaxy/scripts/qunit/test-app.js
index c1f6991aba7..155eb38627c 100644
--- a/client/galaxy/scripts/qunit/test-app.js
+++ b/client/galaxy/scripts/qunit/test-app.js
@@ -1,4 +1,6 @@
/** Creates a generic/global Galaxy environment, loads shared libraries and a fake server */
+/* global define */
+
define(
[
"jquery",
diff --git a/client/galaxy/scripts/qunit/tests/page_tests.js b/client/galaxy/scripts/qunit/tests/page_tests.js
index d7b4951e7e4..5d70e19653d 100644
--- a/client/galaxy/scripts/qunit/tests/page_tests.js
+++ b/client/galaxy/scripts/qunit/tests/page_tests.js
@@ -1,8 +1,7 @@
-/* global define */
+/* global QUnit */
import testApp from "qunit/test-app";
import Page from "layout/page";
-import Panel from "layout/panel";
QUnit.module("Page test", {
beforeEach: function() {
diff --git a/config/plugins/webhooks/demo/search/config/searchover.yaml b/config/plugins/webhooks/demo/search/config.yml
similarity index 82%
rename from config/plugins/webhooks/demo/search/config/searchover.yaml
rename to config/plugins/webhooks/demo/search/config.yml
index f97f5797fc8..92dccea779e 100644
--- a/config/plugins/webhooks/demo/search/config/searchover.yaml
+++ b/config/plugins/webhooks/demo/search/config.yml
@@ -1,4 +1,4 @@
-name: searchover
+id: searchover
type:
- masthead
activate: true
diff --git a/config/plugins/webhooks/demo/search/static/script.js b/config/plugins/webhooks/demo/search/script.js
similarity index 99%
rename from config/plugins/webhooks/demo/search/static/script.js
rename to config/plugins/webhooks/demo/search/script.js
index 8b1a5b5e0b4..339d4b57f43 100644
--- a/config/plugins/webhooks/demo/search/static/script.js
+++ b/config/plugins/webhooks/demo/search/script.js
@@ -38,7 +38,7 @@ $(document).ready(function() {
e.stopPropagation();
if ( $( '.search-screen-overlay' ).is( ':visible' ) ){
self.removeOverlay();
- }
+ }
else {
self.clearSearchResults();
self.showOverlay();
diff --git a/config/plugins/webhooks/demo/search/static/styles.css b/config/plugins/webhooks/demo/search/styles.css
similarity index 100%
rename from config/plugins/webhooks/demo/search/static/styles.css
rename to config/plugins/webhooks/demo/search/styles.css
diff --git a/config/plugins/webhooks/demo/tool_list/helper/__init__.py b/config/plugins/webhooks/demo/tool_list/__init__.py
similarity index 100%
rename from config/plugins/webhooks/demo/tool_list/helper/__init__.py
rename to config/plugins/webhooks/demo/tool_list/__init__.py
diff --git a/config/plugins/webhooks/demo/tool_list/config/tool_list.yml b/config/plugins/webhooks/demo/tool_list/config.yml
similarity index 94%
rename from config/plugins/webhooks/demo/tool_list/config/tool_list.yml
rename to config/plugins/webhooks/demo/tool_list/config.yml
index cc3577068b4..d57c1832e6a 100644
--- a/config/plugins/webhooks/demo/tool_list/config/tool_list.yml
+++ b/config/plugins/webhooks/demo/tool_list/config.yml
@@ -1,4 +1,4 @@
-name: tool_list
+id: tool_list
type:
- masthead
activate: true
@@ -13,7 +13,7 @@ function: >
'be patient, this might take a moment.');
}, 1);
- $.getJSON(Galaxy.root + "api/webhooks/tool_list/get_data", function(data) {
+ $.getJSON(Galaxy.root + "api/webhooks/tool_list/data", function(data) {
var popup = window.open('tool_list.html');
var html = '' +
'Create a Docker flavour of this instance:
' +
diff --git a/config/plugins/webhooks/demo/tour_generator/helper/__init__.py b/config/plugins/webhooks/demo/tour_generator/__init__.py
similarity index 100%
rename from config/plugins/webhooks/demo/tour_generator/helper/__init__.py
rename to config/plugins/webhooks/demo/tour_generator/__init__.py
diff --git a/config/plugins/webhooks/demo/tour_generator/config/tour_generator.yml b/config/plugins/webhooks/demo/tour_generator/config.yml
similarity index 95%
rename from config/plugins/webhooks/demo/tour_generator/config/tour_generator.yml
rename to config/plugins/webhooks/demo/tour_generator/config.yml
index b510bdb0088..309f1791afa 100644
--- a/config/plugins/webhooks/demo/tour_generator/config/tour_generator.yml
+++ b/config/plugins/webhooks/demo/tour_generator/config.yml
@@ -1,4 +1,4 @@
-name: tour_generator
+id: tour_generator
type:
- onload
- tool-menu
diff --git a/config/plugins/webhooks/demo/tour_generator/static/script.js b/config/plugins/webhooks/demo/tour_generator/script.js
similarity index 97%
rename from config/plugins/webhooks/demo/tour_generator/static/script.js
rename to config/plugins/webhooks/demo/tour_generator/script.js
index 4885e37a1bf..b5a499353b1 100644
--- a/config/plugins/webhooks/demo/tour_generator/static/script.js
+++ b/config/plugins/webhooks/demo/tour_generator/script.js
@@ -10,7 +10,7 @@ $(document).ready(function() {
$('#execute').attr('tour_id', 'execute');
Toastr.info('Tour generation might take some time.');
- $.getJSON('/api/webhooks/tour_generator/get_data/', {
+ $.getJSON('/api/webhooks/tour_generator/data/', {
tool_id: me.toolId,
tool_version: me.toolVersion
}, function(obj) {
diff --git a/config/plugins/webhooks/demo/tour_generator/static/styles.css b/config/plugins/webhooks/demo/tour_generator/styles.css
similarity index 100%
rename from config/plugins/webhooks/demo/tour_generator/static/styles.css
rename to config/plugins/webhooks/demo/tour_generator/styles.css
diff --git a/doc/source/admin/framework_dependencies.rst b/doc/source/admin/framework_dependencies.rst
index d0d293a1539..15b9c88b386 100644
--- a/doc/source/admin/framework_dependencies.rst
+++ b/doc/source/admin/framework_dependencies.rst
@@ -3,55 +3,48 @@
Framework Dependencies
======================
-Galaxy is a large Python application with a long list of `Python module
-dependencies`_. As a result, the Galaxy developers have made significant effort
-to provide these dependencies in as simple a method as possible. Prior to the
-16.01 release, this was done by distributing dependencies in Python's old
-standard packaging format (Egg) in a non-standard way. As of the 16.01 release,
-Galaxy now distributes dependencies in Python's new standard packaging format
-(Wheel), albeit still in a non-standard way. Thankfully, the new distribution
-method is far more compatible with the standard package distribution tooling
-(i.e. `pip`_) than the old method.
+Galaxy is a large Python application with a long list of `Python module dependencies`_. As a result, the Galaxy
+developers have made significant effort to provide these dependencies in as simple a method as possible while remaining
+compatible with the `Python packaging best practices`_. Thus, Galaxy's runtime setup procedure makes use of virtualenv_
+for package environment isolation, pip_ for installation, and wheel_ to provide pre-built versions of dependencies.
+
+In addition to framework dependencies, as of Galaxy 18.01, the client (UI) is no longer provided in its built format
+**in the development (``dev``) branch of the source repository**. The built client is still provided in
+``release_YY.MM`` branches and the ``master`` branch.
.. _Python module dependencies: https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/dependencies/requirements.txt
-.. _pip: https://pip.pypa.io/
-.. _wheel: https://wheel.readthedocs.org/
+.. _Python packaging best practices: https://packaging.python.org
+.. _virtualenv: https://packaging.python.org/tutorials/installing-packages/#creating-virtual-environments
+.. _pip: https://packaging.python.org/tutorials/installing-packages/#use-pip-for-installing
+.. _wheel: https://packaging.python.org/tutorials/installing-packages/#source-distributions-vs-wheels
How it works
------------
Upon startup (with ``run.sh``), the startup scripts will:
-1. Create a Python `virtualenv`_ in the directory ``.venv``.
+1. Create a Python virtualenv_ in the directory ``.venv``.
-2. Unset the ``$PYTHONPATH`` environment variable (if set) as this can
- interfere with installing `Galaxy pip`_ and dependencies.
+2. Unset the ``$PYTHONPATH`` environment variable (if set) as this can interfere with installing dependencies.
-3. Replace that virtualenv's pip with `Galaxy pip`_.
+3. Download and install packages from the Galaxy project wheel server, wheels.galaxyproject.org_, as well as the `Python
+ Package Index`_ (aka PyPI) , using pip_.
-4. If applicable, create a ``binary-compatibility.cfg`` (see the `Galaxy pip
- and wheel`_ section for an explanation of this file).
+4. Start Galaxy using ``.venv/bin/python``.
-5. Download and install wheels from the Galaxy project wheel server,
- `wheels.galaxyproject.org`_, using pip.
-
-6. Start Galaxy using ``.venv/bin/python``.
-
-.. _virtualenv: https://virtualenv.readthedocs.org/
.. _wheels.galaxyproject.org: https://wheels.galaxyproject.org/
+.. _Python Package Index: https://pypi.org
Options
-------
A variety of options to ``run.sh`` are available to control the above behavior:
-- ``--skip-venv``: Do not create or use a virtualenv, and do not replace pip
- with Galaxy pip.
+- ``--skip-venv``: Do not create or use a virtualenv.
- ``--skip-wheels``: Do not install wheels.
-- ``--no-create-venv``: Do not create a virtualenv, but use one if it exists at
- ``.venv`` or if ``$VIRTUAL_ENV`` is set (this variable is set by virtualenv's
- ``activate``)
-- ``--replace-pip/--no-replace-pip``: Do/do not replace pip with Galaxy pip.
+- ``--no-create-venv``: Do not create a virtualenv, but use one if it exists at ``.venv`` or if ``$VIRTUAL_ENV`` is set
+ (this variable is set by virtualenv's ``activate``).
+- ``--replace-pip/--no-replace-pip``: Do/do not upgrade pip if necessary.
Managing dependencies manually
------------------------------
@@ -59,29 +52,23 @@ Managing dependencies manually
Create a virtualenv
^^^^^^^^^^^^^^^^^^^
-Using a `virtualenv`_ in ``.venv`` under the Galaxy source tree is not
-required. More complicated Galaxy setups may choose to use a virtualenv
-external to the Galaxy source tree, which can be done either by not using
-``run.sh`` directly (an example of this can be found under the `Supervisor`_
-section) or using the ``--no-create-venv`` option, explained in the `Options`_
-section. It is also possible to force Galaxy to start without a virtualenv at
-all, but you should not do this unless you know what you're doing.
+Using a `virtualenv`_ in ``.venv`` under the Galaxy source tree is not required. More complicated Galaxy setups may
+choose to use a virtualenv external to the Galaxy source tree, which can be done either by not using ``run.sh`` directly
+(an example of this can be found under the `Scaling and Load Balancing` documentation) or using the ``--no-create-venv``
+option, explained in the `Options` section. It is also possible to force Galaxy to start without a virtualenv at all,
+but you should not do this unless you know what you're doing.
-To manually create a virtualenv, you will first need to obtain virtualenv.
-There are a variety of ways to do this:
+To manually create a virtualenv, you will first need to obtain virtualenv. There are a variety of ways to do this:
- ``pip install virtualenv``
- ``brew install virtualenv``
-- Install your Linux distribution's virtualenv package from the system package
- manager (e.g. ``apt-get install python-virtualenv``).
-- Download the `virtualenv source from PyPI
- `_, untar, and run the
- ``virtualenv.py`` script contained within as ``python virtualenv.py
- /path/to/galaxy/virtualenv``
+- Install your Linux distribution's virtualenv package from the system package manager (e.g. ``apt-get install
+ python-virtualenv``).
+- Download the `virtualenv source from PyPI `_, untar, and run the
+ ``virtualenv.py`` script contained within as ``python virtualenv.py /path/to/galaxy/virtualenv``
-Once this is done, create a virtualenv. In our example, the virtualenv will
-live in ``/srv/galaxy/venv`` and the Galaxy source code has been cloned to
-``/srv/galaxy/server``.
+Once this is done, create a virtualenv. In our example, the virtualenv will live in ``/srv/galaxy/venv`` and the Galaxy
+source code has been cloned to ``/srv/galaxy/server``.
.. code-block:: console
@@ -91,550 +78,316 @@ live in ``/srv/galaxy/venv`` and the Galaxy source code has been cloned to
$ . /srv/galaxy/venv/bin/activate
(venv)$
+Next, in ``galaxy.yml``, set the ``virtualenv`` option in the ``uwsgi`` section to point to your new virtualenv:
+
+.. code-block:: yaml
+
+ ---
+ uwsgi:
+ #...
+ virtualenv: /srv/galaxy/venv
+
Install dependencies
^^^^^^^^^^^^^^^^^^^^
-Normally, ``run.sh`` calls `common_startup.sh`_, which creates the virtualenv,
-installs Galaxy pip, and installs dependencies. You can call this script
-yourself to set up Galaxy pip and the dependencies without creating a
-virtualenv using the ``--no-create-venv`` option:
+Normally, ``run.sh`` calls `common_startup.sh`_, which creates the virtualenv and installs dependencies. You can call
+this script yourself to set up Galaxy pip and the dependencies without creating a virtualenv using the
+``--no-create-venv`` option:
.. code-block:: console
(venv)$ PYTHONPATH= sh /srv/galaxy/server/scripts/common_startup.sh --no-create-venv
- Ignoring indexes: https://pypi.python.org/simple
- Collecting pip
- Downloading https://wheels.galaxyproject.org/packages/pip-8.0.0+gx1-py2.py3-none-any.whl (1.2MB)
- 100% |████████████████████████████████| 1.2MB 37.8MB/s
- Installing collected packages: pip
- Found existing installation: pip 7.1.2
- Uninstalling pip-7.1.2:
- Successfully uninstalled pip-7.1.2
- Successfully installed pip-8.0.0+gx1
- Collecting bx-python==0.7.3 (from -r requirements.txt (line 2))
- Downloading https://wheels.galaxyproject.org/packages/bx_python-0.7.3-cp27-cp27mu-linux_x86_64.whl (1.7MB)
- 100% |████████████████████████████████| 1.7MB 25.4MB/s
+ Requirement already satisfied: pip>=8.1 in /home/nate/.virtualenvs/test/lib/python2.7/site-packages
+ Collecting numpy==1.9.2 (from -r requirements.txt (line 4))
+ Downloading https://wheels.galaxyproject.org/packages/numpy-1.9.2-cp27-cp27mu-manylinux1_x86_64.whl (10.2MB)
+ 100% |████████████████████████████████| 10.2MB 21.7MB/s
+ Collecting bx-python==0.7.3 (from -r requirements.txt (line 5))
+ Downloading https://wheels.galaxyproject.org/packages/bx_python-0.7.3-cp27-cp27mu-manylinux1_x86_64.whl (2.1MB)
+ 100% |████████████████████████████████| 2.2MB 97.2MB/s
- ...
+ ...
- Collecting pysam==0.8.3+gx1 (from -r requirements.txt (line 69))
- Downloading https://wheels.galaxyproject.org/packages/pysam-0.8.3+gx1-cp27-cp27mu-linux_x86_64.whl (7.4MB)
- 100% |████████████████████████████████| 7.4MB 15.1MB/s
- Installing collected packages: bx-python, MarkupSafe, PyYAML, SQLAlchemy,
- mercurial, numpy, pycrypto, six, Paste, PasteDeploy, docutils, wchartype,
- repoze.lru, Routes, WebOb, WebHelpers, Mako, pytz, Babel, Beaker,
- Markdown, Cheetah, requests, requests-toolbelt, boto, bioblend, amqp,
- anyjson, kombu, pbr, sqlparse, decorator, Tempita, sqlalchemy-migrate,
- Parsley, nose, svgwrite, ecdsa, paramiko, Fabric, Whoosh, pysam
- Successfully installed Babel-2.0 Beaker-1.7.0 Cheetah-2.4.4 Fabric-1.10.2
- Mako-1.0.2 Markdown-2.6.3 MarkupSafe-0.23 Parsley-1.3 Paste-2.0.2
- PasteDeploy-1.5.2 PyYAML-3.11 Routes-2.2 SQLAlchemy-1.0.8 svgwrite-1.1.6
- Tempita-0.5.3.dev0 WebHelpers-1.3 WebOb-1.4.1 Whoosh-2.4.1+gx1 amqp-1.4.8
- anyjson-0.3.3 bioblend-0.6.1 boto-2.38.0 bx-python-0.7.3 decorator-4.0.2
- docutils-0.12 ecdsa-0.13 kombu-3.0.30 mercurial-3.4.2 nose-1.3.7
- numpy-1.9.2 paramiko-1.15.2 pbr-1.8.0 pycrypto-2.6.1 pysam-0.8.3+gx1
- pytz-2015.4 repoze.lru-0.6 requests-2.8.1 requests-toolbelt-0.4.0
- six-1.9.0 sqlalchemy-migrate-0.10.0 sqlparse-0.1.16 wchartype-0.1
+ Installing collected packages: numpy, bx-python, ...
+ Successfully installed numpy-1.9.2 bx-python-0.7.3 ...
-**Warning:** If your ``$PYTHONPATH`` is set, it may interfere with the
-dependency installation process (this will almost certainly be the case if you
-use `virtualenv-burrito`_). Without ``--no-create-venv`` the ``$PYTHONPATH``
-variable will be automatically unset, but we assume you know what you're doing
-and may want it left intact if you are using ``--no-create-venv``. If you
-encounter problems, try unsetting ``$PYTHONPATH`` as shown in the example
-above.
+.. warning::
+
+ If your ``$PYTHONPATH`` is set, it may interfere with the dependency installation process. Without
+ ``--no-create-venv`` the ``$PYTHONPATH`` variable will be automatically unset, but we assume you know what you're
+ doing and may want it left intact if you are using ``--no-create-venv``. If you encounter problems, try unsetting
+ ``$PYTHONPATH`` as shown in the example above.
.. _common_startup.sh: https://github.com/galaxyproject/galaxy/blob/dev/scripts/common_startup.sh
-.. _virtualenv-burrito: https://github.com/brainsik/virtualenv-burrito
Installing unpinned dependencies
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Galaxy's dependencies can be installed either "pinned" (they will be installed
-at exact versions specified for your Galaxy release) or "unpinned" (the latest
-versions of all dependencies will be installed unless there are known
-incompatibilities with new versions). By default, the release branch(es) of
-Galaxy use pinned versions for three reasons:
+Galaxy's dependencies can be installed either "pinned" (they will be installed at exact versions specified for your
+Galaxy release) or "unpinned" (the latest versions of all dependencies will be installed unless there are known
+incompatibilities with new versions). By default, the release branches of Galaxy use pinned versions for three reasons:
-1. Using pinned versions insures that the prebuilt wheels on
- `wheels.galaxyproject.org`_ will be installed, and no compilation will be
- necesseary.
+1. Using pinned versions insures that the prebuilt wheels on `wheels.galaxyproject.org`_ will be installed, and no
+ compilation will be necesseary.
-2. Galaxy releases are tested with the pinned versions and this allows us to
- give as much assurance as possible that the pinned versions will work with
- the given Galaxy release (especially as time progresses and newer dependency
- versions are released while the Galaxy release receives fewer updates.
+2. Galaxy releases are tested with the pinned versions and this allows us to give as much assurance as possible that the
+ pinned versions will work with the given Galaxy release (especially as time progresses and newer dependency versions
+ are released while the Galaxy release receives fewer updates.
-3. Pinning furthers Galaxy's goal of reproducibility as differing dependency
- versions could result in non-reproducible behavior.
+3. Pinning furthers Galaxy's goal of reproducibility as differing dependency versions could result in non-reproducible
+ behavior.
-Install dependencies using the `unpinned requirements file`_, and then instruct
-Galaxy to start without attempting to fetch wheels:
+If you would like to install unpinned versions of Galaxy's dependencies, install dependencies using the `unpinned
+requirements file`_, and then instruct Galaxy to start without attempting to fetch wheels:
.. code-block:: console
- (venv)$ pip install --index-url=https://wheels.galaxyproject.org/simple/ -r lib/galaxy/dependencies/requirements.txt
+ (venv)$ pip install -r lib/galaxy/dependencies/requirements.txt
(venv)$ deactivate
$ sh run.sh --no-create-venv --skip-wheels
-Including ``--index-url=https://wheels.galaxyproject.org/simple/`` is important
-as two dependencies (pysam, Whoosh) include modifications specific to Galaxy
-which are only available on `wheels.galaxyproject.org`_.
+You may be able to save yourself some compiling by adding the argument ``--index-url
+https://wheels.galaxyproject.org/simple/`` to ``pip install``, but it is possible to install all of Galaxy's
+dependencies directly from PyPI_.
.. _unpinned requirements file: https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/dependencies/requirements.txt
+.. _PyPI: https://pypi.org
-Wheel interaction with other software
--------------------------------------
+Dependency management complications
+-----------------------------------
+
+Certain deployment scenarios or other software may complicate Galaxy dependency management. If you use any of these,
+relevant information can be found in the corresponding subsection below.
Galaxy job handlers
^^^^^^^^^^^^^^^^^^^
-All Galaxy jobs run a metadata detection step on the job outputs upon
-completion of the tool. The metadata detection step requires many of Galaxy's
-dependencies. Because of this, it's necessary to make sure the metadata
-detection step runs in Galaxy's virtualenv. If you run a relatively simple
-Galaxy setup (e.g. single process, or multiple Python Paste processes started
-using ``run.sh``) then this is assured for you automatically. In more
-complicated setups (supervisor, the "headless" Galaxy handler, and/or the
-virtualenv used to start Galaxy is not a shared filesystem) it may be necessary
-to make sure the handlers know where the virtualenv (or a virtualenv containing
-Galaxy's dependencies) can be found.
+All Galaxy jobs run a metadata detection step on the job outputs upon completion of the tool. The metadata detection
+step requires many of Galaxy's dependencies. Because of this, it's necessary to make sure the metadata detection step
+runs in Galaxy's virtualenv. If you run a relatively simple Galaxy deployment (e.g. ``run.sh``) then this is assured for
+you automatically. In more complicated setups (running under supervisor and/or the virtualenv used to start Galaxy is
+not on a shared filesystem) it may be necessary to make sure the handlers know where the virtualenv (or a virtualenv
+containing Galaxy's dependencies) can be found.
-If your jobs are failing due to Python ``ImportError`` exceptions, this is most
-likely the problem. If so, you can use the ```` tag in ``job_conf.xml`` to
-source the virtualenv. For example:
+If the virtualenv cannot be located, you will see job failures due to Python ``ImportError`` exceptions, like so:
+
+.. code-block:: pytb
+
+ Traceback (most recent call last):
+ File "/srv/galaxy/tmp/job_working_directory/001/set_metadata_RK41sy.py", line 1, in
+ from galaxy_ext.metadata.set_metadata import set_metadata; set_metadata()
+ File "/srv/galaxy/server/lib/galaxy_ext/metadata/set_metadata.py", line 23, in
+ from sqlalchemy.orm import clear_mappers
+ ImportError: No module named sqlalchemy.orm
+
+If this is the case, you can instruct jobs to activate the virtualenv with an ``env`` tag in ``job_conf.xml``:
.. code-block:: xml
-
-
- ...
-
-
-
- ...cluster options...
+
+