mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-21 13:50:20 +08:00
Merge branch 'release_18.01' into dev
I removed the conflicted file that was modified in dev after removal in 18.01
This commit is contained in:
Martin Cech
@@ -1,7 +1,7 @@
|
||||
import Utils from "utils/utils";
|
||||
import Menu from "layout/menu";
|
||||
import Scratchbook from "layout/scratchbook";
|
||||
import QuotaMeter from "mvc/user/user-quotameter";
|
||||
|
||||
/** Masthead **/
|
||||
var View = Backbone.View.extend({
|
||||
initialize: function(options) {
|
||||
@@ -62,7 +62,9 @@ var View = Backbone.View.extend({
|
||||
var text = "";
|
||||
self.collection.each(model => {
|
||||
var q = model.get("onbeforeunload") && model.get("onbeforeunload")();
|
||||
q && (text += `${q} `);
|
||||
if (q) {
|
||||
text += `${q} `;
|
||||
}
|
||||
});
|
||||
if (text !== "") {
|
||||
return text;
|
||||
|
||||
@@ -105,15 +105,15 @@ var Collection = Backbone.Collection.extend({
|
||||
//
|
||||
// Webhooks
|
||||
//
|
||||
Webhooks.add({
|
||||
url: "api/webhooks/masthead/all",
|
||||
Webhooks.load({
|
||||
type: "masthead",
|
||||
callback: function(webhooks) {
|
||||
$(document).ready(() => {
|
||||
$.each(webhooks.models, (index, model) => {
|
||||
webhooks.each(model => {
|
||||
var webhook = model.toJSON();
|
||||
if (webhook.activate) {
|
||||
var obj = {
|
||||
id: webhook.name,
|
||||
id: webhook.id,
|
||||
icon: webhook.config.icon,
|
||||
url: webhook.config.url,
|
||||
tooltip: webhook.config.tooltip,
|
||||
|
||||
@@ -2,6 +2,7 @@ import Masthead from "layout/masthead";
|
||||
import Panel from "layout/panel";
|
||||
import Modal from "mvc/ui/ui-modal";
|
||||
import Utils from "utils/utils";
|
||||
|
||||
var View = Backbone.View.extend({
|
||||
el: "body",
|
||||
className: "full-content",
|
||||
@@ -61,11 +62,12 @@ var View = Backbone.View.extend({
|
||||
this.render();
|
||||
|
||||
// start the router
|
||||
this.router &&
|
||||
if (this.router) {
|
||||
Backbone.history.start({
|
||||
root: Galaxy.root,
|
||||
pushState: true
|
||||
});
|
||||
}
|
||||
},
|
||||
|
||||
render: function() {
|
||||
|
||||
@@ -175,13 +175,13 @@ var menu = [
|
||||
];
|
||||
|
||||
// Webhooks
|
||||
Webhooks.add({
|
||||
url: "api/webhooks/history-menu/all",
|
||||
Webhooks.load({
|
||||
type: "history-menu",
|
||||
async: false, // (hypothetically) slows down the performance
|
||||
callback: function(webhooks) {
|
||||
var webhooks_menu = [];
|
||||
|
||||
$.each(webhooks.models, (index, model) => {
|
||||
webhooks.each(model => {
|
||||
var webhook = model.toJSON();
|
||||
if (webhook.activate) {
|
||||
webhooks_menu.push({
|
||||
|
||||
@@ -6,6 +6,7 @@ import Utils from "utils/utils";
|
||||
import Deferred from "utils/deferred";
|
||||
import Ui from "mvc/ui/ui-misc";
|
||||
import FormBase from "mvc/form/form-view";
|
||||
import Webhooks from "mvc/webhooks";
|
||||
import Citations from "components/Citations.vue";
|
||||
import Vue from "vue";
|
||||
export default FormBase.extend({
|
||||
@@ -200,19 +201,23 @@ export default FormBase.extend({
|
||||
}
|
||||
|
||||
// add tool menu webhooks
|
||||
$.getJSON("/api/webhooks/tool-menu/all", webhooks => {
|
||||
_.each(webhooks, webhook => {
|
||||
if (webhook.activate && webhook.config.function) {
|
||||
menu_button.addMenu({
|
||||
icon: webhook.config.icon,
|
||||
title: webhook.config.title,
|
||||
onclick: function() {
|
||||
var func = new Function("options", webhook.config.function);
|
||||
func(options);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
Webhooks.load({
|
||||
type: "tool-menu",
|
||||
callback: function(webhooks) {
|
||||
webhooks.each(model => {
|
||||
var webhook = model.toJSON();
|
||||
if (webhook.activate && webhook.config.function) {
|
||||
menu_button.addMenu({
|
||||
icon: webhook.config.icon,
|
||||
title: webhook.config.title,
|
||||
onclick: function() {
|
||||
var func = new Function("options", webhook.config.function);
|
||||
func(options);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
|
||||
@@ -598,7 +598,7 @@ var View = Backbone.View.extend({
|
||||
if ($.isArray(response) && response.length > 0) {
|
||||
self.$el.append($("<div/>", { id: "webhook-view" }));
|
||||
var WebhookApp = new Webhooks.WebhookView({
|
||||
urlRoot: `${Galaxy.root}api/webhooks/workflow`,
|
||||
type: "workflow",
|
||||
toolId: job_def.tool_id,
|
||||
toolVersion: job_def.tool_version
|
||||
});
|
||||
|
||||
@@ -227,7 +227,7 @@ var View = Backbone.View.extend({
|
||||
if (response.jobs && response.jobs.length > 0) {
|
||||
self.$el.append($("<div/>", { id: "webhook-view" }));
|
||||
var WebhookApp = new Webhooks.WebhookView({
|
||||
urlRoot: `${Galaxy.root}api/webhooks/tool`,
|
||||
type: "tool",
|
||||
toolId: job_def.tool_id
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,66 +1,86 @@
|
||||
/**
|
||||
Webhooks
|
||||
**/
|
||||
import Utils from "utils/utils";
|
||||
|
||||
var WebhookModel = Backbone.Model.extend({
|
||||
defaults: {
|
||||
activate: false
|
||||
const Webhooks = Backbone.Collection.extend({
|
||||
url: function() {
|
||||
return `${Galaxy.root}api/webhooks`;
|
||||
}
|
||||
});
|
||||
|
||||
var Webhooks = Backbone.Collection.extend({
|
||||
model: WebhookModel
|
||||
});
|
||||
|
||||
var WebhookView = Backbone.View.extend({
|
||||
const WebhookView = Backbone.View.extend({
|
||||
el: "#webhook-view",
|
||||
|
||||
initialize: function(options) {
|
||||
var me = this;
|
||||
var toolId = options.toolId || "";
|
||||
var toolVersion = options.toolVersion || "";
|
||||
const toolId = options.toolId || "";
|
||||
const toolVersion = options.toolVersion || "";
|
||||
|
||||
this.$el.attr("tool_id", toolId);
|
||||
this.$el.attr("tool_version", toolVersion);
|
||||
|
||||
this.model = new WebhookModel();
|
||||
this.model.urlRoot = options.urlRoot;
|
||||
this.model.fetch({
|
||||
success: function() {
|
||||
me.render();
|
||||
const webhooks = new Webhooks();
|
||||
webhooks.fetch({
|
||||
success: data => {
|
||||
if (options.type) {
|
||||
data.reset(filterType(data, options.type));
|
||||
}
|
||||
if (data.length > 0) {
|
||||
this.render(weightedRandomPick(data));
|
||||
}
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
render: function() {
|
||||
var webhook = this.model.toJSON();
|
||||
|
||||
this.$el.html(`<div id="${webhook.name}"></div>`);
|
||||
if (webhook.styles)
|
||||
$("<style/>", { type: "text/css" })
|
||||
.text(webhook.styles)
|
||||
.appendTo("head");
|
||||
if (webhook.script)
|
||||
$("<script/>", { type: "text/javascript" })
|
||||
.text(webhook.script)
|
||||
.appendTo("head");
|
||||
|
||||
render: function(model) {
|
||||
const webhook = model.toJSON();
|
||||
this.$el.html(`<div id="${webhook.id}"></div>`);
|
||||
Utils.appendScriptStyle(webhook);
|
||||
return this;
|
||||
}
|
||||
});
|
||||
|
||||
var add = options => {
|
||||
var webhooks = new Webhooks();
|
||||
|
||||
webhooks.url = Galaxy.root + options.url;
|
||||
const load = options => {
|
||||
const webhooks = new Webhooks();
|
||||
webhooks.fetch({
|
||||
async: options.async ? options.async : true,
|
||||
success: options.callback
|
||||
async: options.async !== undefined ? options.async : true,
|
||||
success: data => {
|
||||
if (options.type) {
|
||||
data.reset(filterType(data, options.type));
|
||||
}
|
||||
options.callback(data);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
function filterType(data, type) {
|
||||
return data.models.filter(item => {
|
||||
let itype = item.get("type");
|
||||
if (itype) {
|
||||
return itype.indexOf(type) !== -1;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function weightedRandomPick(data) {
|
||||
const weights = data.pluck("weight");
|
||||
const sum = weights.reduce((a, b) => a + b);
|
||||
|
||||
const normalizedWeightsMap = new Map();
|
||||
weights.forEach((weight, index) => {
|
||||
normalizedWeightsMap.set(index, parseFloat((weight / sum).toFixed(2)));
|
||||
});
|
||||
|
||||
const table = [];
|
||||
for (const [index, weight] of normalizedWeightsMap) {
|
||||
for (let i = 0; i < weight * 100; i++) {
|
||||
table.push(index);
|
||||
}
|
||||
}
|
||||
|
||||
return data.at(table[Math.floor(Math.random() * table.length)]);
|
||||
}
|
||||
|
||||
export default {
|
||||
Webhooks: Webhooks,
|
||||
WebhookView: WebhookView,
|
||||
add: add
|
||||
load: load
|
||||
};
|
||||
|
||||
@@ -25,6 +25,8 @@ window.make_popup_menus = Popupmenu.make_popup_menus;
|
||||
import init_tag_click_function from "ui/autocom_tagging";
|
||||
window.init_tag_click_function = init_tag_click_function;
|
||||
import Tours from "mvc/tours";
|
||||
import Webhooks from "mvc/webhooks";
|
||||
import Utils from "utils/utils";
|
||||
// console.debug( 'galaxy globals loaded' );
|
||||
|
||||
// ============================================================================
|
||||
@@ -176,23 +178,19 @@ $(document).ready(() => {
|
||||
Tours.activeGalaxyTourRunner();
|
||||
|
||||
function onloadWebhooks() {
|
||||
// Wait until Galaxy.config is loaded.
|
||||
if (Galaxy.config) {
|
||||
if (Galaxy.config.enable_webhooks) {
|
||||
// Load all webhooks with the type 'onload'
|
||||
$.getJSON(`${Galaxy.root}api/webhooks/onload/all`, webhooks => {
|
||||
_.each(webhooks, webhook => {
|
||||
if (Galaxy.root !== undefined) {
|
||||
// Load all webhooks with the type 'onload'
|
||||
Webhooks.load({
|
||||
type: "onload",
|
||||
callback: function(webhooks) {
|
||||
webhooks.each(model => {
|
||||
var webhook = model.toJSON();
|
||||
if (webhook.activate && webhook.script) {
|
||||
$("<script/>", { type: "text/javascript" })
|
||||
.text(webhook.script)
|
||||
.appendTo("head");
|
||||
$("<style/>", { type: "text/css" })
|
||||
.text(webhook.styles)
|
||||
.appendTo("head");
|
||||
Utils.appendScriptStyle(webhook);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
} else {
|
||||
setTimeout(onloadWebhooks, 100);
|
||||
}
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
/** Creates a generic/global Galaxy environment, loads shared libraries and a fake server */
|
||||
/* global define */
|
||||
|
||||
define(
|
||||
[
|
||||
"jquery",
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
/* global define */
|
||||
/* global QUnit */
|
||||
|
||||
import testApp from "qunit/test-app";
|
||||
import Page from "layout/page";
|
||||
import Panel from "layout/panel";
|
||||
|
||||
QUnit.module("Page test", {
|
||||
beforeEach: function() {
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
name: searchover
|
||||
id: searchover
|
||||
type:
|
||||
- masthead
|
||||
activate: true
|
||||
+1
-1
@@ -38,7 +38,7 @@ $(document).ready(function() {
|
||||
e.stopPropagation();
|
||||
if ( $( '.search-screen-overlay' ).is( ':visible' ) ){
|
||||
self.removeOverlay();
|
||||
}
|
||||
}
|
||||
else {
|
||||
self.clearSearchResults();
|
||||
self.showOverlay();
|
||||
+2
-2
@@ -1,4 +1,4 @@
|
||||
name: tool_list
|
||||
id: tool_list
|
||||
type:
|
||||
- masthead
|
||||
activate: true
|
||||
@@ -13,7 +13,7 @@ function: >
|
||||
'be patient, this might take a moment.');
|
||||
}, 1);
|
||||
|
||||
$.getJSON(Galaxy.root + "api/webhooks/tool_list/get_data", function(data) {
|
||||
$.getJSON(Galaxy.root + "api/webhooks/tool_list/data", function(data) {
|
||||
var popup = window.open('tool_list.html');
|
||||
var html = '<!DOCTYPE html><body><h2>' +
|
||||
'Create a Docker flavour of this instance:</h2>' +
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
name: tour_generator
|
||||
id: tour_generator
|
||||
type:
|
||||
- onload
|
||||
- tool-menu
|
||||
+1
-1
@@ -10,7 +10,7 @@ $(document).ready(function() {
|
||||
$('#execute').attr('tour_id', 'execute');
|
||||
|
||||
Toastr.info('Tour generation might take some time.');
|
||||
$.getJSON('/api/webhooks/tour_generator/get_data/', {
|
||||
$.getJSON('/api/webhooks/tour_generator/data/', {
|
||||
tool_id: me.toolId,
|
||||
tool_version: me.toolVersion
|
||||
}, function(obj) {
|
||||
@@ -3,55 +3,48 @@
|
||||
Framework Dependencies
|
||||
======================
|
||||
|
||||
Galaxy is a large Python application with a long list of `Python module
|
||||
dependencies`_. As a result, the Galaxy developers have made significant effort
|
||||
to provide these dependencies in as simple a method as possible. Prior to the
|
||||
16.01 release, this was done by distributing dependencies in Python's old
|
||||
standard packaging format (Egg) in a non-standard way. As of the 16.01 release,
|
||||
Galaxy now distributes dependencies in Python's new standard packaging format
|
||||
(Wheel), albeit still in a non-standard way. Thankfully, the new distribution
|
||||
method is far more compatible with the standard package distribution tooling
|
||||
(i.e. `pip`_) than the old method.
|
||||
Galaxy is a large Python application with a long list of `Python module dependencies`_. As a result, the Galaxy
|
||||
developers have made significant effort to provide these dependencies in as simple a method as possible while remaining
|
||||
compatible with the `Python packaging best practices`_. Thus, Galaxy's runtime setup procedure makes use of virtualenv_
|
||||
for package environment isolation, pip_ for installation, and wheel_ to provide pre-built versions of dependencies.
|
||||
|
||||
In addition to framework dependencies, as of Galaxy 18.01, the client (UI) is no longer provided in its built format
|
||||
**in the development (``dev``) branch of the source repository**. The built client is still provided in
|
||||
``release_YY.MM`` branches and the ``master`` branch.
|
||||
|
||||
.. _Python module dependencies: https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/dependencies/requirements.txt
|
||||
.. _pip: https://pip.pypa.io/
|
||||
.. _wheel: https://wheel.readthedocs.org/
|
||||
.. _Python packaging best practices: https://packaging.python.org
|
||||
.. _virtualenv: https://packaging.python.org/tutorials/installing-packages/#creating-virtual-environments
|
||||
.. _pip: https://packaging.python.org/tutorials/installing-packages/#use-pip-for-installing
|
||||
.. _wheel: https://packaging.python.org/tutorials/installing-packages/#source-distributions-vs-wheels
|
||||
|
||||
How it works
|
||||
------------
|
||||
|
||||
Upon startup (with ``run.sh``), the startup scripts will:
|
||||
|
||||
1. Create a Python `virtualenv`_ in the directory ``.venv``.
|
||||
1. Create a Python virtualenv_ in the directory ``.venv``.
|
||||
|
||||
2. Unset the ``$PYTHONPATH`` environment variable (if set) as this can
|
||||
interfere with installing `Galaxy pip`_ and dependencies.
|
||||
2. Unset the ``$PYTHONPATH`` environment variable (if set) as this can interfere with installing dependencies.
|
||||
|
||||
3. Replace that virtualenv's pip with `Galaxy pip`_.
|
||||
3. Download and install packages from the Galaxy project wheel server, wheels.galaxyproject.org_, as well as the `Python
|
||||
Package Index`_ (aka PyPI) , using pip_.
|
||||
|
||||
4. If applicable, create a ``binary-compatibility.cfg`` (see the `Galaxy pip
|
||||
and wheel`_ section for an explanation of this file).
|
||||
4. Start Galaxy using ``.venv/bin/python``.
|
||||
|
||||
5. Download and install wheels from the Galaxy project wheel server,
|
||||
`wheels.galaxyproject.org`_, using pip.
|
||||
|
||||
6. Start Galaxy using ``.venv/bin/python``.
|
||||
|
||||
.. _virtualenv: https://virtualenv.readthedocs.org/
|
||||
.. _wheels.galaxyproject.org: https://wheels.galaxyproject.org/
|
||||
.. _Python Package Index: https://pypi.org
|
||||
|
||||
Options
|
||||
-------
|
||||
|
||||
A variety of options to ``run.sh`` are available to control the above behavior:
|
||||
|
||||
- ``--skip-venv``: Do not create or use a virtualenv, and do not replace pip
|
||||
with Galaxy pip.
|
||||
- ``--skip-venv``: Do not create or use a virtualenv.
|
||||
- ``--skip-wheels``: Do not install wheels.
|
||||
- ``--no-create-venv``: Do not create a virtualenv, but use one if it exists at
|
||||
``.venv`` or if ``$VIRTUAL_ENV`` is set (this variable is set by virtualenv's
|
||||
``activate``)
|
||||
- ``--replace-pip/--no-replace-pip``: Do/do not replace pip with Galaxy pip.
|
||||
- ``--no-create-venv``: Do not create a virtualenv, but use one if it exists at ``.venv`` or if ``$VIRTUAL_ENV`` is set
|
||||
(this variable is set by virtualenv's ``activate``).
|
||||
- ``--replace-pip/--no-replace-pip``: Do/do not upgrade pip if necessary.
|
||||
|
||||
Managing dependencies manually
|
||||
------------------------------
|
||||
@@ -59,29 +52,23 @@ Managing dependencies manually
|
||||
Create a virtualenv
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Using a `virtualenv`_ in ``.venv`` under the Galaxy source tree is not
|
||||
required. More complicated Galaxy setups may choose to use a virtualenv
|
||||
external to the Galaxy source tree, which can be done either by not using
|
||||
``run.sh`` directly (an example of this can be found under the `Supervisor`_
|
||||
section) or using the ``--no-create-venv`` option, explained in the `Options`_
|
||||
section. It is also possible to force Galaxy to start without a virtualenv at
|
||||
all, but you should not do this unless you know what you're doing.
|
||||
Using a `virtualenv`_ in ``.venv`` under the Galaxy source tree is not required. More complicated Galaxy setups may
|
||||
choose to use a virtualenv external to the Galaxy source tree, which can be done either by not using ``run.sh`` directly
|
||||
(an example of this can be found under the `Scaling and Load Balancing` documentation) or using the ``--no-create-venv``
|
||||
option, explained in the `Options` section. It is also possible to force Galaxy to start without a virtualenv at all,
|
||||
but you should not do this unless you know what you're doing.
|
||||
|
||||
To manually create a virtualenv, you will first need to obtain virtualenv.
|
||||
There are a variety of ways to do this:
|
||||
To manually create a virtualenv, you will first need to obtain virtualenv. There are a variety of ways to do this:
|
||||
|
||||
- ``pip install virtualenv``
|
||||
- ``brew install virtualenv``
|
||||
- Install your Linux distribution's virtualenv package from the system package
|
||||
manager (e.g. ``apt-get install python-virtualenv``).
|
||||
- Download the `virtualenv source from PyPI
|
||||
<https://pypi.python.org/pypi/virtualenv>`_, untar, and run the
|
||||
``virtualenv.py`` script contained within as ``python virtualenv.py
|
||||
/path/to/galaxy/virtualenv``
|
||||
- Install your Linux distribution's virtualenv package from the system package manager (e.g. ``apt-get install
|
||||
python-virtualenv``).
|
||||
- Download the `virtualenv source from PyPI <https://pypi.python.org/pypi/virtualenv>`_, untar, and run the
|
||||
``virtualenv.py`` script contained within as ``python virtualenv.py /path/to/galaxy/virtualenv``
|
||||
|
||||
Once this is done, create a virtualenv. In our example, the virtualenv will
|
||||
live in ``/srv/galaxy/venv`` and the Galaxy source code has been cloned to
|
||||
``/srv/galaxy/server``.
|
||||
Once this is done, create a virtualenv. In our example, the virtualenv will live in ``/srv/galaxy/venv`` and the Galaxy
|
||||
source code has been cloned to ``/srv/galaxy/server``.
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
@@ -91,550 +78,316 @@ live in ``/srv/galaxy/venv`` and the Galaxy source code has been cloned to
|
||||
$ . /srv/galaxy/venv/bin/activate
|
||||
(venv)$
|
||||
|
||||
Next, in ``galaxy.yml``, set the ``virtualenv`` option in the ``uwsgi`` section to point to your new virtualenv:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
---
|
||||
uwsgi:
|
||||
#...
|
||||
virtualenv: /srv/galaxy/venv
|
||||
|
||||
Install dependencies
|
||||
^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Normally, ``run.sh`` calls `common_startup.sh`_, which creates the virtualenv,
|
||||
installs Galaxy pip, and installs dependencies. You can call this script
|
||||
yourself to set up Galaxy pip and the dependencies without creating a
|
||||
virtualenv using the ``--no-create-venv`` option:
|
||||
Normally, ``run.sh`` calls `common_startup.sh`_, which creates the virtualenv and installs dependencies. You can call
|
||||
this script yourself to set up Galaxy pip and the dependencies without creating a virtualenv using the
|
||||
``--no-create-venv`` option:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
(venv)$ PYTHONPATH= sh /srv/galaxy/server/scripts/common_startup.sh --no-create-venv
|
||||
Ignoring indexes: https://pypi.python.org/simple
|
||||
Collecting pip
|
||||
Downloading https://wheels.galaxyproject.org/packages/pip-8.0.0+gx1-py2.py3-none-any.whl (1.2MB)
|
||||
100% |████████████████████████████████| 1.2MB 37.8MB/s
|
||||
Installing collected packages: pip
|
||||
Found existing installation: pip 7.1.2
|
||||
Uninstalling pip-7.1.2:
|
||||
Successfully uninstalled pip-7.1.2
|
||||
Successfully installed pip-8.0.0+gx1
|
||||
Collecting bx-python==0.7.3 (from -r requirements.txt (line 2))
|
||||
Downloading https://wheels.galaxyproject.org/packages/bx_python-0.7.3-cp27-cp27mu-linux_x86_64.whl (1.7MB)
|
||||
100% |████████████████████████████████| 1.7MB 25.4MB/s
|
||||
Requirement already satisfied: pip>=8.1 in /home/nate/.virtualenvs/test/lib/python2.7/site-packages
|
||||
Collecting numpy==1.9.2 (from -r requirements.txt (line 4))
|
||||
Downloading https://wheels.galaxyproject.org/packages/numpy-1.9.2-cp27-cp27mu-manylinux1_x86_64.whl (10.2MB)
|
||||
100% |████████████████████████████████| 10.2MB 21.7MB/s
|
||||
Collecting bx-python==0.7.3 (from -r requirements.txt (line 5))
|
||||
Downloading https://wheels.galaxyproject.org/packages/bx_python-0.7.3-cp27-cp27mu-manylinux1_x86_64.whl (2.1MB)
|
||||
100% |████████████████████████████████| 2.2MB 97.2MB/s
|
||||
|
||||
...
|
||||
...
|
||||
|
||||
Collecting pysam==0.8.3+gx1 (from -r requirements.txt (line 69))
|
||||
Downloading https://wheels.galaxyproject.org/packages/pysam-0.8.3+gx1-cp27-cp27mu-linux_x86_64.whl (7.4MB)
|
||||
100% |████████████████████████████████| 7.4MB 15.1MB/s
|
||||
Installing collected packages: bx-python, MarkupSafe, PyYAML, SQLAlchemy,
|
||||
mercurial, numpy, pycrypto, six, Paste, PasteDeploy, docutils, wchartype,
|
||||
repoze.lru, Routes, WebOb, WebHelpers, Mako, pytz, Babel, Beaker,
|
||||
Markdown, Cheetah, requests, requests-toolbelt, boto, bioblend, amqp,
|
||||
anyjson, kombu, pbr, sqlparse, decorator, Tempita, sqlalchemy-migrate,
|
||||
Parsley, nose, svgwrite, ecdsa, paramiko, Fabric, Whoosh, pysam
|
||||
Successfully installed Babel-2.0 Beaker-1.7.0 Cheetah-2.4.4 Fabric-1.10.2
|
||||
Mako-1.0.2 Markdown-2.6.3 MarkupSafe-0.23 Parsley-1.3 Paste-2.0.2
|
||||
PasteDeploy-1.5.2 PyYAML-3.11 Routes-2.2 SQLAlchemy-1.0.8 svgwrite-1.1.6
|
||||
Tempita-0.5.3.dev0 WebHelpers-1.3 WebOb-1.4.1 Whoosh-2.4.1+gx1 amqp-1.4.8
|
||||
anyjson-0.3.3 bioblend-0.6.1 boto-2.38.0 bx-python-0.7.3 decorator-4.0.2
|
||||
docutils-0.12 ecdsa-0.13 kombu-3.0.30 mercurial-3.4.2 nose-1.3.7
|
||||
numpy-1.9.2 paramiko-1.15.2 pbr-1.8.0 pycrypto-2.6.1 pysam-0.8.3+gx1
|
||||
pytz-2015.4 repoze.lru-0.6 requests-2.8.1 requests-toolbelt-0.4.0
|
||||
six-1.9.0 sqlalchemy-migrate-0.10.0 sqlparse-0.1.16 wchartype-0.1
|
||||
Installing collected packages: numpy, bx-python, ...
|
||||
Successfully installed numpy-1.9.2 bx-python-0.7.3 ...
|
||||
|
||||
**Warning:** If your ``$PYTHONPATH`` is set, it may interfere with the
|
||||
dependency installation process (this will almost certainly be the case if you
|
||||
use `virtualenv-burrito`_). Without ``--no-create-venv`` the ``$PYTHONPATH``
|
||||
variable will be automatically unset, but we assume you know what you're doing
|
||||
and may want it left intact if you are using ``--no-create-venv``. If you
|
||||
encounter problems, try unsetting ``$PYTHONPATH`` as shown in the example
|
||||
above.
|
||||
.. warning::
|
||||
|
||||
If your ``$PYTHONPATH`` is set, it may interfere with the dependency installation process. Without
|
||||
``--no-create-venv`` the ``$PYTHONPATH`` variable will be automatically unset, but we assume you know what you're
|
||||
doing and may want it left intact if you are using ``--no-create-venv``. If you encounter problems, try unsetting
|
||||
``$PYTHONPATH`` as shown in the example above.
|
||||
|
||||
.. _common_startup.sh: https://github.com/galaxyproject/galaxy/blob/dev/scripts/common_startup.sh
|
||||
.. _virtualenv-burrito: https://github.com/brainsik/virtualenv-burrito
|
||||
|
||||
Installing unpinned dependencies
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Galaxy's dependencies can be installed either "pinned" (they will be installed
|
||||
at exact versions specified for your Galaxy release) or "unpinned" (the latest
|
||||
versions of all dependencies will be installed unless there are known
|
||||
incompatibilities with new versions). By default, the release branch(es) of
|
||||
Galaxy use pinned versions for three reasons:
|
||||
Galaxy's dependencies can be installed either "pinned" (they will be installed at exact versions specified for your
|
||||
Galaxy release) or "unpinned" (the latest versions of all dependencies will be installed unless there are known
|
||||
incompatibilities with new versions). By default, the release branches of Galaxy use pinned versions for three reasons:
|
||||
|
||||
1. Using pinned versions insures that the prebuilt wheels on
|
||||
`wheels.galaxyproject.org`_ will be installed, and no compilation will be
|
||||
necesseary.
|
||||
1. Using pinned versions insures that the prebuilt wheels on `wheels.galaxyproject.org`_ will be installed, and no
|
||||
compilation will be necesseary.
|
||||
|
||||
2. Galaxy releases are tested with the pinned versions and this allows us to
|
||||
give as much assurance as possible that the pinned versions will work with
|
||||
the given Galaxy release (especially as time progresses and newer dependency
|
||||
versions are released while the Galaxy release receives fewer updates.
|
||||
2. Galaxy releases are tested with the pinned versions and this allows us to give as much assurance as possible that the
|
||||
pinned versions will work with the given Galaxy release (especially as time progresses and newer dependency versions
|
||||
are released while the Galaxy release receives fewer updates.
|
||||
|
||||
3. Pinning furthers Galaxy's goal of reproducibility as differing dependency
|
||||
versions could result in non-reproducible behavior.
|
||||
3. Pinning furthers Galaxy's goal of reproducibility as differing dependency versions could result in non-reproducible
|
||||
behavior.
|
||||
|
||||
Install dependencies using the `unpinned requirements file`_, and then instruct
|
||||
Galaxy to start without attempting to fetch wheels:
|
||||
If you would like to install unpinned versions of Galaxy's dependencies, install dependencies using the `unpinned
|
||||
requirements file`_, and then instruct Galaxy to start without attempting to fetch wheels:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
(venv)$ pip install --index-url=https://wheels.galaxyproject.org/simple/ -r lib/galaxy/dependencies/requirements.txt
|
||||
(venv)$ pip install -r lib/galaxy/dependencies/requirements.txt
|
||||
(venv)$ deactivate
|
||||
$ sh run.sh --no-create-venv --skip-wheels
|
||||
|
||||
Including ``--index-url=https://wheels.galaxyproject.org/simple/`` is important
|
||||
as two dependencies (pysam, Whoosh) include modifications specific to Galaxy
|
||||
which are only available on `wheels.galaxyproject.org`_.
|
||||
You may be able to save yourself some compiling by adding the argument ``--index-url
|
||||
https://wheels.galaxyproject.org/simple/`` to ``pip install``, but it is possible to install all of Galaxy's
|
||||
dependencies directly from PyPI_.
|
||||
|
||||
.. _unpinned requirements file: https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/dependencies/requirements.txt
|
||||
.. _PyPI: https://pypi.org
|
||||
|
||||
Wheel interaction with other software
|
||||
-------------------------------------
|
||||
Dependency management complications
|
||||
-----------------------------------
|
||||
|
||||
Certain deployment scenarios or other software may complicate Galaxy dependency management. If you use any of these,
|
||||
relevant information can be found in the corresponding subsection below.
|
||||
|
||||
Galaxy job handlers
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
All Galaxy jobs run a metadata detection step on the job outputs upon
|
||||
completion of the tool. The metadata detection step requires many of Galaxy's
|
||||
dependencies. Because of this, it's necessary to make sure the metadata
|
||||
detection step runs in Galaxy's virtualenv. If you run a relatively simple
|
||||
Galaxy setup (e.g. single process, or multiple Python Paste processes started
|
||||
using ``run.sh``) then this is assured for you automatically. In more
|
||||
complicated setups (supervisor, the "headless" Galaxy handler, and/or the
|
||||
virtualenv used to start Galaxy is not a shared filesystem) it may be necessary
|
||||
to make sure the handlers know where the virtualenv (or a virtualenv containing
|
||||
Galaxy's dependencies) can be found.
|
||||
All Galaxy jobs run a metadata detection step on the job outputs upon completion of the tool. The metadata detection
|
||||
step requires many of Galaxy's dependencies. Because of this, it's necessary to make sure the metadata detection step
|
||||
runs in Galaxy's virtualenv. If you run a relatively simple Galaxy deployment (e.g. ``run.sh``) then this is assured for
|
||||
you automatically. In more complicated setups (running under supervisor and/or the virtualenv used to start Galaxy is
|
||||
not on a shared filesystem) it may be necessary to make sure the handlers know where the virtualenv (or a virtualenv
|
||||
containing Galaxy's dependencies) can be found.
|
||||
|
||||
If your jobs are failing due to Python ``ImportError`` exceptions, this is most
|
||||
likely the problem. If so, you can use the ``<env>`` tag in ``job_conf.xml`` to
|
||||
source the virtualenv. For example:
|
||||
If the virtualenv cannot be located, you will see job failures due to Python ``ImportError`` exceptions, like so:
|
||||
|
||||
.. code-block:: pytb
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "/srv/galaxy/tmp/job_working_directory/001/set_metadata_RK41sy.py", line 1, in <module>
|
||||
from galaxy_ext.metadata.set_metadata import set_metadata; set_metadata()
|
||||
File "/srv/galaxy/server/lib/galaxy_ext/metadata/set_metadata.py", line 23, in <module>
|
||||
from sqlalchemy.orm import clear_mappers
|
||||
ImportError: No module named sqlalchemy.orm
|
||||
|
||||
If this is the case, you can instruct jobs to activate the virtualenv with an ``env`` tag in ``job_conf.xml``:
|
||||
|
||||
.. code-block:: xml
|
||||
|
||||
<job_conf>
|
||||
<plugins>
|
||||
...
|
||||
</plugins>
|
||||
<destinations default="cluster">
|
||||
<destination id="cluster" runner="drmaa">
|
||||
<param id="nativeSpecification"> ...cluster options... </param>
|
||||
<destination id="cluster" runner="drmaa">
|
||||
<!-- ... other destination params -- >
|
||||
<env file="/cluster/galaxy/venv/bin/activate" />
|
||||
</destination>
|
||||
|
||||
<env file="/galaxy/server/.venv/bin/activate" />
|
||||
|
||||
</destination>
|
||||
</destinations>
|
||||
</job_conf>
|
||||
|
||||
If your Galaxy server's virtualenv isn't available on the cluster you can
|
||||
create one manually using the instructions under `Managing dependencies
|
||||
manually`_.
|
||||
If your Galaxy server's virtualenv isn't available on the cluster you can create one manually using the instructions
|
||||
under `Managing dependencies manually`_.
|
||||
|
||||
Pulsar
|
||||
^^^^^^
|
||||
|
||||
If using `Pulsar`_'s option to set metadata on the remote server, the same
|
||||
conditions as with `Galaxy job handlers`_ apply. You should create a virtualenv
|
||||
on the remote resource, install Galaxy's dependencies in to it, and set an
|
||||
``<env>`` tag pointing to the virtualenv's ``activate`` as in the `Galaxy job
|
||||
handlers`_ section. Instructions on how to create a virtualenv can be found
|
||||
under the `Managing dependencies manually`_ section.
|
||||
If using `Pulsar`_'s option to set metadata on the remote server, the same conditions as with `Galaxy job handlers`_
|
||||
apply. You should create a virtualenv on the remote resource, install Galaxy's dependencies in to it, and set an
|
||||
``<env>`` tag pointing to the virtualenv's ``activate`` as in the `Galaxy job handlers`_ section. Instructions on how to
|
||||
create a virtualenv can be found under the `Managing dependencies manually`_ section.
|
||||
|
||||
.. _Pulsar: http://pulsar.readthedocs.org/
|
||||
|
||||
Conda
|
||||
^^^^^
|
||||
|
||||
`Conda`_ and `virtualenv`_ are incompatible. However, Conda provides its own
|
||||
environment separation functionality in the form of `Conda environments`_.
|
||||
Starting Galaxy with Conda Python will cause ``--skip-venv`` to be implicitly
|
||||
set, and the currently active Conda environment will be used to install Galaxy
|
||||
framework dependencies instaead. Be sure to create and activate a Conda
|
||||
environment for Galaxy prior to installing packages and/or starting Galaxy.
|
||||
`Conda`_ and `virtualenv`_ are incompatible. However, Conda provides its own environment separation functionality in the
|
||||
form of `Conda environments`_. Starting Galaxy with Conda Python will cause ``--skip-venv`` to be implicitly set, and
|
||||
the currently active Conda environment will be used to install Galaxy framework dependencies instead.
|
||||
|
||||
You may choose to install Galaxy's dependencies either at their `pinned`_
|
||||
versions using pip or `unpinned`_ using a combination of conda and pip. When
|
||||
running under Conda, pip is not replaced with Galaxy pip, so installing pinned
|
||||
dependencies will require compilation, will be slower and requires having those
|
||||
dependencies' build-time dependencies installed, but has benefits as explained
|
||||
under the `Installing unpinned dependencies`_ section. Installing unpinned
|
||||
dependencies allows you to use Conda's binary packages for quick and easy
|
||||
installation.
|
||||
.. caution::
|
||||
|
||||
Pinned dependencies will be installed by default when running ``run.sh``. To
|
||||
install unpinned dependencies, the process is similar as to installing unpinned
|
||||
versions without Conda, with the extra step of installing as much as possible
|
||||
from Conda/Bioconda before installing from pip. Begin by adding the `Bioconda`_
|
||||
channel as explained in the `Bioconda instructions`_ and then creating a new
|
||||
Conda environment using the provided Conda environment file. Then, install
|
||||
remaining dependencies using pip and start Galaxy, instructing it to skip the
|
||||
automatic fetching of pinned dependencies.
|
||||
Be sure to create and activate a Conda environment for Galaxy prior to installing packages and/or starting Galaxy or
|
||||
else they will be installed in the Conda root environment.
|
||||
|
||||
Because Conda package names typically match PyPI package names, you can install Conda versions of what dependencies are
|
||||
available from conda-forge_ and Bioconda_ using a script provided with Galaxy:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ conda config --add channels r
|
||||
$ conda config --add channels conda-forge
|
||||
$ conda config --add channels bioconda
|
||||
$ conda create --name galaxy --file lib/galaxy/dependencies/conda-environment.txt
|
||||
Fetching package metadata: ........
|
||||
Solving package specifications: ............................................
|
||||
Package plan for installation in environment /home/nate/conda/envs/galaxy:
|
||||
$ conda create --name galaxy --file <(lib/galaxy/dependencies/conda-file.sh)
|
||||
Filtering out requirements not available in conda... done
|
||||
Solving environment: done
|
||||
|
||||
The following packages will be downloaded:
|
||||
## Package Plan ##
|
||||
|
||||
environment location: /srv/galaxy/conda/envs/galaxy
|
||||
|
||||
added / updated specs:
|
||||
- anyjson==0.3.3
|
||||
#...
|
||||
- whoosh==2.7.4
|
||||
|
||||
package | build
|
||||
---------------------------|-----------------
|
||||
boto-2.38.0 | py27_0 1.3 MB
|
||||
cheetah-2.4.4 | py27_0 267 KB
|
||||
decorator-4.0.6 | py27_0 11 KB
|
||||
docutils-0.12 | py27_0 636 KB
|
||||
ecdsa-0.11 | py27_0 73 KB
|
||||
markupsafe-0.23 | py27_0 30 KB
|
||||
mercurial-3.4.2 | py27_0 2.9 MB
|
||||
nose-1.3.7 | py27_0 194 KB
|
||||
paste-1.7.5.1 | py27_0 490 KB
|
||||
pytz-2015.7 | py27_0 174 KB
|
||||
repoze.lru-0.6 | py27_0 15 KB
|
||||
requests-2.9.1 | py27_0 605 KB
|
||||
six-1.10.0 | py27_0 16 KB
|
||||
sqlalchemy-1.0.11 | py27_0 1.3 MB
|
||||
sqlparse-0.1.18 | py27_0 51 KB
|
||||
webob-1.4.1 | py27_0 108 KB
|
||||
babel-2.1.1 | py27_0 2.3 MB
|
||||
bx-python-0.7.3 | np110py27_1 2.1 MB
|
||||
mako-1.0.3 | py27_0 105 KB
|
||||
paramiko-1.15.2 | py27_0 197 KB
|
||||
pastedeploy-1.5.2 | py27_1 23 KB
|
||||
requests-toolbelt-0.5.0 | py27_0 83 KB
|
||||
routes-2.2 | py27_0 48 KB
|
||||
bioblend-0.7.0 | py27_0 181 KB
|
||||
fabric-1.10.2 | py27_0 108 KB
|
||||
------------------------------------------------------------
|
||||
Total: 13.2 MB
|
||||
|
||||
The following NEW packages will be INSTALLED:
|
||||
|
||||
babel: 2.1.1-py27_0
|
||||
bioblend: 0.7.0-py27_0
|
||||
boto: 2.38.0-py27_0
|
||||
bx-python: 0.7.3-np110py27_1
|
||||
cheetah: 2.4.4-py27_0
|
||||
decorator: 4.0.6-py27_0
|
||||
docutils: 0.12-py27_0
|
||||
ecdsa: 0.11-py27_0
|
||||
fabric: 1.10.2-py27_0
|
||||
libgfortran: 1.0-0
|
||||
mako: 1.0.3-py27_0
|
||||
markupsafe: 0.23-py27_0
|
||||
mercurial: 3.4.2-py27_0
|
||||
nose: 1.3.7-py27_0
|
||||
numpy: 1.10.2-py27_0
|
||||
openblas: 0.2.14-3
|
||||
openssl: 1.0.2e-0
|
||||
paramiko: 1.15.2-py27_0
|
||||
paste: 1.7.5.1-py27_0
|
||||
pastedeploy: 1.5.2-py27_1
|
||||
pip: 7.1.2-py27_0
|
||||
pycrypto: 2.6.1-py27_0
|
||||
python: 2.7.11-0
|
||||
pytz: 2015.7-py27_0
|
||||
pyyaml: 3.11-py27_1
|
||||
readline: 6.2-2
|
||||
repoze.lru: 0.6-py27_0
|
||||
requests: 2.9.1-py27_0
|
||||
requests-toolbelt: 0.5.0-py27_0
|
||||
routes: 2.2-py27_0
|
||||
setuptools: 19.2-py27_0
|
||||
six: 1.10.0-py27_0
|
||||
sqlalchemy: 1.0.11-py27_0
|
||||
sqlite: 3.9.2-0
|
||||
sqlparse: 0.1.18-py27_0
|
||||
tk: 8.5.18-0
|
||||
webob: 1.4.1-py27_0
|
||||
wheel: 0.26.0-py27_1
|
||||
yaml: 0.1.6-0
|
||||
zlib: 1.2.8-0
|
||||
anyjson: 0.3.3-py27_1 conda-forge
|
||||
#...
|
||||
zlib: 1.2.8-3 conda-forge
|
||||
|
||||
Proceed ([y]/n)?
|
||||
Proceed ([y]/n)?
|
||||
|
||||
Fetching packages ...
|
||||
boto-2.38.0-py 100% |############################################| Time: 0:00:00 3.27 MB/s
|
||||
cheetah-2.4.4- 100% |############################################| Time: 0:00:00 1.65 MB/s
|
||||
decorator-4.0. 100% |############################################| Time: 0:00:00 20.38 MB/s
|
||||
docutils-0.12- 100% |############################################| Time: 0:00:00 2.21 MB/s
|
||||
ecdsa-0.11-py2 100% |############################################| Time: 0:00:00 762.58 kB/s
|
||||
markupsafe-0.2 100% |############################################| Time: 0:00:00 931.23 kB/s
|
||||
mercurial-3.4. 100% |############################################| Time: 0:00:00 5.36 MB/s
|
||||
nose-1.3.7-py2 100% |############################################| Time: 0:00:00 1.12 MB/s
|
||||
paste-1.7.5.1- 100% |############################################| Time: 0:00:00 1.91 MB/s
|
||||
pytz-2015.7-py 100% |############################################| Time: 0:00:00 1.08 MB/s
|
||||
repoze.lru-0.6 100% |############################################| Time: 0:00:00 465.26 kB/s
|
||||
requests-2.9.1 100% |############################################| Time: 0:00:00 2.28 MB/s
|
||||
six-1.10.0-py2 100% |############################################| Time: 0:00:00 477.04 kB/s
|
||||
sqlalchemy-1.0 100% |############################################| Time: 0:00:00 4.25 MB/s
|
||||
sqlparse-0.1.1 100% |############################################| Time: 0:00:00 774.57 kB/s
|
||||
webob-1.4.1-py 100% |############################################| Time: 0:00:00 819.13 kB/s
|
||||
babel-2.1.1-py 100% |############################################| Time: 0:00:00 5.53 MB/s
|
||||
bx-python-0.7. 100% |############################################| Time: 0:00:00 5.11 MB/s
|
||||
mako-1.0.3-py2 100% |############################################| Time: 0:00:00 813.04 kB/s
|
||||
paramiko-1.15. 100% |############################################| Time: 0:00:00 1.23 MB/s
|
||||
pastedeploy-1. 100% |############################################| Time: 0:00:00 721.20 kB/s
|
||||
requests-toolb 100% |############################################| Time: 0:00:00 856.06 kB/s
|
||||
routes-2.2-py2 100% |############################################| Time: 0:00:00 666.70 kB/s
|
||||
bioblend-0.7.0 100% |############################################| Time: 0:00:00 1.15 MB/s
|
||||
fabric-1.10.2- 100% |############################################| Time: 0:00:00 843.81 kB/s
|
||||
Extracting packages ...
|
||||
[ COMPLETE ]|###############################################################| 100%
|
||||
Linking packages ...
|
||||
[ COMPLETE ]|###############################################################| 100%
|
||||
Preparing transaction: done
|
||||
Verifying transaction: done
|
||||
Executing transaction: done
|
||||
#
|
||||
# To activate this environment, use:
|
||||
# $ source activate galaxy
|
||||
# To activate this environment, use
|
||||
#
|
||||
# To deactivate this environment, use:
|
||||
# $ source deactivate
|
||||
# $ conda activate galaxy
|
||||
#
|
||||
$ source activate galaxy
|
||||
discarding /home/nate/conda/bin from PATH
|
||||
prepending /home/nate/conda/envs/galaxy/bin to PATH
|
||||
$ pip install --index-url=https://wheels.galaxyproject.org/simple/ -r lib/galaxy/dependencies/requirements.txt
|
||||
Requirement already satisfied (use --upgrade to upgrade): numpy in /home/nate/conda/envs/galaxy/lib/python2.7/site-packages (from -r lib/galaxy/dependencies/requirements.txt (line 1))
|
||||
# To deactivate an active environment, use
|
||||
#
|
||||
# $ conda deactivate
|
||||
|
||||
...
|
||||
Next, activate the environment and run ``pip`` to fetch the remaining dependencies:
|
||||
|
||||
Collecting WebHelpers (from -r lib/galaxy/dependencies/requirements.txt (line 15))
|
||||
Downloading https://wheels.galaxyproject.org/packages/WebHelpers-1.3-py2-none-any.whl (149kB)
|
||||
100% |████████████████████████████████| 151kB 55.7MB/s
|
||||
.. code-block:: console
|
||||
|
||||
...
|
||||
$ conda activate galaxy
|
||||
$ pip install --index-url https://wheels.galaxyproject.org/simple/ --extra-index-url https://pypi.python.org/simple/ -r requirements.txt
|
||||
Requirement already satisfied: pip>=8.1 in /srv/galaxy/conda/envs/galaxy/lib/python2.7/site-packages
|
||||
#...
|
||||
Collecting SQLAlchemy==1.0.15 (from -r requirements.txt (line 8))
|
||||
Downloading https://wheels.galaxyproject.org/packages/SQLAlchemy-1.0.15-cp27-cp27mu-manylinux1_x86_64.whl (1.0MB)
|
||||
100% |████████████████████████████████| 1.0MB 48.6MB/s
|
||||
#...
|
||||
Installing collected packages: SQLAlchemy, ...
|
||||
Successfully installed SQLAlchemy-1.0.15 ...
|
||||
|
||||
Building wheels for collected packages: pysam
|
||||
Running setup.py bdist_wheel for pysam
|
||||
``run.sh`` is not currently compatible with running without a virtualenv. In this case, you should start with uWSGI
|
||||
directly. Be sure to uncomment the required options in the ``uwsgi`` section of ``galaxy.yml`` since ``run.sh`` normally
|
||||
sets these for you on the command line:
|
||||
|
||||
$ sh run.sh --skip-wheels
|
||||
.. code-block:: console
|
||||
|
||||
$ uwsgi --yaml config/galaxy.yml
|
||||
[uWSGI] getting YAML configuration from config/galaxy.yml
|
||||
[uwsgi-static] added mapping for /static/style => static/style/blue
|
||||
[uwsgi-static] added mapping for /static => static
|
||||
*** Starting uWSGI 2.0.15 (64bit) on [Thu Jan 25 11:57:17 2018] ***
|
||||
|
||||
You may encounter the following traceback when starting Galaxy:
|
||||
|
||||
.. code-block:: pytb
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "lib/galaxy/main.py", line 278, in <module>
|
||||
main()
|
||||
File "lib/galaxy/main.py", line 274, in main
|
||||
app_loop(args, log)
|
||||
File "lib/galaxy/main.py", line 124, in app_loop
|
||||
log=log,
|
||||
File "lib/galaxy/main.py", line 91, in load_galaxy_app
|
||||
from galaxy.app import UniverseApplication
|
||||
File "/srv/galaxy/server/lib/galaxy/app.py", line 30, in <module>
|
||||
from galaxy.visualization.data_providers.registry import DataProviderRegistry
|
||||
File "/srv/galaxy/server/lib/galaxy/visualization/data_providers/registry.py", line 15, in <module>
|
||||
from galaxy.visualization.data_providers import genome
|
||||
File "/srv/galaxy/server/lib/galaxy/visualization/data_providers/genome.py", line 17, in <module>
|
||||
from bx.bbi.bigbed_file import BigBedFile
|
||||
ImportError: cannot import name BigBedFile
|
||||
|
||||
If this is the case, uninstall bx-python from Conda and reinstall it from the Galaxy wheel:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ conda remove bx-python
|
||||
Solving environment: done
|
||||
|
||||
## Package Plan ##
|
||||
|
||||
environment location: /srv/galaxy/conda/envs/galaxy
|
||||
|
||||
removed specs:
|
||||
- bx-python
|
||||
|
||||
|
||||
The following packages will be REMOVED:
|
||||
|
||||
bx-python: 0.7.3-py27_0 bioconda
|
||||
|
||||
Proceed ([y]/n)?
|
||||
|
||||
Preparing transaction: done
|
||||
Verifying transaction: done
|
||||
Executing transaction: done
|
||||
$ pip install --index-url https://wheels.galaxyproject.org/simple bx-python
|
||||
Collecting bx-python
|
||||
Downloading https://wheels.galaxyproject.org/packages/bx_python-0.7.3-cp27-cp27mu-manylinux1_x86_64.whl (2.1MB)
|
||||
100% |████████████████████████████████| 2.2MB 66.1MB/s
|
||||
Installing collected packages: bx-python
|
||||
Successfully installed bx-python-0.7.3
|
||||
|
||||
.. _Conda: http://conda.pydata.org/
|
||||
.. _Conda environments: http://conda.pydata.org/docs/using/envs.html
|
||||
.. _conda-forge: https://conda-forge.org/
|
||||
.. _Bioconda: https://bioconda.github.io/
|
||||
.. _Bioconda instructions: Bioconda_
|
||||
.. _pinned: `Installing unpinned dependencies`_
|
||||
.. _unpinned: pinned_
|
||||
|
||||
uWSGI
|
||||
^^^^^
|
||||
|
||||
The simplest scenario to using uWSGI with the wheel-based dependencies is to
|
||||
install uWSGI into Galaxy virtualenv (by default, ``.venv``) using pip, e.g.:
|
||||
``run.sh`` should automatically set ``--virtualenv`` on uWSGI's command line. However, you can override this using the
|
||||
``virtualenv`` option in the ``uwsgi`` section of ``galaxy.yml`` as described in the `Managing dependencies manually`_
|
||||
section.
|
||||
|
||||
.. code-block:: console
|
||||
Adding additional Galaxy dependencies
|
||||
-------------------------------------
|
||||
|
||||
$ . ./.venv/bin/activate
|
||||
(.venv)$ pip install uwsgi
|
||||
Collecting uwsgi
|
||||
Downloading uwsgi-2.0.12.tar.gz (784kB)
|
||||
100% |████████████████████████████████| 786kB 981kB/s
|
||||
Building wheels for collected packages: uwsgi
|
||||
Running setup.py bdist_wheel for uwsgi
|
||||
Stored in directory: /home/nate/.cache/pip/wheels/a4/7b/7c/8cbe2fe2c2b963173361cc18aa726f165dc4803effbb8195fc
|
||||
Successfully built uwsgi
|
||||
Installing collected packages: uwsgi
|
||||
Successfully installed uwsgi-2.0.12
|
||||
New packages can be added to Galaxy, or the versions of existing packages can be updated, using `pipenv`_ and `Galaxy
|
||||
Starforge`_, Galaxy's Docker-based build system.
|
||||
|
||||
Because uWSGI is installed in the virtualenv, Galaxy's dependencies will be
|
||||
found upon startup.
|
||||
.. note::
|
||||
|
||||
If uWSGI is installed outside of the virtualenv (e.g. from apt) you will need
|
||||
to pass the ``-H`` option (or one of `its many aliases
|
||||
<http://uwsgi-docs.readthedocs.org/en/latest/Options.html#home>`_) on the uWSGI
|
||||
command line:
|
||||
Dependency pinning management is being migrated to pipenv_. As of this release, pinning for packages used for Galaxy
|
||||
development are managed by pipenv_, but pinning for regular runtime packages are still managed with manual changes
|
||||
to ``pinned-requirements.txt``. See `Pull Request #4891`_ for details.
|
||||
|
||||
.. code-block:: console
|
||||
The process is still under development and will be streamlined and automated over time. For the time being, please use
|
||||
the following process to add new packages and have their wheels built:
|
||||
|
||||
$ uwsgi --ini /srv/galaxy/config/uwsgi.ini -H /srv/galaxy/venv
|
||||
1. Install `Starforge`_ (e.g. with ``pip install starforge`` or ``python setup.py install`` from the source). You will
|
||||
also need to have Docker installed on your system.
|
||||
|
||||
Or in the uWSGI config file:
|
||||
2. Obtain `wheels.yml`_ (this file will most likely be moved in to Galaxy in the future) and add/modify the wheel
|
||||
definition.
|
||||
|
||||
.. code-block:: ini
|
||||
3. Use ``starforge wheel --wheels-config=wheels.yml <wheel-name>`` to build the wheel. If the wheel includes C
|
||||
extensions, you will probably want to also use the ``--no-qemu`` flag to prevent Starforge from attempting to build
|
||||
on Mac OS X using QEMU/KVM.
|
||||
|
||||
[uwsgi]
|
||||
processes = 8
|
||||
threads = 4
|
||||
socket = /srv/galaxy/var/uwgi.sock
|
||||
logto = /srv/galaxy/var/uwsgi.log
|
||||
master = True
|
||||
pythonpath = /srv/galaxy/server/lib
|
||||
pythonhome = /srv/galaxy/venv
|
||||
module = galaxy.webapps.galaxy.buildapp:uwsgi_app_factory()
|
||||
set = galaxy_config_file=/srv/galaxy/config/galaxy.ini
|
||||
set = galaxy_root=/srv/galaxy/server
|
||||
4. If the wheel build is successful, submit a pull request to `Starforge`_ with your changes to `wheels.yml`_.
|
||||
|
||||
Supervisor
|
||||
^^^^^^^^^^
|
||||
5. A `Galaxy Committers group`_ member will need to trigger an automated build of the wheel changes in your pull
|
||||
request. Galaxy's Jenkins_ service will build these changes using Starforge.
|
||||
|
||||
Many production sites use `supervisord`_ to manage their Galaxy processes
|
||||
rather than relying on ``run.sh`` or other means. There's no simple way to
|
||||
activate a virtualenv when using supervisor, but you can simulate the effects
|
||||
by setting ``$PATH`` and ``$VIRTUAL_ENV`` in your supervisor config:
|
||||
6. If the pull request is merged, submit a pull request to Galaxy modifying the files in `lib/galaxy/dependencies`_ as
|
||||
appropriate.
|
||||
|
||||
.. code-block:: ini
|
||||
|
||||
[program:galaxy_uwsgi]
|
||||
command = /srv/galaxy/venv/bin/uwsgi --ini /srv/galaxy/config/uwsgi.ini
|
||||
directory = /srv/galaxy/server
|
||||
environment = VIRTUAL_ENV="/srv/galaxy/venv",PATH="/srv/galaxy/venv/bin:%(ENV_PATH)s"
|
||||
numprocs = 1
|
||||
|
||||
[program:galaxy_handler]
|
||||
command = /srv/galaxy/venv/bin/python ./scripts/galaxy-main -c /srv/galaxy/config/galaxy.ini --server-name=handler%(process_num)s
|
||||
directory = /srv/galaxy/server
|
||||
process_name = handler%(process_num)s
|
||||
numprocs = 4
|
||||
environment = VIRTUAL_ENV="/srv/galaxy/venv",PATH="/srv/galaxy/venv/bin:%(ENV_PATH)s"
|
||||
|
||||
With supervisor < 3.0 you cannot use the ``%(ENV_PATH)s`` template variable and
|
||||
must instead specify the full desired ``$PATH``.
|
||||
|
||||
.. _supervisord: http://supervisord.org/
|
||||
|
||||
Custom pip/wheel rationale
|
||||
--------------------------
|
||||
|
||||
We chose to use a modified version of the `pip`_ and `wheel`_ packages in order
|
||||
to make Galaxy easy to use. People wishing to run Galaxy (especially only for
|
||||
tool development) may not be systems or command line experts. Unfortunately,
|
||||
Python modules with C extensions may not always compile out of the box
|
||||
(typically due to missing compilers, headers, or other system packages) and the
|
||||
failure messages generated are typically only decipherable to people
|
||||
experienced with software compilation and almost never indicate how to fix the
|
||||
problem. In addition, the process of compiling all of Galaxy's C extension
|
||||
dependencies can be very long if it does succeed. As a result, we want to
|
||||
precompile Galaxy's dependencies. However, the egg format was never prepared
|
||||
for doing this on any platform and wheels could not do it on Linux because
|
||||
there is no ABI compatibility between Linux distributions or versions.
|
||||
|
||||
As a benefit of using the standard tooling (pip), if you choose not to use
|
||||
Galaxy pip, all of Galaxy's dependencies should still be installable using
|
||||
standard pip. You will still need to point pip at `wheels.galaxyproject.org`_
|
||||
in order to fetch some modified packages and ones that aren't available on
|
||||
PyPI, but this can be done with the unmodified version of pip.
|
||||
|
||||
A good early discussion of these problems can be found in Armin Ronacher's
|
||||
`blog post on wheels <http://lucumr.pocoo.org/2014/1/27/python-on-wheels/>`_.
|
||||
One of the problems Armin discusses, Python interpreter ABI incompatibilites
|
||||
depending on build-time options (UCS2 vs. UCS4), has been fixed by us and
|
||||
accepted into pip >= 8.0 in `pip pull request #3075`_. The other major problem
|
||||
(the non-portability of wheels between Linux distributions) remains. `Galaxy
|
||||
pip`_ provides one solution to this problem.
|
||||
|
||||
More recently, the proposed `PEP 513`_ proposes a different solution to the
|
||||
cross-distro problem. PEP 513 also contains a very detailed technical
|
||||
explanation of the problem.
|
||||
|
||||
.. _PEP 513: https://www.python.org/dev/peps/pep-0513/
|
||||
.. _pip pull request #3075: https://github.com/pypa/pip/pull/3075
|
||||
|
||||
Galaxy pip and wheel
|
||||
--------------------
|
||||
.. _Galaxy pip:
|
||||
.. _Galaxy wheel: `Galaxy pip and wheel`_
|
||||
|
||||
`Galaxy pip is a fork <https://github.com/natefoo/pip/tree/linux-wheels>`_ of
|
||||
`pip`_ in which we have added support for installing wheels containing C
|
||||
extensions (wheels that have compiled binary code) on Linux. `Galaxy wheel is
|
||||
a fork <https://bitbucket.org/natefoo/wheel>`_ of `wheel`_ in which we have
|
||||
added support for building wheels installable with Galaxy pip.
|
||||
|
||||
Two different types of wheels can be created:
|
||||
|
||||
1. "Simple" wheels with very few dependencies outside of libc and libm built on
|
||||
a "suitably old" platform (currently Debian Squeeze) such that they should
|
||||
work on all newer systems (e.g. RHEL 6+, Ubuntu 12.04+). These wheels carry
|
||||
the unmodified ``linux_{arch}`` platform tag (e.g. ``linux_x86_64``) as
|
||||
specified in `PEP 425`_ and that you will find on wheels built with an
|
||||
unmodified `wheel`_.
|
||||
|
||||
2. Wheels with specific external dependencies (for example, ``libpq.so``, the
|
||||
PostgreSQL library, used by `psycopg2`_) can be built on each supported
|
||||
Linux distribution and tagged more specifically for each distribution. These
|
||||
wheels carry a ``linux_{arch}_{distro}_{version}`` platform tag (e.g.
|
||||
``linux_x86_ubuntu_14_04``) and can be created using `Galaxy wheel`_.
|
||||
|
||||
The `manylinux`_ project implements the "Simple" wheels in a more clearly
|
||||
defined way and allows for the inclusion of "non-standard" external
|
||||
dependencies directly into the wheel. Galaxy will officially support any
|
||||
standard which allows for Linux wheels in PyPI once such a standard is
|
||||
complete.
|
||||
|
||||
.. _PEP 425: https://www.python.org/dev/peps/pep-0425/
|
||||
.. _manylinux: https://github.com/manylinux/manylinux/
|
||||
.. _psycopg2: http://initd.org/psycopg/
|
||||
|
||||
Wheel platform compatibility
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Galaxy pip and Galaxy wheel also include support for the proposed
|
||||
`binary-compatibility.cfg`_ file. This file allows distributions that are
|
||||
binary compatibile (e.g. Red Hat Enterprise Linux 6 and CentOS 6) to use the
|
||||
same wheels.
|
||||
|
||||
This is a JSON format file which can be installed in ``/etc/python`` or the
|
||||
root of a virtualenv (`common_startup.sh`_ creates it here) and provides a
|
||||
mapping between `PEP 425`_ platform tags. For example, the following
|
||||
``binary-compatibility.cfg`` indicates that wheels built on the platform
|
||||
``linux_x86_centos_6_7`` will have their platform tag overridden to
|
||||
``linux_x86_rhel_6``. In addition, wheels tagged with ``linux_x86_64_rhel_6_7``
|
||||
and ``linux_x86_64_rhel_6`` will be installable on a ``linux_x86_centos_6_7``
|
||||
system:
|
||||
|
||||
.. code-block:: json
|
||||
|
||||
{
|
||||
"linux_x86_64_centos_6_7": {
|
||||
"build": "linux_x86_64_rhel_6",
|
||||
"install": ["linux_x86_64_rhel_6_7", "linux_x86_64_rhel_6"]
|
||||
}
|
||||
}
|
||||
|
||||
Currently, Scientific Linux, CentOS, and Red Hat Enterprise Linux will be set
|
||||
as binary compatible by `common_startup.sh`_.
|
||||
|
||||
.. _binary-compatibility.cfg: https://mail.python.org/pipermail/distutils-sig/2015-July/026617.html
|
||||
|
||||
Adding additional wheels as Galaxy dependencies
|
||||
-----------------------------------------------
|
||||
|
||||
New wheels can be added to Galaxy, or the versions of existing wheels can be
|
||||
updated, using `Galaxy Starforge`_, Galaxy's Docker-based build system.
|
||||
|
||||
The process is still under development and will be streamlined and automated
|
||||
over time. For the time being, please use the following process to add new
|
||||
wheels:
|
||||
|
||||
1. Install `Starforge`_ (e.g. with ``pip install starforge`` or ``python
|
||||
setup.py install`` from the source). You will also need to have Docker
|
||||
installed on your system.
|
||||
|
||||
2. Obtain `wheels.yml`_ (this file will most likely be moved in to Galaxy in
|
||||
the future) and add/modify the wheel definition.
|
||||
|
||||
3. Use ``starforge wheel --wheels-config=wheels.yml <wheel-name>`` to build the
|
||||
wheel. If the wheel includes C extensions, you will probably want to also
|
||||
use the ``--no-qemu`` flag to prevent Starforge from attempting to build on
|
||||
Mac OS X using QEMU/KVM.
|
||||
|
||||
4. If the wheel build is successful, submit a pull request to `Starforge`_ with
|
||||
your changes to `wheels.yml`_.
|
||||
|
||||
5. A `Galaxy Committers group`_ member will need to trigger an automated build
|
||||
of the wheel changes in your pull request. Galaxy's Jenkins_ service will
|
||||
build these changes using Starforge.
|
||||
|
||||
6. If the pull request is merged, submit a pull request to Galaxy modifying the
|
||||
files in `lib/galaxy/dependencies`_ as appropriate.
|
||||
|
||||
You may attempt to skip directly to step 4 and let the Starforge wheel PR
|
||||
builder build your wheels for you. This is especially useful if you are simply
|
||||
updating an existing wheel's version. However, if you are adding a new C
|
||||
extension wheel that is not simple to build, you may need to go through many
|
||||
iterations of updating the PR and having a `Galaxy Committers group`_ member
|
||||
triggering builds before wheels are successfully built. You can avoid this
|
||||
cycle by performing steps 1-3 locally.
|
||||
You may attempt to skip directly to step 4 and let the Starforge wheel PR builder build your wheels for you. This is
|
||||
especially useful if you are simply updating an existing wheel's version. However, if you are adding a new C extension
|
||||
wheel that is not simple to build, you may need to go through many iterations of updating the PR and having a `Galaxy
|
||||
Committers group`_ member triggering builds before wheels are successfully built. You can avoid this cycle by performing
|
||||
steps 1-3 locally.
|
||||
|
||||
.. _pipenv: http://pipenv.readthedocs.io/
|
||||
.. _Starforge:
|
||||
.. _Galaxy Starforge: https://github.com/galaxyproject/starforge/
|
||||
.. _Pull Request #4891: https://github.com/galaxyproject/galaxy/pull/4891
|
||||
.. _wheels.yml: https://github.com/galaxyproject/starforge/blob/master/wheels/build/wheels.yml
|
||||
.. _Galaxy Committers group: https://github.com/galaxyproject/galaxy/blob/dev/doc/source/project/organization.rst#committers
|
||||
.. _Jenkins: https://jenkins.galaxyproject.org/
|
||||
|
||||
@@ -510,7 +510,7 @@ If using the **uWSGI + Webless** scenario, you'll need to addtionally define job
|
||||
|
||||
```ini
|
||||
[program:handler]
|
||||
command = python ./scripts/galaxy-main -c /srv/galaxy/config/galaxy.yml --server-name=handler%(process_num)s --pid-file=/srv/galaxy/var/handler%(process_num)s.pid --log-file=/srv/galaxy/log/handler%(process_num)s.log
|
||||
command = /srv/galaxy/venv/bin/python ./scripts/galaxy-main -c /srv/galaxy/config/galaxy.yml --server-name=handler%(process_num)s --pid-file=/srv/galaxy/var/handler%(process_num)s.pid --log-file=/srv/galaxy/log/handler%(process_num)s.log
|
||||
directory = /srv/galaxy/server
|
||||
process_name = handler%(process_num)s
|
||||
numprocs = 3
|
||||
@@ -519,6 +519,7 @@ autostart = true
|
||||
autorestart = true
|
||||
startsecs = 15
|
||||
user = galaxy
|
||||
environment = VIRTUAL_ENV="/srv/galaxy/venv",PATH="/srv/galaxy/venv/bin:%(ENV_PATH)s"
|
||||
```
|
||||
|
||||
This is similar to the "web" definition above, however, you'll notice that we use `%(process_num)s`. That's a variable
|
||||
|
||||
@@ -64,21 +64,17 @@ Each plugin has the following folder structure:
|
||||
.. code-block::
|
||||
|
||||
- plugin_name
|
||||
- config
|
||||
- plugin_name.yaml (mandatory)
|
||||
- helper
|
||||
- __init__.py (optional)
|
||||
- static
|
||||
- script.js (optional)
|
||||
- styles.css (optional)
|
||||
- config.yml (mandatory)
|
||||
- __init__.py (optional)
|
||||
- script.js (optional)
|
||||
- styles.css (optional)
|
||||
|
||||
|
||||
config
|
||||
------
|
||||
config.yml
|
||||
----------
|
||||
|
||||
The configuration file is just a .yml (or .yaml) file with a few options. The following options are mandatory:
|
||||
|
||||
- **name** - must be the same as the plugin's root directory name
|
||||
- **id** - must be the same as the plugin's root directory name
|
||||
- **type** (see Entry points) - can be combined with others
|
||||
- **activate** - *true* or *false* - whether show the plugin on a page or not
|
||||
- **icon** Icon to show (if masthead)
|
||||
@@ -88,8 +84,8 @@ The configuration file is just a .yml (or .yaml) file with a few options. The fo
|
||||
All other options can be anything used by the plugin and accessed later via *webhook.config['...']*.
|
||||
|
||||
|
||||
helper/__init__.py
|
||||
------------------
|
||||
__init__.py
|
||||
-----------
|
||||
|
||||
*__init__.py has* to have the **main()** function with the following (or similar) structure:
|
||||
|
||||
@@ -116,10 +112,8 @@ helper/__init__.py
|
||||
As an example please take a look at the *phdcomics* example plugin: https://github.com/galaxyproject/galaxy/blob/release_17.05/test/functional/webhooks/phdcomics/helper/__init__.py
|
||||
|
||||
|
||||
static
|
||||
------
|
||||
|
||||
The *static* folder contains only two files with the specified above names (otherwise, they won’t be read on Galaxy run).
|
||||
static files
|
||||
------------
|
||||
|
||||
- script.js - all JavaScript code (with all third-party dependencies) must be here
|
||||
- styles.css - all CSS styles, used by the plugin
|
||||
@@ -149,41 +143,3 @@ tool/workflow
|
||||
If a tool or a workflow plugin has script.js and/or styles.css, the content of these files will be read as two strings and sent to the client and appended to DOM’s <head>.
|
||||
|
||||
Such approach is a possible bottleneck if the two files are big (however, this shouldn’t ever happen because plugins are supposed to be small and simple).
|
||||
|
||||
masthead
|
||||
--------
|
||||
|
||||
Topbar buttons are hard coded, so they’re rendered only after *make client*.
|
||||
|
||||
The plugin system is entirely dynamic. All plugins are detected during Galaxy load and their configs and statics are being saved. So, every plugin must be shown/rendered dynamically.
|
||||
|
||||
I found a not very optimal way to add buttons to the topbar (masthead):
|
||||
|
||||
.. code-block:: javascript
|
||||
|
||||
$(document).ready(function() {
|
||||
Galaxy.page.masthead.collection.add({
|
||||
id : ... ,
|
||||
icon : ... ,
|
||||
url : ... ,
|
||||
tooltip : ... ,
|
||||
onlick : function() { ... }
|
||||
});
|
||||
});
|
||||
|
||||
history-menu
|
||||
------------
|
||||
|
||||
History Panel items are again hard coded, but in the current implementation they’re rendered as html elements (so, they’re not even stored in a collection or any other object).
|
||||
|
||||
To add new menu items, I do the following:
|
||||
|
||||
.. code-block:: javascript
|
||||
|
||||
menu.push({
|
||||
html : _l( ... ),
|
||||
anon : true,
|
||||
func : function() { ... }
|
||||
});
|
||||
|
||||
But in order to fetch all plugin menu items before rendering, I get them via API in a synchronous manner. The problem is that History Panel now may load a bit longer.
|
||||
|
||||
Executable
+29
@@ -0,0 +1,29 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Conda does not have all of Galaxy's dependencies, and the list of which ones it does is always changing. As a result,
|
||||
# it's not possible to keep a conda requirements file up to date. However, with Conda >= 4.4, we can use Conda itself to
|
||||
# determine which dependencies it can install. Pip will be used (upon Galaxy startup) to install the rest.
|
||||
#
|
||||
# You should use this script like so:
|
||||
#
|
||||
# conda create -n <env> --file <(lib/galaxy/dependencies/conda-file.sh)
|
||||
#
|
||||
# Ensure you have enabled the bioconda and conda-forge repositories first!:
|
||||
#
|
||||
# conda config --add channels conda-forge
|
||||
# conda config --add channels bioconda
|
||||
|
||||
here=$(dirname $0)
|
||||
|
||||
if ! command -v conda >/dev/null; then
|
||||
printf "$0: command not found: conda\n" >&2
|
||||
printf "hint: did you run 'conda activate base'?\n" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf "Filtering out Galaxy requirements not available from Conda:" >&2
|
||||
egrep -iv $( \
|
||||
conda create -n _gx_test_env --dry-run --file <(sed 's/;.*//' $here/pinned-requirements.txt) python=2.7 2>&1 \
|
||||
| grep '^\s*-' | grep -v https: | awk '{print $NF}' | paste -s -d'|' \
|
||||
) $here/pinned-requirements.txt | sed 's/;.*//'
|
||||
printf " done\n" >&2
|
||||
@@ -3,7 +3,6 @@ API Controller providing Galaxy Webhooks
|
||||
"""
|
||||
import imp
|
||||
import logging
|
||||
import random
|
||||
|
||||
from galaxy.web import _future_expose_api_anonymous_and_sessionless as \
|
||||
expose_api_anonymous_and_sessionless
|
||||
@@ -17,7 +16,7 @@ class WebhooksController(BaseAPIController):
|
||||
super(WebhooksController, self).__init__(app)
|
||||
|
||||
@expose_api_anonymous_and_sessionless
|
||||
def get_all(self, trans, **kwd):
|
||||
def all_webhooks(self, trans, **kwd):
|
||||
"""
|
||||
*GET /api/webhooks/
|
||||
Returns all webhooks
|
||||
@@ -28,35 +27,9 @@ class WebhooksController(BaseAPIController):
|
||||
]
|
||||
|
||||
@expose_api_anonymous_and_sessionless
|
||||
def get_random(self, trans, webhook_type, **kwd):
|
||||
def webhook_data(self, trans, webhook_id, **kwd):
|
||||
"""
|
||||
*GET /api/webhooks/{webhook_type}
|
||||
Returns a random webhook for a given type
|
||||
"""
|
||||
webhooks = [
|
||||
webhook
|
||||
for webhook in self.app.webhooks_registry.webhooks
|
||||
if webhook_type in webhook.type and
|
||||
webhook.activate is True
|
||||
]
|
||||
return random.choice(webhooks).to_dict() if webhooks else {}
|
||||
|
||||
@expose_api_anonymous_and_sessionless
|
||||
def get_all_by_type(self, trans, webhook_type, **kwd):
|
||||
"""
|
||||
*GET /api/webhooks/{webhook_type}/all
|
||||
Returns all webhooks for a given type
|
||||
"""
|
||||
return [
|
||||
webhook.to_dict()
|
||||
for webhook in self.app.webhooks_registry.webhooks
|
||||
if webhook_type in webhook.type
|
||||
]
|
||||
|
||||
@expose_api_anonymous_and_sessionless
|
||||
def get_data(self, trans, webhook_name, **kwd):
|
||||
"""
|
||||
*GET /api/webhooks/{webhook_name}/get_data/{params}
|
||||
*GET /api/webhooks/{webhook_id}/data/{params}
|
||||
Returns the result of executing helper function
|
||||
"""
|
||||
params = {}
|
||||
@@ -64,14 +37,12 @@ class WebhooksController(BaseAPIController):
|
||||
for key, value in kwd.items():
|
||||
params[key] = value
|
||||
|
||||
webhook = [
|
||||
webhook = (
|
||||
webhook
|
||||
for webhook in self.app.webhooks_registry.webhooks
|
||||
if webhook.name == webhook_name
|
||||
]
|
||||
if webhook.id == webhook_id
|
||||
).next()
|
||||
|
||||
return imp.load_source('helper', webhook[0].helper).main(
|
||||
trans,
|
||||
webhook[0],
|
||||
params,
|
||||
) if webhook and webhook[0].helper != '' else {}
|
||||
return imp.load_source(webhook.path, webhook.helper).main(
|
||||
trans, webhook, params,
|
||||
) if webhook and webhook.helper != '' else {}
|
||||
|
||||
@@ -609,29 +609,17 @@ def populate_api_routes(webapp, app):
|
||||
# ===== WEBHOOKS API =====
|
||||
# ========================
|
||||
|
||||
webapp.mapper.connect('get_all',
|
||||
webapp.mapper.connect('get_all_webhooks',
|
||||
'/api/webhooks',
|
||||
controller='webhooks',
|
||||
action='get_all',
|
||||
conditions=dict(method=["GET"]))
|
||||
action='all_webhooks',
|
||||
conditions=dict(method=['GET']))
|
||||
|
||||
webapp.mapper.connect('get_random',
|
||||
'/api/webhooks/{webhook_type}',
|
||||
webapp.mapper.connect('get_webhook_data',
|
||||
'/api/webhooks/{webhook_id}/data',
|
||||
controller='webhooks',
|
||||
action='get_random',
|
||||
conditions=dict(method=["GET"]))
|
||||
|
||||
webapp.mapper.connect('get_all_by_type',
|
||||
'/api/webhooks/{webhook_type}/all',
|
||||
controller='webhooks',
|
||||
action='get_all_by_type',
|
||||
conditions=dict(method=["GET"]))
|
||||
|
||||
webapp.mapper.connect('get_data',
|
||||
'/api/webhooks/{webhook_name}/get_data',
|
||||
controller='webhooks',
|
||||
action='get_data',
|
||||
conditions=dict(method=["GET"]))
|
||||
action='webhook_data',
|
||||
conditions=dict(method=['GET']))
|
||||
|
||||
# =======================
|
||||
# ===== LIBRARY API =====
|
||||
|
||||
@@ -12,11 +12,12 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Webhook(object):
|
||||
def __init__(self, w_name, w_type, w_activate, w_path):
|
||||
self.name = w_name
|
||||
self.type = w_type
|
||||
self.activate = w_activate
|
||||
self.path = w_path
|
||||
def __init__(self, id, type, activate, weight, path):
|
||||
self.id = id
|
||||
self.type = type
|
||||
self.activate = activate
|
||||
self.weight = weight
|
||||
self.path = path
|
||||
self.styles = ''
|
||||
self.script = ''
|
||||
self.helper = ''
|
||||
@@ -24,12 +25,13 @@ class Webhook(object):
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
'name': self.name,
|
||||
'id': self.id,
|
||||
'type': self.type,
|
||||
'activate': self.activate,
|
||||
'weight': self.weight,
|
||||
'styles': self.styles,
|
||||
'script': self.script,
|
||||
'config': self.config
|
||||
'config': self.config,
|
||||
}
|
||||
|
||||
|
||||
@@ -48,56 +50,57 @@ class WebhooksRegistry(object):
|
||||
|
||||
def load_webhooks(self):
|
||||
for directory in self.webhooks_directories:
|
||||
config_dir = os.path.join(directory, 'config')
|
||||
config_file_path = None
|
||||
for config_file in ['config.yml', 'config.yaml']:
|
||||
path = os.path.join(directory, config_file)
|
||||
if os.path.isfile(path):
|
||||
config_file_path = path
|
||||
break
|
||||
|
||||
if not os.path.exists(config_dir):
|
||||
log.warning('directory not found: %s', config_dir)
|
||||
continue
|
||||
if config_file_path:
|
||||
try:
|
||||
self.load_webhook_from_config(directory, config_file_path)
|
||||
except Exception as e:
|
||||
log.exception(e)
|
||||
|
||||
config_dir_contents = os.listdir(config_dir)
|
||||
# We are assuming that all yml/yaml files in a webhooks'
|
||||
# config directory are webhook config files.
|
||||
for config_file in config_dir_contents:
|
||||
if config_file.endswith('.yml') or config_file.endswith('.yaml'):
|
||||
self.load_webhook_from_config(config_dir, config_file)
|
||||
def load_webhook_from_config(self, webhook_dir, config_file_path):
|
||||
with open(config_file_path) as file:
|
||||
config = yaml.safe_load(file)
|
||||
|
||||
def load_webhook_from_config(self, config_dir, config_file):
|
||||
weight = config.get('weight', 1)
|
||||
if weight < 1:
|
||||
raise ValueError('Webhook weight must be greater or equal 1.')
|
||||
|
||||
webhook = Webhook(
|
||||
config.get('id'),
|
||||
config.get('type'),
|
||||
config.get('activate', False),
|
||||
weight,
|
||||
webhook_dir,
|
||||
)
|
||||
|
||||
# Read styles into a string, assuming all styles are in a
|
||||
# single file
|
||||
try:
|
||||
with open(os.path.join(config_dir, config_file)) as file:
|
||||
config = yaml.safe_load(file)
|
||||
path = os.path.normpath(os.path.join(config_dir, '..'))
|
||||
webhook = Webhook(
|
||||
config['name'],
|
||||
config['type'],
|
||||
config['activate'],
|
||||
path,
|
||||
)
|
||||
styles_file = os.path.join(webhook_dir, 'styles.css')
|
||||
with open(styles_file, 'r') as file:
|
||||
webhook.styles = file.read().replace('\n', '')
|
||||
except IOError:
|
||||
pass
|
||||
|
||||
# Read styles into a string, assuming all styles are in a
|
||||
# single file
|
||||
try:
|
||||
styles_file = os.path.join(path, 'static/styles.css')
|
||||
with open(styles_file, 'r') as file:
|
||||
webhook.styles = file.read().replace('\n', '')
|
||||
except IOError:
|
||||
pass
|
||||
# Read script into a string, assuming everything is in a
|
||||
# single file
|
||||
try:
|
||||
script_file = os.path.join(webhook_dir, 'script.js')
|
||||
with open(script_file, 'r') as file:
|
||||
webhook.script = file.read()
|
||||
except IOError:
|
||||
pass
|
||||
|
||||
# Read script into a string, assuming everything is in a
|
||||
# single file
|
||||
try:
|
||||
script_file = os.path.join(path, 'static/script.js')
|
||||
with open(script_file, 'r') as file:
|
||||
webhook.script = file.read()
|
||||
except IOError:
|
||||
pass
|
||||
# Save helper function path if it exists
|
||||
helper_path = os.path.join(webhook_dir, '__init__.py')
|
||||
if os.path.isfile(helper_path):
|
||||
webhook.helper = helper_path
|
||||
|
||||
# Save helper function path if it exists
|
||||
helper_path = os.path.join(path, 'helper/__init__.py')
|
||||
if os.path.isfile(helper_path):
|
||||
webhook.helper = helper_path
|
||||
|
||||
webhook.config = config
|
||||
self.webhooks.append(webhook)
|
||||
|
||||
except Exception as e:
|
||||
log.exception(e)
|
||||
webhook.config = config
|
||||
self.webhooks.append(webhook)
|
||||
|
||||
@@ -9,6 +9,8 @@ done
|
||||
# Conda Python is in use, do not use virtualenv
|
||||
if python -V 2>&1 | grep -q -e 'Anaconda' -e 'Continuum Analytics' ; then
|
||||
CONDA_ALREADY_INSTALLED=1
|
||||
elif python -c 'import sys; print(sys.version.replace("\n", " "))' | grep -q -e 'packaged by conda-forge' ; then
|
||||
CONDA_ALREADY_INSTALLED=1
|
||||
else
|
||||
CONDA_ALREADY_INSTALLED=0
|
||||
fi
|
||||
|
||||
@@ -14,28 +14,13 @@ class WebhooksApiTestCase(api.ApiTestCase):
|
||||
|
||||
self._assert_status_code_is(response, 200)
|
||||
webhook_objs = self._assert_are_webhooks(response)
|
||||
names = self._get_webhook_names(webhook_objs)
|
||||
for expected_name in ["history_test1", "history_test2", "masthead_test", "phdcomics", "trans_object", "xkcd"]:
|
||||
assert expected_name in names
|
||||
|
||||
def test_get_random(self):
|
||||
response = self._get('webhooks/tool')
|
||||
self._assert_status_code_is(response, 200)
|
||||
self._assert_is_webhook(response.json())
|
||||
|
||||
def test_get_all_by_type(self):
|
||||
# Ensure tool type filtering include a valid webhook of type tool and excludes a webhook
|
||||
# that isn't of type tool.
|
||||
response = self._get('webhooks/tool/all')
|
||||
|
||||
self._assert_status_code_is(response, 200)
|
||||
webhook_objs = self._assert_are_webhooks(response)
|
||||
names = self._get_webhook_names(webhook_objs)
|
||||
assert "phdcomics" in names
|
||||
assert "trans_object" not in names # properly filtered out by type
|
||||
ids = self._get_webhook_ids(webhook_objs)
|
||||
for expected_id in ['history_test1', 'history_test2', 'masthead_test',
|
||||
'phdcomics', 'trans_object', 'xkcd']:
|
||||
assert expected_id in ids
|
||||
|
||||
def test_get_data(self):
|
||||
response = self._get('webhooks/trans_object/get_data')
|
||||
response = self._get('webhooks/trans_object/data')
|
||||
self._assert_status_code_is(response, 200)
|
||||
self._assert_has_keys(response.json(), 'username')
|
||||
|
||||
@@ -48,8 +33,9 @@ class WebhooksApiTestCase(api.ApiTestCase):
|
||||
|
||||
def _assert_is_webhook(self, obj):
|
||||
assert isinstance(obj, dict)
|
||||
self._assert_has_keys(obj, 'styles', 'activate', 'name', 'script', 'type', 'config')
|
||||
self._assert_has_keys(obj,
|
||||
'id', 'type', 'activate', 'weight', 'script', 'styles', 'config')
|
||||
|
||||
def _get_webhook_names(self, webhook_objs):
|
||||
names = [w.get("name") for w in webhook_objs]
|
||||
def _get_webhook_ids(self, webhook_objs):
|
||||
names = [w.get('id') for w in webhook_objs]
|
||||
return names
|
||||
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
name: history_test1
|
||||
id: history_test1
|
||||
title: History Menu Webhook Item 1
|
||||
type:
|
||||
type:
|
||||
- history-menu
|
||||
activate: true
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
name: history_test2
|
||||
id: history_test2
|
||||
title: History Menu Webhook Item 2
|
||||
type:
|
||||
type:
|
||||
- history-menu
|
||||
activate: true
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
name: masthead_test
|
||||
type:
|
||||
id: masthead_test
|
||||
type:
|
||||
- masthead
|
||||
activate: true
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
name: phdcomics
|
||||
id: phdcomics
|
||||
type:
|
||||
- tool
|
||||
- workflow
|
||||
+1
-1
@@ -32,7 +32,7 @@ $(document).ready(function() {
|
||||
|
||||
getRandomComic: function() {
|
||||
var me = this,
|
||||
url = galaxyRoot + 'api/webhooks/phdcomics/get_data';
|
||||
url = galaxyRoot + 'api/webhooks/phdcomics/data';
|
||||
|
||||
this.$comicImg.html($('<div/>', {
|
||||
id: 'phdcomics-loader'
|
||||
+2
-2
@@ -1,4 +1,4 @@
|
||||
name: trans_object
|
||||
id: trans_object
|
||||
type:
|
||||
- masthead
|
||||
activate: true
|
||||
@@ -7,6 +7,6 @@ icon: fa-user
|
||||
tooltip: Show Username
|
||||
|
||||
function: >
|
||||
$.getJSON(Galaxy.root + "api/webhooks/trans_object/get_data", function(data) {
|
||||
$.getJSON(Galaxy.root + "api/webhooks/trans_object/data", function(data) {
|
||||
alert('Username: ' + data.username);
|
||||
});
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
name: xkcd
|
||||
id: xkcd
|
||||
type:
|
||||
- tool
|
||||
- workflow
|
||||
@@ -72,9 +72,9 @@ class ToolDescribingToursTestCase(SeleniumTestCase):
|
||||
|
||||
def _ensure_tdt_available(self):
|
||||
""" Skip a test if the webhook TDT doesn't appear. """
|
||||
response = self.api_get('webhooks/tool-menu/all', raw=True)
|
||||
response = self.api_get('webhooks', raw=True)
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = response.json()
|
||||
webhooks = [x['name'] for x in data]
|
||||
webhooks = [x['id'] for x in data]
|
||||
if 'tour_generator' not in webhooks:
|
||||
raise unittest.SkipTest('Skipping test, webhook "Tool-Describing-Tours" doesn\'t appear to be configured.')
|
||||
|
||||
Reference in New Issue
Block a user