Merge pull request #5342 from dannon/integration.webhooks_improvements

[18.01] Webhooks improvements
This commit is contained in:
Martin Cech
2018-02-01 15:14:47 -05:00
committed by GitHub
38 changed files with 220 additions and 288 deletions
+4 -2
View File
@@ -1,7 +1,7 @@
import Utils from "utils/utils";
import Menu from "layout/menu";
import Scratchbook from "layout/scratchbook";
import QuotaMeter from "mvc/user/user-quotameter";
/** Masthead **/
var View = Backbone.View.extend({
initialize: function(options) {
@@ -62,7 +62,9 @@ var View = Backbone.View.extend({
var text = "";
self.collection.each(model => {
var q = model.get("onbeforeunload") && model.get("onbeforeunload")();
q && (text += `${q} `);
if (q) {
text += `${q} `;
}
});
if (text !== "") {
return text;
+4 -4
View File
@@ -105,15 +105,15 @@ var Collection = Backbone.Collection.extend({
//
// Webhooks
//
Webhooks.add({
url: "api/webhooks/masthead/all",
Webhooks.load({
type: "masthead",
callback: function(webhooks) {
$(document).ready(() => {
$.each(webhooks.models, (index, model) => {
webhooks.each(model => {
var webhook = model.toJSON();
if (webhook.activate) {
var obj = {
id: webhook.name,
id: webhook.id,
icon: webhook.config.icon,
url: webhook.config.url,
tooltip: webhook.config.tooltip,
+3 -1
View File
@@ -2,6 +2,7 @@ import Masthead from "layout/masthead";
import Panel from "layout/panel";
import Modal from "mvc/ui/ui-modal";
import Utils from "utils/utils";
var View = Backbone.View.extend({
el: "body",
className: "full-content",
@@ -61,11 +62,12 @@ var View = Backbone.View.extend({
this.render();
// start the router
this.router &&
if (this.router) {
Backbone.history.start({
root: Galaxy.root,
pushState: true
});
}
},
render: function() {
@@ -175,13 +175,13 @@ var menu = [
];
// Webhooks
Webhooks.add({
url: "api/webhooks/history-menu/all",
Webhooks.load({
type: "history-menu",
async: false, // (hypothetically) slows down the performance
callback: function(webhooks) {
var webhooks_menu = [];
$.each(webhooks.models, (index, model) => {
webhooks.each(model => {
var webhook = model.toJSON();
if (webhook.activate) {
webhooks_menu.push({
@@ -6,6 +6,7 @@ import Utils from "utils/utils";
import Deferred from "utils/deferred";
import Ui from "mvc/ui/ui-misc";
import FormBase from "mvc/form/form-view";
import Webhooks from "mvc/webhooks";
import Citations from "components/Citations.vue";
import Vue from "vue";
export default FormBase.extend({
@@ -200,19 +201,23 @@ export default FormBase.extend({
}
// add tool menu webhooks
$.getJSON("/api/webhooks/tool-menu/all", webhooks => {
_.each(webhooks, webhook => {
if (webhook.activate && webhook.config.function) {
menu_button.addMenu({
icon: webhook.config.icon,
title: webhook.config.title,
onclick: function() {
var func = new Function("options", webhook.config.function);
func(options);
}
});
}
});
Webhooks.load({
type: "tool-menu",
callback: function(webhooks) {
webhooks.each(model => {
var webhook = model.toJSON();
if (webhook.activate && webhook.config.function) {
menu_button.addMenu({
icon: webhook.config.icon,
title: webhook.config.title,
onclick: function() {
var func = new Function("options", webhook.config.function);
func(options);
}
});
}
});
}
});
return {
@@ -598,7 +598,7 @@ var View = Backbone.View.extend({
if ($.isArray(response) && response.length > 0) {
self.$el.append($("<div/>", { id: "webhook-view" }));
var WebhookApp = new Webhooks.WebhookView({
urlRoot: `${Galaxy.root}api/webhooks/workflow`,
type: "workflow",
toolId: job_def.tool_id,
toolVersion: job_def.tool_version
});
+1 -1
View File
@@ -227,7 +227,7 @@ var View = Backbone.View.extend({
if (response.jobs && response.jobs.length > 0) {
self.$el.append($("<div/>", { id: "webhook-view" }));
var WebhookApp = new Webhooks.WebhookView({
urlRoot: `${Galaxy.root}api/webhooks/tool`,
type: "tool",
toolId: job_def.tool_id
});
}
+60 -40
View File
@@ -1,66 +1,86 @@
/**
Webhooks
**/
import Utils from "utils/utils";
var WebhookModel = Backbone.Model.extend({
defaults: {
activate: false
const Webhooks = Backbone.Collection.extend({
url: function() {
return `${Galaxy.root}api/webhooks`;
}
});
var Webhooks = Backbone.Collection.extend({
model: WebhookModel
});
var WebhookView = Backbone.View.extend({
const WebhookView = Backbone.View.extend({
el: "#webhook-view",
initialize: function(options) {
var me = this;
var toolId = options.toolId || "";
var toolVersion = options.toolVersion || "";
const toolId = options.toolId || "";
const toolVersion = options.toolVersion || "";
this.$el.attr("tool_id", toolId);
this.$el.attr("tool_version", toolVersion);
this.model = new WebhookModel();
this.model.urlRoot = options.urlRoot;
this.model.fetch({
success: function() {
me.render();
const webhooks = new Webhooks();
webhooks.fetch({
success: data => {
if (options.type) {
data.reset(filterType(data, options.type));
}
if (data.length > 0) {
this.render(weightedRandomPick(data));
}
}
});
},
render: function() {
var webhook = this.model.toJSON();
this.$el.html(`<div id="${webhook.name}"></div>`);
if (webhook.styles)
$("<style/>", { type: "text/css" })
.text(webhook.styles)
.appendTo("head");
if (webhook.script)
$("<script/>", { type: "text/javascript" })
.text(webhook.script)
.appendTo("head");
render: function(model) {
const webhook = model.toJSON();
this.$el.html(`<div id="${webhook.id}"></div>`);
Utils.appendScriptStyle(webhook);
return this;
}
});
var add = options => {
var webhooks = new Webhooks();
webhooks.url = Galaxy.root + options.url;
const load = options => {
const webhooks = new Webhooks();
webhooks.fetch({
async: options.async ? options.async : true,
success: options.callback
async: options.async !== undefined ? options.async : true,
success: data => {
if (options.type) {
data.reset(filterType(data, options.type));
}
options.callback(data);
}
});
};
function filterType(data, type) {
return data.models.filter(item => {
let itype = item.get("type");
if (itype) {
return itype.indexOf(type) !== -1;
} else {
return false;
}
});
}
function weightedRandomPick(data) {
const weights = data.pluck("weight");
const sum = weights.reduce((a, b) => a + b);
const normalizedWeightsMap = new Map();
weights.forEach((weight, index) => {
normalizedWeightsMap.set(index, parseFloat((weight / sum).toFixed(2)));
});
const table = [];
for (const [index, weight] of normalizedWeightsMap) {
for (let i = 0; i < weight * 100; i++) {
table.push(index);
}
}
return data.at(table[Math.floor(Math.random() * table.length)]);
}
export default {
Webhooks: Webhooks,
WebhookView: WebhookView,
add: add
load: load
};
+12 -14
View File
@@ -25,6 +25,8 @@ window.make_popup_menus = Popupmenu.make_popup_menus;
import init_tag_click_function from "ui/autocom_tagging";
window.init_tag_click_function = init_tag_click_function;
import Tours from "mvc/tours";
import Webhooks from "mvc/webhooks";
import Utils from "utils/utils";
// console.debug( 'galaxy globals loaded' );
// ============================================================================
@@ -176,23 +178,19 @@ $(document).ready(() => {
Tours.activeGalaxyTourRunner();
function onloadWebhooks() {
// Wait until Galaxy.config is loaded.
if (Galaxy.config) {
if (Galaxy.config.enable_webhooks) {
// Load all webhooks with the type 'onload'
$.getJSON(`${Galaxy.root}api/webhooks/onload/all`, webhooks => {
_.each(webhooks, webhook => {
if (Galaxy.root !== undefined) {
// Load all webhooks with the type 'onload'
Webhooks.load({
type: "onload",
callback: function(webhooks) {
webhooks.each(model => {
var webhook = model.toJSON();
if (webhook.activate && webhook.script) {
$("<script/>", { type: "text/javascript" })
.text(webhook.script)
.appendTo("head");
$("<style/>", { type: "text/css" })
.text(webhook.styles)
.appendTo("head");
Utils.appendScriptStyle(webhook);
}
});
});
}
}
});
} else {
setTimeout(onloadWebhooks, 100);
}
+2
View File
@@ -1,4 +1,6 @@
/** Creates a generic/global Galaxy environment, loads shared libraries and a fake server */
/* global define */
define(
[
"jquery",
@@ -1,8 +1,7 @@
/* global define */
/* global QUnit */
import testApp from "qunit/test-app";
import Page from "layout/page";
import Panel from "layout/panel";
QUnit.module("Page test", {
beforeEach: function() {
@@ -1,4 +1,4 @@
name: searchover
id: searchover
type:
- masthead
activate: true
@@ -38,7 +38,7 @@ $(document).ready(function() {
e.stopPropagation();
if ( $( '.search-screen-overlay' ).is( ':visible' ) ){
self.removeOverlay();
}
}
else {
self.clearSearchResults();
self.showOverlay();
@@ -1,4 +1,4 @@
name: tool_list
id: tool_list
type:
- masthead
activate: true
@@ -13,7 +13,7 @@ function: >
'be patient, this might take a moment.');
}, 1);
$.getJSON(Galaxy.root + "api/webhooks/tool_list/get_data", function(data) {
$.getJSON(Galaxy.root + "api/webhooks/tool_list/data", function(data) {
var popup = window.open('tool_list.html');
var html = '<!DOCTYPE html><body><h2>' +
'Create a Docker flavour of this instance:</h2>' +
@@ -1,4 +1,4 @@
name: tour_generator
id: tour_generator
type:
- onload
- tool-menu
@@ -10,7 +10,7 @@ $(document).ready(function() {
$('#execute').attr('tour_id', 'execute');
Toastr.info('Tour generation might take some time.');
$.getJSON('/api/webhooks/tour_generator/get_data/', {
$.getJSON('/api/webhooks/tour_generator/data/', {
tool_id: me.toolId,
tool_version: me.toolVersion
}, function(obj) {
+11 -55
View File
@@ -64,21 +64,17 @@ Each plugin has the following folder structure:
.. code-block::
- plugin_name
- config
- plugin_name.yaml (mandatory)
- helper
- __init__.py (optional)
- static
- script.js (optional)
- styles.css (optional)
- config.yml (mandatory)
- __init__.py (optional)
- script.js (optional)
- styles.css (optional)
config
------
config.yml
----------
The configuration file is just a .yml (or .yaml) file with a few options. The following options are mandatory:
- **name** - must be the same as the plugin's root directory name
- **id** - must be the same as the plugin's root directory name
- **type** (see Entry points) - can be combined with others
- **activate** - *true* or *false* - whether show the plugin on a page or not
- **icon** Icon to show (if masthead)
@@ -88,8 +84,8 @@ The configuration file is just a .yml (or .yaml) file with a few options. The fo
All other options can be anything used by the plugin and accessed later via *webhook.config['...']*.
helper/__init__.py
------------------
__init__.py
-----------
*__init__.py has* to have the **main()** function with the following (or similar) structure:
@@ -116,10 +112,8 @@ helper/__init__.py
As an example please take a look at the *phdcomics* example plugin: https://github.com/galaxyproject/galaxy/blob/release_17.05/test/functional/webhooks/phdcomics/helper/__init__.py
static
------
The *static* folder contains only two files with the specified above names (otherwise, they won’t be read on Galaxy run).
static files
------------
- script.js - all JavaScript code (with all third-party dependencies) must be here
- styles.css - all CSS styles, used by the plugin
@@ -149,41 +143,3 @@ tool/workflow
If a tool or a workflow plugin has script.js and/or styles.css, the content of these files will be read as two strings and sent to the client and appended to DOM’s <head>.
Such approach is a possible bottleneck if the two files are big (however, this shouldn’t ever happen because plugins are supposed to be small and simple).
masthead
--------
Topbar buttons are hard coded, so they’re rendered only after *make client*.
The plugin system is entirely dynamic. All plugins are detected during Galaxy load and their configs and statics are being saved. So, every plugin must be shown/rendered dynamically.
I found a not very optimal way to add buttons to the topbar (masthead):
.. code-block:: javascript
$(document).ready(function() {
Galaxy.page.masthead.collection.add({
id : ... ,
icon : ... ,
url : ... ,
tooltip : ... ,
onlick : function() { ... }
});
});
history-menu
------------
History Panel items are again hard coded, but in the current implementation they’re rendered as html elements (so, they’re not even stored in a collection or any other object).
To add new menu items, I do the following:
.. code-block:: javascript
menu.push({
html : _l( ... ),
anon : true,
func : function() { ... }
});
But in order to fetch all plugin menu items before rendering, I get them via API in a synchronous manner. The problem is that History Panel now may load a bit longer.
+9 -38
View File
@@ -3,7 +3,6 @@ API Controller providing Galaxy Webhooks
"""
import imp
import logging
import random
from galaxy.web import _future_expose_api_anonymous_and_sessionless as \
expose_api_anonymous_and_sessionless
@@ -17,7 +16,7 @@ class WebhooksController(BaseAPIController):
super(WebhooksController, self).__init__(app)
@expose_api_anonymous_and_sessionless
def get_all(self, trans, **kwd):
def all_webhooks(self, trans, **kwd):
"""
*GET /api/webhooks/
Returns all webhooks
@@ -28,35 +27,9 @@ class WebhooksController(BaseAPIController):
]
@expose_api_anonymous_and_sessionless
def get_random(self, trans, webhook_type, **kwd):
def webhook_data(self, trans, webhook_id, **kwd):
"""
*GET /api/webhooks/{webhook_type}
Returns a random webhook for a given type
"""
webhooks = [
webhook
for webhook in self.app.webhooks_registry.webhooks
if webhook_type in webhook.type and
webhook.activate is True
]
return random.choice(webhooks).to_dict() if webhooks else {}
@expose_api_anonymous_and_sessionless
def get_all_by_type(self, trans, webhook_type, **kwd):
"""
*GET /api/webhooks/{webhook_type}/all
Returns all webhooks for a given type
"""
return [
webhook.to_dict()
for webhook in self.app.webhooks_registry.webhooks
if webhook_type in webhook.type
]
@expose_api_anonymous_and_sessionless
def get_data(self, trans, webhook_name, **kwd):
"""
*GET /api/webhooks/{webhook_name}/get_data/{params}
*GET /api/webhooks/{webhook_id}/data/{params}
Returns the result of executing helper function
"""
params = {}
@@ -64,14 +37,12 @@ class WebhooksController(BaseAPIController):
for key, value in kwd.items():
params[key] = value
webhook = [
webhook = (
webhook
for webhook in self.app.webhooks_registry.webhooks
if webhook.name == webhook_name
]
if webhook.id == webhook_id
).next()
return imp.load_source('helper', webhook[0].helper).main(
trans,
webhook[0],
params,
) if webhook and webhook[0].helper != '' else {}
return imp.load_source(webhook.path, webhook.helper).main(
trans, webhook, params,
) if webhook and webhook.helper != '' else {}
+7 -19
View File
@@ -609,29 +609,17 @@ def populate_api_routes(webapp, app):
# ===== WEBHOOKS API =====
# ========================
webapp.mapper.connect('get_all',
webapp.mapper.connect('get_all_webhooks',
'/api/webhooks',
controller='webhooks',
action='get_all',
conditions=dict(method=["GET"]))
action='all_webhooks',
conditions=dict(method=['GET']))
webapp.mapper.connect('get_random',
'/api/webhooks/{webhook_type}',
webapp.mapper.connect('get_webhook_data',
'/api/webhooks/{webhook_id}/data',
controller='webhooks',
action='get_random',
conditions=dict(method=["GET"]))
webapp.mapper.connect('get_all_by_type',
'/api/webhooks/{webhook_type}/all',
controller='webhooks',
action='get_all_by_type',
conditions=dict(method=["GET"]))
webapp.mapper.connect('get_data',
'/api/webhooks/{webhook_name}/get_data',
controller='webhooks',
action='get_data',
conditions=dict(method=["GET"]))
action='webhook_data',
conditions=dict(method=['GET']))
# =======================
# ===== LIBRARY API =====
+56 -53
View File
@@ -12,11 +12,12 @@ log = logging.getLogger(__name__)
class Webhook(object):
def __init__(self, w_name, w_type, w_activate, w_path):
self.name = w_name
self.type = w_type
self.activate = w_activate
self.path = w_path
def __init__(self, id, type, activate, weight, path):
self.id = id
self.type = type
self.activate = activate
self.weight = weight
self.path = path
self.styles = ''
self.script = ''
self.helper = ''
@@ -24,12 +25,13 @@ class Webhook(object):
def to_dict(self):
return {
'name': self.name,
'id': self.id,
'type': self.type,
'activate': self.activate,
'weight': self.weight,
'styles': self.styles,
'script': self.script,
'config': self.config
'config': self.config,
}
@@ -48,56 +50,57 @@ class WebhooksRegistry(object):
def load_webhooks(self):
for directory in self.webhooks_directories:
config_dir = os.path.join(directory, 'config')
config_file_path = None
for config_file in ['config.yml', 'config.yaml']:
path = os.path.join(directory, config_file)
if os.path.isfile(path):
config_file_path = path
break
if not os.path.exists(config_dir):
log.warning('directory not found: %s', config_dir)
continue
if config_file_path:
try:
self.load_webhook_from_config(directory, config_file_path)
except Exception as e:
log.exception(e)
config_dir_contents = os.listdir(config_dir)
# We are assuming that all yml/yaml files in a webhooks'
# config directory are webhook config files.
for config_file in config_dir_contents:
if config_file.endswith('.yml') or config_file.endswith('.yaml'):
self.load_webhook_from_config(config_dir, config_file)
def load_webhook_from_config(self, webhook_dir, config_file_path):
with open(config_file_path) as file:
config = yaml.safe_load(file)
def load_webhook_from_config(self, config_dir, config_file):
weight = config.get('weight', 1)
if weight < 1:
raise ValueError('Webhook weight must be greater or equal 1.')
webhook = Webhook(
config.get('id'),
config.get('type'),
config.get('activate', False),
weight,
webhook_dir,
)
# Read styles into a string, assuming all styles are in a
# single file
try:
with open(os.path.join(config_dir, config_file)) as file:
config = yaml.safe_load(file)
path = os.path.normpath(os.path.join(config_dir, '..'))
webhook = Webhook(
config['name'],
config['type'],
config['activate'],
path,
)
styles_file = os.path.join(webhook_dir, 'styles.css')
with open(styles_file, 'r') as file:
webhook.styles = file.read().replace('\n', '')
except IOError:
pass
# Read styles into a string, assuming all styles are in a
# single file
try:
styles_file = os.path.join(path, 'static/styles.css')
with open(styles_file, 'r') as file:
webhook.styles = file.read().replace('\n', '')
except IOError:
pass
# Read script into a string, assuming everything is in a
# single file
try:
script_file = os.path.join(webhook_dir, 'script.js')
with open(script_file, 'r') as file:
webhook.script = file.read()
except IOError:
pass
# Read script into a string, assuming everything is in a
# single file
try:
script_file = os.path.join(path, 'static/script.js')
with open(script_file, 'r') as file:
webhook.script = file.read()
except IOError:
pass
# Save helper function path if it exists
helper_path = os.path.join(webhook_dir, '__init__.py')
if os.path.isfile(helper_path):
webhook.helper = helper_path
# Save helper function path if it exists
helper_path = os.path.join(path, 'helper/__init__.py')
if os.path.isfile(helper_path):
webhook.helper = helper_path
webhook.config = config
self.webhooks.append(webhook)
except Exception as e:
log.exception(e)
webhook.config = config
self.webhooks.append(webhook)
+9 -23
View File
@@ -14,28 +14,13 @@ class WebhooksApiTestCase(api.ApiTestCase):
self._assert_status_code_is(response, 200)
webhook_objs = self._assert_are_webhooks(response)
names = self._get_webhook_names(webhook_objs)
for expected_name in ["history_test1", "history_test2", "masthead_test", "phdcomics", "trans_object", "xkcd"]:
assert expected_name in names
def test_get_random(self):
response = self._get('webhooks/tool')
self._assert_status_code_is(response, 200)
self._assert_is_webhook(response.json())
def test_get_all_by_type(self):
# Ensure tool type filtering include a valid webhook of type tool and excludes a webhook
# that isn't of type tool.
response = self._get('webhooks/tool/all')
self._assert_status_code_is(response, 200)
webhook_objs = self._assert_are_webhooks(response)
names = self._get_webhook_names(webhook_objs)
assert "phdcomics" in names
assert "trans_object" not in names # properly filtered out by type
ids = self._get_webhook_ids(webhook_objs)
for expected_id in ['history_test1', 'history_test2', 'masthead_test',
'phdcomics', 'trans_object', 'xkcd']:
assert expected_id in ids
def test_get_data(self):
response = self._get('webhooks/trans_object/get_data')
response = self._get('webhooks/trans_object/data')
self._assert_status_code_is(response, 200)
self._assert_has_keys(response.json(), 'username')
@@ -48,8 +33,9 @@ class WebhooksApiTestCase(api.ApiTestCase):
def _assert_is_webhook(self, obj):
assert isinstance(obj, dict)
self._assert_has_keys(obj, 'styles', 'activate', 'name', 'script', 'type', 'config')
self._assert_has_keys(obj,
'id', 'type', 'activate', 'weight', 'script', 'styles', 'config')
def _get_webhook_names(self, webhook_objs):
names = [w.get("name") for w in webhook_objs]
def _get_webhook_ids(self, webhook_objs):
names = [w.get('id') for w in webhook_objs]
return names
@@ -1,5 +1,5 @@
name: history_test1
id: history_test1
title: History Menu Webhook Item 1
type:
type:
- history-menu
activate: true
@@ -1,5 +1,5 @@
name: history_test2
id: history_test2
title: History Menu Webhook Item 2
type:
type:
- history-menu
activate: true
@@ -1,5 +1,5 @@
name: masthead_test
type:
id: masthead_test
type:
- masthead
activate: true
@@ -1,4 +1,4 @@
name: phdcomics
id: phdcomics
type:
- tool
- workflow
@@ -32,7 +32,7 @@ $(document).ready(function() {
getRandomComic: function() {
var me = this,
url = galaxyRoot + 'api/webhooks/phdcomics/get_data';
url = galaxyRoot + 'api/webhooks/phdcomics/data';
this.$comicImg.html($('<div/>', {
id: 'phdcomics-loader'
@@ -1,4 +1,4 @@
name: trans_object
id: trans_object
type:
- masthead
activate: true
@@ -7,6 +7,6 @@ icon: fa-user
tooltip: Show Username
function: >
$.getJSON(Galaxy.root + "api/webhooks/trans_object/get_data", function(data) {
$.getJSON(Galaxy.root + "api/webhooks/trans_object/data", function(data) {
alert('Username: ' + data.username);
});
@@ -1,4 +1,4 @@
name: xkcd
id: xkcd
type:
- tool
- workflow
@@ -72,9 +72,9 @@ class ToolDescribingToursTestCase(SeleniumTestCase):
def _ensure_tdt_available(self):
""" Skip a test if the webhook TDT doesn't appear. """
response = self.api_get('webhooks/tool-menu/all', raw=True)
response = self.api_get('webhooks', raw=True)
self.assertEqual(response.status_code, 200)
data = response.json()
webhooks = [x['name'] for x in data]
webhooks = [x['id'] for x in data]
if 'tour_generator' not in webhooks:
raise unittest.SkipTest('Skipping test, webhook "Tool-Describing-Tours" doesn\'t appear to be configured.')