opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-01 15:37:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'release_21.09' into dev

Browse Source
This commit is contained in:
Nicola Soranzo
2021-10-04 13:44:23 +01:00
parent 54df81a4f5 28430bf053
commit 92c49b9757
38 changed files with 337 additions and 341 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/src/components/Form/utilities.js
+16 -8
View File
@@ -4,7 +4,7 @@ import _ from "underscore";
* @param{dict} inputs - Nested dictionary of input elements
* @param{dict} callback - Called with the mapped dictionary object and corresponding model node
*/
export var visitInputs = (inputs, callback, prefix, context) => {
export function visitInputs(inputs, callback, prefix, context) {
context = Object.assign({}, context);
_.each(inputs, (input) => {
if (input && input.type && input.name) {
@@ -41,24 +41,32 @@ export var visitInputs = (inputs, callback, prefix, context) => {
callback(node, name, context);
}
}
};
}
/** Match conditional values to selected cases
* @param{dict} input - Definition of conditional input parameter
* @param{dict} value - Current value
*/
export var matchCase = (input, value) => {
export function matchCase(input, value) {
if (input.test_param.type == "boolean") {
if (value == "true") {
value = input.test_param.truevalue || "true";
if (["true", true].includes(value)) {
if (input.test_param.truevalue !== undefined) {
value = input.test_param.truevalue;
} else {
value = "true";
}
} else {
value = input.test_param.falsevalue || "false";
if (input.test_param.falsevalue !== undefined) {
value = input.test_param.falsevalue;
} else {
value = "false";
}
}
}
for (var i in input.cases) {
for (let i = 0; i < input.cases.length; i++) {
if (input.cases[i].value == value) {
return i;
}
}
return -1;
};
}
client/src/components/Form/utilities.test.js
+86
View File
@@ -0,0 +1,86 @@
import { matchCase, visitInputs } from "./utilities";
function visitInputsString(inputs) {
let results = "";
visitInputs(inputs, (input, identifier) => {
results += `${identifier}=${input.value};`;
});
return results;
}
describe("form component utilities", () => {
it("conditional case matching", () => {
const input = {
name: "a",
type: "conditional",
test_param: {
name: "b",
type: "boolean",
value: "true",
truevalue: undefined,
falsevalue: undefined,
},
cases: [
{
value: "true",
inputs: [
{
name: "c",
type: "text",
value: "cvalue",
},
],
},
{
value: "false",
inputs: [
{
name: "d",
type: "text",
value: "dvalue",
},
],
},
],
};
// test simple case matching
expect(matchCase(input, "true")).toEqual(0);
expect(matchCase(input, true)).toEqual(0);
expect(matchCase(input, "false")).toEqual(1);
expect(matchCase(input, false)).toEqual(1);
// test truevalue
input.test_param.truevalue = "truevalue";
expect(matchCase(input, "true")).toEqual(-1);
input.cases[0].value = "truevalue";
expect(matchCase(input, "true")).toEqual(0);
// test falsevalue
input.test_param.falsevalue = "falsevalue";
expect(matchCase(input, "true")).toEqual(0);
expect(matchCase(input, "false")).toEqual(-1);
input.cases[1].value = "falsevalue";
expect(matchCase(input, "false")).toEqual(1);
// test (empty) truevalue
input.test_param.truevalue = undefined;
input.cases[0].value = "true";
expect(matchCase(input, "true")).toEqual(0);
input.test_param.truevalue = "";
expect(matchCase(input, "true")).toEqual(-1);
input.cases[0].value = "";
expect(matchCase(input, "true")).toEqual(0);
// test visit inputs
expect(visitInputsString([input])).toEqual("a|b=true;a|c=cvalue;");
input.test_param.value = "false";
expect(visitInputsString([input])).toEqual("a|b=false;a|d=dvalue;");
// switch test parameter to other type than boolean e.g. select
input.test_param.type = "select";
expect(matchCase(input, "")).toEqual(0);
expect(matchCase(input, "unavailable")).toEqual(-1);
expect(matchCase(input, "falsevalue")).toEqual(1);
});
});
client/src/components/HistoryExport/ExportLink.vue
+1 -1
View File
@@ -55,7 +55,7 @@ export default {
this.details = true;
},
copyUrl() {
copy(this.latestExportUrl, "Export URL copied to your clipboard");
copy(this.link, "Export URL copied to your clipboard");
},
},
};
client/src/components/Markdown/Elements/HistoryDatasetCollection/CollectionDisplay.vue
+1
View File
@@ -131,5 +131,6 @@ export default {
<style scoped>
.content-height {
max-height: 15rem;
overflow-y: auto;
}
</style>
client/src/components/Markdown/Elements/HistoryDatasetDisplay.vue
+1
View File
@@ -177,5 +177,6 @@ export default {
<style scoped>
.content-height {
max-height: 20rem;
overflow-y: auto;
}
</style>
client/src/components/Markdown/Elements/ToolStd.vue
+1
View File
@@ -33,5 +33,6 @@ export default {
<style scoped>
.content-height {
max-height: 15rem;
overflow-y: auto;
}
</style>
client/src/components/Markdown/Elements/Workflow/WorkflowDisplay.vue
+1
View File
@@ -109,5 +109,6 @@ export default {
<style scoped>
.content-height {
max-height: 15rem;
overflow-y: auto;
}
</style>
client/src/components/NewUserWelcome/NewUserWelcome.vue
+1 -1
View File
@@ -85,7 +85,7 @@ export default {
};
</script>
<style type="text/css">
<style scoped type="text/css">
.card {
border: 0px;
}
client/src/components/Tool/ToolForm.vue
+15 -12
View File
@@ -14,14 +14,16 @@
:tool-name="toolName"
/>
<Webhook v-if="showSuccess" type="tool" :tool-id="jobDef.tool_id" />
<b-alert v-if="showError" show variant="danger">
<h4>{{ errorTitle | l }}</h4>
<p>
The server could not complete the request. Please contact the Galaxy Team if this error
persists.
</p>
<pre>{{ errorContentPretty }}</pre>
</b-alert>
<b-modal v-model="showError" size="sm" :title="errorTitle | l" scrollable ok-only>
<b-alert show variant="danger">
The server could not complete this request. Please verify your parameter settings, retry
submission and contact the Galaxy Team if this error persists. A transcript of the submitted
data is shown below.
</b-alert>
<small class="text-muted">
<pre>{{ errorContentPretty }}</pre>
</small>
</b-modal>
<ToolRecommendation v-if="showRecommendation" :tool-id="formConfig.id" />
<ToolCard
v-if="showForm"
@@ -263,22 +265,24 @@ export default {
console.debug("toolForm::onExecute()", jobDef);
submitJob(jobDef).then(
(jobResponse) => {
this.showExecuting = false;
if (Galaxy.currHistoryPanel) {
Galaxy.currHistoryPanel.refreshContents();
}
this.showForm = false;
if (jobResponse.produces_entry_points) {
this.showEntryPoints = true;
this.entryPoints = jobResponse.jobs;
}
const nJobs = jobResponse && jobResponse.jobs ? jobResponse.jobs.length : 0;
if (nJobs > 0) {
this.showForm = false;
this.showSuccess = true;
this.jobDef = jobDef;
this.jobResponse = jobResponse;
} else {
this.showError = true;
this.errorTitle = "Invalid success response. No jobs found.";
this.showForm = true;
this.errorTitle = "Job submission rejected.";
this.errorContent = jobResponse;
}
if ([true, "true"].includes(config.enable_tool_recommendations)) {
@@ -299,8 +303,7 @@ export default {
}
if (genericError) {
this.showError = true;
this.showForm = false;
this.errorTitle = "Job submission failed";
this.errorTitle = "Job submission failed.";
this.errorContent = this.jobDef;
}
}
client/src/components/Workflow/Editor/Node.vue
-3
View File
@@ -251,9 +251,6 @@ export default {
this.onRedraw();
},
onAddOutput(output, terminal) {
if (this.mapOver) {
terminal.setMapOver(this.mapOver);
}
this.outputTerminals[output.name] = terminal;
this.onRedraw();
},
client/src/components/Workflow/Editor/NodeInput.vue
+1 -1
View File
@@ -103,7 +103,7 @@ export default {
return terminal;
},
onChange() {
this.isMultiple = this.terminal.mapOver && this.terminal.mapOver.isCollection;
this.isMultiple = this.terminal.isMappedOver();
this.$emit("onChange");
},
onRemove() {
client/src/components/Workflow/Editor/NodeOutput.vue
+13 -15
View File
@@ -78,18 +78,17 @@ export default {
} else {
// create new terminal, connect like old terminal, destroy old terminal
this.$emit("onRemove", this.output);
const newTerminal = this.createTerminal(newOutput);
newTerminal.connectors = this.terminal.connectors.map((c) => {
return new Connector(this.getManager(), newTerminal, c.inputHandle);
this.createTerminal(newOutput);
this.terminal.connectors = oldTerminal.connectors.map((c) => {
return new Connector(this.getManager(), this.terminal, c.inputHandle);
});
newTerminal.destroyInvalidConnections();
this.terminal = newTerminal;
this.terminal.destroyInvalidConnections();
oldTerminal.destroy();
}
},
},
mounted() {
this.terminal = this.createTerminal(this.output);
this.createTerminal(this.output);
},
methods: {
terminalClassForOutput(output) {
@@ -102,7 +101,6 @@ export default {
return terminalClass;
},
createTerminal(output) {
let terminal;
const terminalClass = this.terminalClassForOutput(output);
const parameters = {
node: this.getNode(),
@@ -113,33 +111,33 @@ export default {
if (output.collection) {
const collection_type = output.collection_type;
const collection_type_source = output.collection_type_source;
terminal = new terminalClass({
this.terminal = new terminalClass({
...parameters,
collection_type: collection_type,
collection_type_source: collection_type_source,
datatypes: output.extensions,
});
} else if (output.parameter) {
terminal = new terminalClass({
this.terminal = new terminalClass({
...parameters,
type: output.type,
});
} else {
terminal = new terminalClass({
this.terminal = new terminalClass({
...parameters,
datatypes: output.extensions,
});
}
terminal.on("change", this.onChange.bind(this));
new OutputDragging(this.getManager(), {
el: this.$refs.terminal,
terminal: terminal,
terminal: this.terminal,
});
this.$emit("onAdd", this.output, terminal);
return terminal;
this.terminal.on("change", this.onChange.bind(this));
this.terminal.emit("change");
this.$emit("onAdd", this.output, this.terminal);
},
onChange() {
this.isMultiple = this.terminal.mapOver && this.terminal.mapOver.isCollection;
this.isMultiple = this.terminal.isMappedOver();
this.$emit("onChange");
},
onToggle() {
client/src/components/Workflow/Editor/modules/terminals.js
+4
View File
@@ -169,6 +169,7 @@ class Terminal extends EventEmitter {
}
resetMapping() {
this.mapOver = NULL_COLLECTION_TYPE_DESCRIPTION;
this.node.mapOver = undefined;
this.emit("change");
}
resetCollectionTypeSource() {
@@ -590,6 +591,9 @@ class BaseOutputTerminal extends Terminal {
super(attr);
this.datatypes = attr.datatypes;
this.optional = attr.optional;
if (this.node.mapOver) {
this.setMapOver(this.node.mapOver);
}
}
get force_datatype() {
const changeOutputDatatype = this.node.postJobActions["ChangeDatatypeAction" + this.name];
client/src/layout/menu.js
+10 -19
View File
@@ -90,25 +90,16 @@ export function fetchMenu(options = {}) {
// Visualization tab.
//
if (Galaxy.config.visualizations_visible) {
menu.push({
id: "visualization",
title: _l("Visualize"),
url: "javascript:void(0)",
tooltip: _l("Visualize datasets"),
disabled: !Galaxy.user.id,
menu: [
{
title: _l("Create Visualization"),
url: "visualizations",
target: "__use_router__",
},
{
title: _l("Interactive Environments"),
url: "visualization/gie_list",
target: "galaxy_main",
},
],
});
if (Galaxy.config.visualizations_visible) {
menu.push({
id: "visualization",
title: _l("Visualize"),
tooltip: _l("Visualize datasets"),
disabled: !Galaxy.user.id,
url: "visualizations",
target: "__use_router__",
});
}
}
//
client/src/mvc/form/form-data.js
+2 -3
View File
@@ -125,10 +125,9 @@ export var Manager = Backbone.Model.extend({
/** Matches a new tool model to the current input elements e.g. used to update dynamic options
*/
matchModel: function (inputs, callback) {
var self = this;
visitInputs(inputs, (input, name) => {
if (self.flat_dict[name]) {
callback(input, self.flat_dict[name]);
if (this.flat_dict[name]) {
callback(input, this.flat_dict[name]);
}
});
},
client/tests/qunit/tests/workflow_editor_tests.js
+27 -2
View File
@@ -326,6 +326,7 @@ QUnit.test("Collection output can connect to same collection input type", functi
const outputTerminal = new Terminals.OutputCollectionTerminal({
datatypes: "txt",
collection_type: "list",
node: {},
});
outputTerminal.node = {postJobActions: {}};
assert.ok(
@@ -340,6 +341,7 @@ QUnit.test("Optional collection output can not connect to required collection in
datatypes: "txt",
collection_type: "list",
optional: true,
node: {},
});
outputTerminal.node = {};
assert.ok(!inputTerminal.canAccept(outputTerminal).canAccept);
@@ -350,6 +352,7 @@ QUnit.test("Collection output cannot connect to different collection input type"
const outputTerminal = new Terminals.OutputCollectionTerminal({
datatypes: "txt",
collection_type: "paired",
node: {},
});
outputTerminal.node = {};
assert.ok(!inputTerminal.canAccept(outputTerminal).canAccept);
@@ -542,6 +545,7 @@ QUnit.module("Node view", {
datatypes: [outputType],
mapOver: Terminals.NULL_COLLECTION_TYPE_DESCRIPTION,
element: inputEl,
node: {},
});
outputTerminal.node = {
markChanged: function () {},
@@ -572,6 +576,7 @@ QUnit.module("Node view", {
datatypes: ["txt"],
mapOver: new Terminals.CollectionTypeDescription("list"),
element: inputEl,
node: {},
});
outputTerminal.node = {
markChanged: function () {},
@@ -598,6 +603,7 @@ QUnit.module("Node view", {
datatypes: ["txt"],
mapOver: new Terminals.CollectionTypeDescription("list"),
element: inputEl,
node: {},
});
outputTerminal.node = {
markChanged: function () {},
@@ -832,7 +838,10 @@ QUnit.test("equal", function (assert) {
});
QUnit.test("default constructor", function (assert) {
const terminal = new Terminals.InputTerminal({ input: {} });
const terminal = new Terminals.InputTerminal({
input: {},
node: {},
});
assert.ok(terminal.mapOver === Terminals.NULL_COLLECTION_TYPE_DESCRIPTION);
});
@@ -902,7 +911,11 @@ QUnit.module("terminal mapping logic", {
output["extensions"] = ["data"];
}
const outputEl = $("<div>")[0];
const outputTerminal = new Terminals.OutputTerminal({ element: outputEl, datatypes: output.extensions });
const outputTerminal = new Terminals.OutputTerminal({
element: outputEl,
datatypes: output.extensions,
node: {},
});
outputTerminal.node = node;
if (mapOver) {
outputTerminal.setMapOver(new Terminals.CollectionTypeDescription(mapOver));
@@ -921,6 +934,7 @@ QUnit.module("terminal mapping logic", {
element: outputEl,
datatypes: output.extensions,
collection_type: collectionType,
node: {},
});
outputTerminal.node = node;
if (mapOver) {
@@ -1265,3 +1279,14 @@ QUnit.test("simple mapping over collection outputs works correctly", function (a
const testTerminal1 = this.newInputTerminal("list:list:list");
this.verifyNotAttachable(assert, testTerminal1, connectedOutput);
});
QUnit.test("node mapping state over collection outputs works correctly", function (assert) {
const inputTerminal1 = this.newInputTerminal();
const outputCollectionTerminal1 = this.newOutputCollectionTerminal("list");
assert.ok(!inputTerminal1.node.mapOver);
const connector = new Connector({}, outputCollectionTerminal1, inputTerminal1);
outputCollectionTerminal1.connect(connector);
assert.ok(inputTerminal1.node.mapOver);
inputTerminal1.disconnect(connector);
assert.ok(!inputTerminal1.node.mapOver);
});
config/plugins/webhooks/news/script.js
+1 -1
View File
@@ -33,7 +33,7 @@ function addNewsIframe() {
currentGalaxyVersion = "21.09";
}
const releaseNotes = `https://docs.galaxyproject.org/en/master/releases/${currentGalaxyVersion}_announce_user.html`;
const releaseNotes = `https://docs.galaxyproject.org/en/latest/releases/${currentGalaxyVersion}_announce_user.html`;
const lastSeenVersion = window.localStorage.getItem("galaxy-news-seen-release");
// Check that they've seen the current version's release notes.
if (lastSeenVersion != currentGalaxyVersion) {
doc/source/admin/galaxy_options.rst
+8 -10
View File
@@ -2788,18 +2788,16 @@
:Type: str
~~~~~~~~~~~~~~~~~~~~~~~~~~~
``sentry_sloreq_threshold``
~~~~~~~~~~~~~~~~~~~~~~~~~~~
~~~~~~~~~~~~~~~~~~~~~~
``sentry_event_level``
~~~~~~~~~~~~~~~~~~~~~~
:Description:
Sentry slow request logging. Requests slower than the threshold
indicated below will be sent as events to the configured Sentry
server (above, sentry_dsn). A value of '0' is disabled. For
example, you would set this to .005 to log all queries taking
longer than 5 milliseconds.
:Default: ``0.0``
:Type: float
Determines the minimum log level that will be sent as an event to
Sentry. Possible values are DEBUG, INFO, WARNING, ERROR or
CRITICAL.
:Default: ``ERROR``
:Type: str
~~~~~~~~~~~~~~~
lib/galaxy/app.py
+14 -2
View File
@@ -193,10 +193,22 @@ class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication):
self.sentry_client = None
if self.config.sentry_dsn:
event_level = self.config.sentry_event_level.upper()
assert event_level in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], f"Invalid sentry event level '{self.config.sentry.event_level}'"
def postfork_sentry_client():
import raven
self.sentry_client = raven.Client(self.config.sentry_dsn, transport=raven.transport.HTTPTransport)
import sentry_sdk
from sentry_sdk.integrations.logging import LoggingIntegration
sentry_logging = LoggingIntegration(
level=logging.INFO, # Capture info and above as breadcrumbs
event_level=getattr(logging, event_level) # Send errors as events
)
self.sentry_client = sentry_sdk.init(
self.config.sentry_dsn,
release=f"{self.config.version_major}.{self.config.version_minor}",
integrations=[sentry_logging]
)
self.application_stack.register_postfork_function(postfork_sentry_client)
lib/galaxy/config/__init__.py
+1 -10
View File
@@ -45,10 +45,7 @@ from galaxy.util.properties import (
running_from_source,
)
from galaxy.web.formatting import expand_pretty_datetime_format
from galaxy.web_stack import (
get_stack_facts,
register_postfork_function
)
from galaxy.web_stack import get_stack_facts
from ..version import VERSION_MAJOR, VERSION_MINOR
log = logging.getLogger(__name__)
@@ -1106,7 +1103,6 @@ def configure_logging(config):
"""
# Get root logger
logging.addLevelName(LOGLV_TRACE, "TRACE")
root = logging.getLogger()
# PasteScript will have already configured the logger if the
# 'loggers' section was found in the config file, otherwise we do
# some simple setup using the 'log_*' values from the config.
@@ -1129,11 +1125,6 @@ def configure_logging(config):
conf['filename'] = conf.pop('filename_template').format(**get_stack_facts(config=config))
logging_conf['handlers'][name] = conf
logging.config.dictConfig(logging_conf)
if getattr(config, "sentry_dsn", None):
from raven.handlers.logging import SentryHandler
sentry_handler = SentryHandler(config.sentry_dsn)
sentry_handler.setLevel(logging.WARN)
register_postfork_function(root.addHandler, sentry_handler)
class ConfiguresGalaxyMixin:
lib/galaxy/config/sample/error_report.yml.sample
+4 -5
View File
@@ -15,7 +15,7 @@
# verbose/user_submission, but those are not necessary to provide.
# The default Email bug reporter. By default, the standard
# configuration is taken from your galaxy.ini
# configuration is taken from your galaxy.yml
- type: email
verbose: true
user_submission: true
@@ -29,10 +29,9 @@
# directory: /tmp/reports/
# Submit error reports to sentry. If a sentry_dsn is configured in your
# galaxy.ini, then Galaxy will submit the job error to Sentry. You may supply a
# separate DSN for tool reports by supplying a ``custom_dsn`` parameter.
- type: sentry
user_submission: false
# galaxy.yml, then Galaxy will submit the job error to Sentry.
# - type: sentry
# user_submission: false
# InfluxDB error reporting backend. You will need to `pip install
# influxdb` in the galaxy virtualenv yourself. This sends well tagged
lib/galaxy/config/sample/galaxy.yml.sample
+3 -6
View File
@@ -1388,12 +1388,9 @@ galaxy:
# <project_name> -> Settings -> API Keys.
#sentry_dsn: null
# Sentry slow request logging. Requests slower than the threshold
# indicated below will be sent as events to the configured Sentry
# server (above, sentry_dsn). A value of '0' is disabled. For
# example, you would set this to .005 to log all queries taking longer
# than 5 milliseconds.
#sentry_sloreq_threshold: 0.0
# Determines the minimum log level that will be sent as an event to
# Sentry. Possible values are DEBUG, INFO, WARNING, ERROR or CRITICAL.
#sentry_event_level: ERROR
# Log to statsd Statsd is an external statistics aggregator
# (https://github.com/etsy/statsd) Enabling the following options will
lib/galaxy/dependencies/__init__.py
+1 -1
View File
@@ -191,7 +191,7 @@ class ConditionalDependencies:
def check_fluent_logger(self):
return asbool(self.config["fluent_log"])
def check_raven(self):
def check_sentry_sdk(self):
return self.config.get("sentry_dsn", None) is not None
def check_statsd(self):
lib/galaxy/dependencies/conditional-requirements.txt
+1 -1
View File
@@ -2,7 +2,7 @@
psycopg2-binary==2.8.4
mysqlclient
fluent-logger
raven
sentry-sdk
pbs_python
drmaa
statsd
lib/galaxy/files/__init__.py
+2
View File
@@ -146,6 +146,8 @@ class ConfiguredFileSources:
def plugins_to_dict(self, for_serialization=False, user_context=None):
rval = []
for file_source in self._file_sources:
if not file_source.user_has_access(user_context):
continue
el = file_source.to_dict(for_serialization=for_serialization, user_context=user_context)
rval.append(el)
return rval
lib/galaxy/files/sources/__init__.py
+11 -1
View File
@@ -71,6 +71,8 @@ class BaseFilesSource(FilesSource):
return self.writable
def user_has_access(self, user_context) -> bool:
if user_context is None and self.user_context_required:
return False
return (
user_context is None
or user_context.is_admin
@@ -80,6 +82,10 @@ class BaseFilesSource(FilesSource):
)
)
@property
def user_context_required(self) -> bool:
return self.requires_roles is not None or self.requires_groups is not None
def get_uri_root(self):
prefix = self.get_prefix()
scheme = self.get_scheme()
@@ -164,7 +170,11 @@ class BaseFilesSource(FilesSource):
pass
def _check_user_access(self, user_context):
"""Raises an exception if the given user doesn't have the rights to access this file source."""
"""Raises an exception if the given user doesn't have the rights to access this file source.
Warning: if the user_context is None, then the check is skipped. This is due to tool executions context
not having access to the user_context. The validation will be done when checking the tool parameters.
"""
if user_context is not None and not self.user_has_access(user_context):
raise ItemAccessibilityException(f"User {user_context.username} has no access to file source.")
lib/galaxy/managers/remote_files.py
+4 -5
View File
@@ -9,8 +9,6 @@ from typing import (
Optional,
)
from pydantic.tools import parse_obj_as
from galaxy import exceptions
from galaxy.app import MinimalManagerApp
from galaxy.files import (
@@ -115,10 +113,11 @@ class RemoteFilesManager:
return index
def get_files_source_plugins(self) -> FilesSourcePluginList:
def get_files_source_plugins(self, user_context: ProvidesUserContext) -> FilesSourcePluginList:
"""Display plugin information for each of the gxfiles:// URI targets available."""
plugins = self._file_sources.plugins_to_dict()
return parse_obj_as(FilesSourcePluginList, plugins)
user_file_source_context = ProvidesUserFileSourcesUserContext(user_context)
plugins = self._file_sources.plugins_to_dict(user_context=user_file_source_context)
return FilesSourcePluginList.parse_obj(plugins)
@property
def _file_sources(self) -> ConfiguredFileSources:
lib/galaxy/tools/data/__init__.py
+7 -1
View File
@@ -854,7 +854,13 @@ class RefgenieToolDataTable(TabularToolDataTable):
self.columns['name'] = self.columns['value']
def parse_file_fields(self, filename, errors=None, here="__HERE__"):
rgc = refgenconf.RefGenConf(filename)
try:
rgc = refgenconf.RefGenConf(filename, writable=False, skip_read_lock=True)
except refgenconf.exceptions.RefgenconfError as e:
log.error('Unable to load refgenie config file "%s": %s', filename, e)
if errors is not None:
errors.append(e)
return []
rval = []
for genome in rgc.list_genomes_by_asset(self.rg_asset):
genome_attributes = rgc.get_genome_attributes(genome)
lib/galaxy/tools/error_reports/__init__.py
-4
View File
@@ -14,10 +14,6 @@ DEFAULT_CONFIG = [
'verbose': True,
'user_submission': True,
},
{
'type': 'sentry',
'user_submission': False,
},
]
DEFAULT_PLUGINS_SOURCE = plugin_config.PluginConfigSource('dict', DEFAULT_CONFIG)
lib/galaxy/tools/error_reports/plugins/sentry.py
+69 -77
View File
@@ -1,12 +1,18 @@
"""The module describes the ``sentry`` error plugin plugin."""
import logging
try:
import sentry_sdk
except ImportError:
sentry_sdk = None
from galaxy import web
from galaxy.util import string_as_bool, unicodify
from galaxy.util import string_as_bool
from . import ErrorPlugin
log = logging.getLogger(__name__)
SENTRY_SDK_IMPORT_MESSAGE = 'The Python sentry-sdk package is required to use this feature, please install it'
ERROR_TEMPLATE = """Galaxy Job Error: {tool_id} v{tool_version}
Command Line:
@@ -32,94 +38,80 @@ class SentryPlugin(ErrorPlugin):
self.redact_user_details_in_bugreport = self.app.config.redact_user_details_in_bugreport
self.verbose = string_as_bool(kwargs.get('verbose', False))
self.user_submission = string_as_bool(kwargs.get('user_submission', False))
self.custom_dsn = kwargs.get('custom_dsn', None)
self.sentry = None
# Use the built in one by default
if hasattr(self.app, 'sentry_client'):
self.sentry = self.app.sentry_client
# if they've set a custom one, override.
if self.custom_dsn:
import raven
self.sentry = raven.Client(self.custom_dsn, transport=raven.transport.HTTPTransport)
assert sentry_sdk, SENTRY_SDK_IMPORT_MESSAGE
def submit_report(self, dataset, job, tool, **kwargs):
"""Submit the error report to sentry
"""
if self.sentry:
user = job.get_user()
extra = {
'info': job.info,
'id': job.id,
'command_line': unicodify(job.command_line),
'destination_id': unicodify(job.destination_id),
'stderr': unicodify(job.stderr),
'traceback': unicodify(job.traceback),
'exit_code': job.exit_code,
'stdout': unicodify(job.stdout),
'handler': unicodify(job.handler),
'tool_id': unicodify(job.tool_id),
'tool_version': unicodify(job.tool_version),
'tool_xml': unicodify(tool.config_file) if tool else None
}
if self.redact_user_details_in_bugreport:
extra['email'] = 'redacted'
else:
if 'email' in kwargs:
extra['email'] = unicodify(kwargs['email'])
extra = {
'info': job.info,
'id': job.id,
'command_line': job.command_line,
'destination_id': job.destination_id,
'stderr': job.stderr,
'traceback': job.traceback,
'exit_code': job.exit_code,
'stdout': job.stdout,
'handler': job.handler,
'tool_id': job.tool_id,
'tool_version': job.tool_version,
'tool_xml': tool.config_file if tool else None
}
if self.redact_user_details_in_bugreport:
extra['email'] = 'redacted'
else:
if 'email' in kwargs:
extra['email'] = kwargs['email']
# User submitted message
extra['message'] = unicodify(kwargs.get('message', ''))
# User submitted message
extra['message'] = kwargs.get('message', '')
# Construct the error message to send to sentry. The first line
# will be the issue title, everything after that becomes the
# "message"
error_message = ERROR_TEMPLATE.format(**extra)
# Construct the error message to send to sentry. The first line
# will be the issue title, everything after that becomes the
# "message"
error_message = ERROR_TEMPLATE.format(**extra)
# Update context with user information in a sentry-specific manner
context = {}
# Update context with user information in a sentry-specific manner
context = {}
# Getting the url allows us to link to the dataset info page in case
# anything is missing from this report.
try:
url = web.url_for(controller="dataset",
action="details",
dataset_id=self.app.security.encode_id(dataset.id),
qualified=True)
except AttributeError:
# The above does not work when handlers are separate from the web handlers
url = None
# Getting the url allows us to link to the dataset info page in case
# anything is missing from this report.
try:
url = web.url_for(controller="dataset",
action="show_params",
dataset_id=self.app.security.encode_id(dataset.id),
qualified=True)
except AttributeError:
# The above does not work when handlers are separate from the web handlers
url = None
if self.redact_user_details_in_bugreport:
if user:
# Opauqe identifier
context['user'] = {
'id': user.id
}
else:
if user:
# User information here also places email links + allows seeing
# a list of affected users in the tags/filtering.
context['user'] = {
'name': user.username,
'email': user.email,
}
user = job.get_user()
if self.redact_user_details_in_bugreport:
if user:
# Opaque identifier
context['user'] = {
'id': user.id
}
else:
if user:
# User information here also places email links + allows seeing
# a list of affected users in the tags/filtering.
context['user'] = {
'name': user.username,
'email': user.email,
}
context['request'] = {'url': url}
context['request'] = {'url': url}
self.sentry_client.context.merge(context)
for key, value in context.items():
sentry_sdk.set_context(key, value)
sentry_sdk.set_context('job', extra)
sentry_sdk.set_tag('tool_id', job.tool_id)
sentry_sdk.set_tag('tool_version', job.tool_version)
# Send the message, using message because
response = self.sentry_client.capture(
'raven.events.Message',
tags={
'tool_id': job.tool_id,
'tool_version': job.tool_version,
},
extra=extra,
message=unicodify(error_message),
)
return (f'Submitted bug report to Sentry. Your guru meditation number is {response}', 'success')
# Send the message, using message because
response = sentry_sdk.capture_message(error_message)
return (f'Submitted bug report to Sentry. Your guru meditation number is {response}', 'success')
__all__ = ('SentryPlugin', )
lib/galaxy/tools/parameters/basic.py
+9
View File
@@ -13,6 +13,7 @@ from webob.compat import cgi_FieldStorage
import galaxy.model
from galaxy import util
from galaxy.files import ProvidesUserFileSourcesUserContext
from galaxy.tool_util.parser import get_input_source as ensure_input_source
from galaxy.util import (
dbkeys,
@@ -2414,6 +2415,14 @@ class DirectoryUriToolParameter(SimpleTextToolParameter):
input_source = ensure_input_source(input_source)
SimpleTextToolParameter.__init__(self, tool, input_source)
def validate(self, value, trans=None):
super().validate(value, trans=trans)
file_source = trans.app.file_sources.get_file_source_path(value).file_source
user_context = ProvidesUserFileSourcesUserContext(trans)
user_has_access = file_source.user_has_access(user_context)
if not user_has_access:
raise ParameterValueError(f"The user cannot access {value}.", self.name)
class RulesListToolParameter(BaseJsonToolParameter):
"""
lib/galaxy/web/framework/middleware/sentry.py
-138
View File
@@ -1,138 +0,0 @@
"""
raven.middleware
~~~~~~~~~~~~~~~~~~~~~~~~
:copyright: (c) 2010-2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
import time
try:
from raven import Client
from raven.utils.wsgi import get_current_url, get_headers, get_environ
except ImportError:
Client = None
from galaxy.web_stack import register_postfork_function
RAVEN_IMPORT_MESSAGE = ('The Python raven package is required to use this '
'feature, please install it')
class Sentry:
"""
A WSGI middleware which will attempt to capture any
uncaught exceptions and send them to Sentry.
"""
def __init__(self, application, dsn, sloreq=0):
assert Client is not None, RAVEN_IMPORT_MESSAGE
self.application = application
self.client = None
self.sloreq_threshold = sloreq
def postfork_sentry_client():
self.client = Client(dsn)
register_postfork_function(postfork_sentry_client)
def __call__(self, environ, start_response):
try:
start_time = time.time()
iterable = self.application(environ, start_response)
dt = (time.time() - start_time)
if self.sloreq_threshold and dt > self.sloreq_threshold:
self.handle_slow_request(environ, dt)
except Exception:
self.handle_exception(environ)
raise
try:
yield from iterable
except Exception:
self.handle_exception(environ)
raise
finally:
# wsgi spec requires iterable to call close if it exists
# see http://blog.dscpl.com.au/2012/10/obligations-for-calling-close-on.html
if iterable and hasattr(iterable, 'close') and callable(iterable.close):
try:
iterable.close()
except Exception:
self.handle_exception(environ)
def handle_slow_request(self, environ, dt):
headers = dict(get_headers(environ))
if 'Authorization' in headers:
headers['Authorization'] = 'redacted'
if 'Cookie' in headers:
headers['Cookie'] = 'redacted'
cak = environ.get('controller_action_key', None) or environ.get('PATH_INFO', "NOPATH").strip('/').replace('/', '.')
event_id = self.client.captureMessage(
f"SLOREQ: {cak}",
data={
'sentry.interfaces.Http': {
'method': environ.get('REQUEST_METHOD'),
'url': get_current_url(environ, strip_querystring=True),
'query_string': environ.get('QUERY_STRING'),
'headers': headers,
'env': dict(get_environ(environ)),
}
},
extra={
'request_id': environ.get('request_id', 'Unknown'),
'request_duration_millis': dt * 1000
},
level="warning",
tags={
'type': 'sloreq',
'action_key': cak
}
)
# Galaxy: store event_id in environment so we can show it to the user
environ['sentry_event_id'] = event_id
return event_id
def handle_exception(self, environ):
headers = dict(get_headers(environ))
# Authorization header for REMOTE_USER sites consists of a base64() of
# their plaintext password. It is a security issue for this password to
# be exposed to a third party system which may or may not be under
# control of the same administrators as the local Authentication
# system. E.g. university LDAP systems.
if 'Authorization' in headers:
# Redact so the administrator knows that a value is indeed present.
headers['Authorization'] = 'redacted'
# Passing cookies allows for impersonation of users (depending on
# remote service) and can be considered a security risk as well. For
# multiple services running alongside Galaxy on the same host, this
# could allow a sentry user with access to logs to impersonate a user
# on another service. In the case of services like Jupyter, this can be
# a serious concern as that would allow for terminal access. Furthermore,
# very little debugging information can be gained as a result of having
# access to all of the users cookies (including Galaxy cookies)
if 'Cookie' in headers:
headers['Cookie'] = 'redacted'
event_id = self.client.captureException(
data={
'sentry.interfaces.Http': {
'method': environ.get('REQUEST_METHOD'),
'url': get_current_url(environ, strip_querystring=True),
'query_string': environ.get('QUERY_STRING'),
# TODO
# 'data': environ.get('wsgi.input'),
'headers': headers,
'env': dict(get_environ(environ)),
}
},
# Galaxy: add request id from environment if available
extra={
'request_id': environ.get('request_id', 'Unknown')
}
)
# Galaxy: store event_id in environment so we can show it to the user
environ['sentry_event_id'] = event_id
return event_id
lib/galaxy/webapps/galaxy/api/datasets.py
+2 -1
View File
@@ -157,7 +157,8 @@ class DatasetsController(BaseGalaxyAPIController, UsesVisualizationMixin):
return self.hda_serializer.serialize_to_view(dataset,
view=kwd.get('view', 'detailed'), user=trans.user, trans=trans)
else:
rval = dataset.to_dict()
dataset_dict = dataset.to_dict()
rval = self.encode_all_ids(dataset_dict)
return rval
@web.expose_api_anonymous
lib/galaxy/webapps/galaxy/api/remote_files.py
+2 -2
View File
@@ -95,7 +95,7 @@ class FastAPIRemoteFiles:
user_ctx: ProvidesUserContext = DependsOnTrans,
) -> FilesSourcePluginList:
"""Display plugin information for each of the gxfiles:// URI targets available."""
return self.manager.get_files_source_plugins()
return self.manager.get_files_source_plugins(user_ctx)
class RemoteFilesAPIController(BaseGalaxyAPIController):
@@ -136,4 +136,4 @@ class RemoteFilesAPIController(BaseGalaxyAPIController):
:returns: list of configured plugins
:rtype: list
"""
return self.manager.get_files_source_plugins()
return self.manager.get_files_source_plugins(trans)
lib/galaxy/webapps/galaxy/buildapp.py
+2 -3
View File
@@ -1369,10 +1369,9 @@ def wrap_in_middleware(app, global_conf, application_stack, **local_conf):
# If sentry logging is enabled, log here before propogating up to
# the error middleware
sentry_dsn = conf.get('sentry_dsn', None)
sentry_sloreq = float(conf.get('sentry_sloreq_threshold', 0))
if sentry_dsn:
from galaxy.web.framework.middleware.sentry import Sentry
app = wrap_if_allowed(app, stack, Sentry, args=(sentry_dsn, sentry_sloreq))
from sentry_sdk.integrations.wsgi import SentryWsgiMiddleware
app = wrap_if_allowed(app, stack, SentryWsgiMiddleware)
# Various debug middleware that can only be turned on if the debug
# flag is set, either because they are insecure or greatly hurt
# performance
lib/galaxy/webapps/galaxy/config_schema.yml
+5 -8
View File
@@ -2029,16 +2029,13 @@ mapping:
indicated sentry instance. This connection string is available in your
sentry instance under <project_name> -> Settings -> API Keys.
sentry_sloreq_threshold:
type: float
default: 0.0
sentry_event_level:
type: str
default: ERROR
required: false
desc: |
Sentry slow request logging. Requests slower than the threshold
indicated below will be sent as events to the configured Sentry
server (above, sentry_dsn). A value of '0' is disabled. For
example, you would set this to .005 to log all queries taking longer
than 5 milliseconds.
Determines the minimum log level that will be sent as an event to Sentry.
Possible values are DEBUG, INFO, WARNING, ERROR or CRITICAL.
statsd_host:
type: str
lib/galaxy/webapps/galaxy/fast_app.py
+5
View File
@@ -80,6 +80,11 @@ def add_galaxy_middleware(app: FastAPI, gx_app):
nginx_x_accel_redirect_base = gx_app.config.nginx_x_accel_redirect_base
apache_xsendfile = gx_app.config.apache_xsendfile
if gx_app.config.sentry_dsn:
from sentry_sdk.integrations.asgi import SentryAsgiMiddleware
app.add_middleware(SentryAsgiMiddleware)
if nginx_x_accel_redirect_base or apache_xsendfile:
@app.middleware("http")
test/integration/test_remote_files_posix.py
+6
View File
@@ -19,9 +19,15 @@ class PosixFileSourceIntegrationTestCase(PosixFileSourceSetup, integration_util.
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
def test_plugin_config(self):
# Default user has required role but not required group, so cannot see plugin
plugin_config_response = self.galaxy_interactor.get("remote_files/plugins")
api_asserts.assert_status_code_is_ok(plugin_config_response)
plugins = plugin_config_response.json()
assert len(plugins) == 0
# Admins can see plugins
plugin_config_response = self.galaxy_interactor.get("remote_files/plugins", admin=True)
api_asserts.assert_status_code_is_ok(plugin_config_response)
plugins = plugin_config_response.json()
assert len(plugins) == 1
assert plugins[0]["type"] == "posix"
assert plugins[0]["uri_root"] == "gxfiles://posix_test"