diff --git a/config/plugins/webhooks/demo/tool_list/__init__.py b/config/plugins/webhooks/demo/tool_list/__init__.py
index 31db2b819c1..db8e58ceeca 100644
--- a/config/plugins/webhooks/demo/tool_list/__init__.py
+++ b/config/plugins/webhooks/demo/tool_list/__init__.py
@@ -3,7 +3,7 @@ import yaml
def main(trans, webhook, params):
data = {}
- data['tools'] = []
+ data["tools"] = []
unique_tools = []
tools = trans.app.toolbox.tools()
@@ -16,24 +16,23 @@ def main(trans, webhook, params):
except AttributeError:
continue
- if (ts_data['name'] + ts_data['installed_changeset_revision'] not in
- unique_tools):
+ if ts_data["name"] + ts_data["installed_changeset_revision"] not in unique_tools:
- unique_tools.append(
- ts_data['name'] + ts_data['installed_changeset_revision']
+ unique_tools.append(ts_data["name"] + ts_data["installed_changeset_revision"])
+
+ data["tools"].append(
+ {
+ "name": ts_data["name"],
+ "owner": ts_data["owner"],
+ "tool_panel_section_label": panel[1],
+ "tool_shed_url": ts_data["tool_shed"],
+ "install_tool_dependencies": True,
+ "install_repository_dependencies": True,
+ "install_resolver_dependencies": True,
+ "revisions": [ts_data["installed_changeset_revision"]],
+ }
)
-
- data['tools'].append({
- 'name': ts_data['name'],
- 'owner': ts_data['owner'],
- 'tool_panel_section_label': panel[1],
- 'tool_shed_url': ts_data['tool_shed'],
- 'install_tool_dependencies': True,
- 'install_repository_dependencies': True,
- 'install_resolver_dependencies': True,
- 'revisions': [ts_data['installed_changeset_revision']]
- })
if panel[0]:
- data['tools'][-1]['tool_panel_section_id'] = panel[0]
+ data["tools"][-1]["tool_panel_section_id"] = panel[0]
- return {'yaml': yaml.safe_dump(data, default_flow_style=False)}
+ return {"yaml": yaml.safe_dump(data, default_flow_style=False)}
diff --git a/config/plugins/webhooks/demo/tour_generator/__init__.py b/config/plugins/webhooks/demo/tour_generator/__init__.py
index 8e4d77e5bfc..c132aff0501 100644
--- a/config/plugins/webhooks/demo/tour_generator/__init__.py
+++ b/config/plugins/webhooks/demo/tour_generator/__init__.py
@@ -28,13 +28,12 @@ class TourGenerator:
section of the provided tool.
"""
if not self._tool.tests:
- raise ValueError('Tests are not defined.')
+ raise ValueError("Tests are not defined.")
self._test = self._tool.tests[0]
# All inputs with the type 'data'
- self._data_inputs = {x.name: x for x in self._tool.input_params if
- x.type == 'data'}
+ self._data_inputs = {x.name: x for x in self._tool.input_params if x.type == "data"}
# Datasets from the section
test_datasets = {
@@ -45,20 +44,17 @@ class TourGenerator:
# Conditional datasets
for name in self._test.inputs.keys():
- if '|' in name:
- input_name = name.split('|')[1]
+ if "|" in name:
+ input_name = name.split("|")[1]
if input_name in self._data_inputs.keys():
- test_datasets.update({
- input_name: self._test.inputs[name][0]
- })
+ test_datasets.update({input_name: self._test.inputs[name][0]})
if not test_datasets.keys():
not_supported_input_types = [
- k for k, v in self._tool.inputs.items() if
- v.type == 'repeat' or v.type == 'data_collection'
+ k for k, v in self._tool.inputs.items() if v.type == "repeat" or v.type == "data_collection"
]
if not_supported_input_types:
- raise ValueError('Not supported input types.')
+ raise ValueError("Not supported input types.")
else:
# Some tests don't have data inputs at all,
# so we can generate a tour without them
@@ -73,16 +69,15 @@ class TourGenerator:
if not input_path:
raise ValueError('Test dataset "%s" doesn\'t exist.' % input_name)
- upload_tool = self._trans.app.toolbox.get_tool('upload1')
+ upload_tool = self._trans.app.toolbox.get_tool("upload1")
filename = os.path.basename(input_path)
- with open(input_path, 'rb') as f:
+ with open(input_path, "rb") as f:
content = f.read()
headers = {
- 'content-disposition':
- 'form-data; name="{}"; filename="{}"'.format(
- 'files_0|file_data', filename
- ),
+ "content-disposition": 'form-data; name="{}"; filename="{}"'.format(
+ "files_0|file_data", filename
+ ),
}
input_file = cgi_FieldStorage(headers=headers)
@@ -90,76 +85,68 @@ class TourGenerator:
input_file.file.write(content)
inputs = {
- 'dbkey': '?', # is it always a question mark?
- 'file_type': input.extensions[0],
- 'files_0|type': 'upload_dataset',
- 'files_0|space_to_tab': None,
- 'files_0|to_posix_lines': 'Yes',
- 'files_0|file_data': input_file,
+ "dbkey": "?", # is it always a question mark?
+ "file_type": input.extensions[0],
+ "files_0|type": "upload_dataset",
+ "files_0|space_to_tab": None,
+ "files_0|to_posix_lines": "Yes",
+ "files_0|file_data": input_file,
}
params = Params(inputs, sanitize=False)
incoming = params.__dict__
- output = upload_tool.handle_input(self._trans, incoming,
- history=None)
+ output = upload_tool.handle_input(self._trans, incoming, history=None)
- job_errors = output.get('job_errors', [])
+ job_errors = output.get("job_errors", [])
if job_errors:
# self._errors.extend(job_errors)
- raise ValueError('Cannot upload a dataset.')
+ raise ValueError("Cannot upload a dataset.")
else:
- self._hids.update({
- input_name: output['out_data'][0][1].hid
- })
+ self._hids.update({input_name: output["out_data"][0][1].hid})
def _generate_tour(self):
- """ Generate a tour. """
- tour_name = self._tool.name + ' Tour'
+ """Generate a tour."""
+ tour_name = self._tool.name + " Tour"
test_inputs = self._test.inputs.keys()
- steps = [{
- 'title': tour_name,
- 'content': 'This short tour will guide you through the '
- + self._tool.name + ' tool.',
- 'orphan': True
- }]
+ steps = [
+ {
+ "title": tour_name,
+ "content": "This short tour will guide you through the " + self._tool.name + " tool.",
+ "orphan": True,
+ }
+ ]
for name, input in self._tool.inputs.items():
cond_case_steps = []
- if input.type == 'repeat':
+ if input.type == "repeat":
continue
- step = {
- 'title': input.label,
- 'element': '[tour_id=%s]' % name,
- 'placement': 'right',
- 'content': ''
- }
+ step = {"title": input.label, "element": "[tour_id=%s]" % name, "placement": "right", "content": ""}
- if input.type == 'text':
+ if input.type == "text":
if name in test_inputs:
param = self._test.inputs[name]
- step['content'] = 'Enter value(s): %s' % param
+ step["content"] = "Enter value(s): %s" % param
else:
- step['content'] = 'Enter a value'
+ step["content"] = "Enter a value"
- elif input.type == 'integer' or input.type == 'float':
+ elif input.type == "integer" or input.type == "float":
if name in test_inputs:
num_param = self._test.inputs[name][0]
- step['content'] = 'Enter number: %s' % num_param
+ step["content"] = "Enter number: %s" % num_param
else:
- step['content'] = 'Enter a number'
+ step["content"] = "Enter a number"
- elif input.type == 'boolean':
+ elif input.type == "boolean":
if name in test_inputs:
- choice = 'Yes' if self._test.inputs[name][0] is True \
- else 'No'
- step['content'] = 'Choose %s' % choice
+ choice = "Yes" if self._test.inputs[name][0] is True else "No"
+ step["content"] = "Choose %s" % choice
else:
- step['content'] = 'Choose Yes/No'
+ step["content"] = "Choose Yes/No"
- elif input.type == 'select':
+ elif input.type == "select":
params = []
if name in test_inputs:
for option in input.static_options:
@@ -167,26 +154,23 @@ class TourGenerator:
if test_option == option[1]:
params.append(option[0])
if params:
- select_param = ', '.join(params)
- step['content'] = 'Select parameter(s): %s' % \
- select_param
+ select_param = ", ".join(params)
+ step["content"] = "Select parameter(s): %s" % select_param
else:
- step['content'] = 'Select a parameter'
+ step["content"] = "Select a parameter"
- elif input.type == 'data':
+ elif input.type == "data":
if name in test_inputs:
hid = self._hids[name]
dataset = self._test.inputs[name][0]
- step['content'] = 'Select dataset: {}: {}'.format(
- hid, dataset
- )
+ step["content"] = "Select dataset: {}: {}".format(hid, dataset)
else:
- step['content'] = 'Select a dataset'
+ step["content"] = "Select a dataset"
- elif input.type == 'conditional':
- param_id = f'{input.name}|{input.test_param.name}'
- step['title'] = input.test_param.label
- step['element'] = '[tour_id="%s"]' % param_id
+ elif input.type == "conditional":
+ param_id = f"{input.name}|{input.test_param.name}"
+ step["title"] = input.test_param.label
+ step["element"] = '[tour_id="%s"]' % param_id
params = []
if param_id in self._test.inputs.keys():
@@ -203,60 +187,60 @@ class TourGenerator:
cases[key] = value.label
for case_id, case_title in cases.items():
- tour_id = f'{input.name}|{case_id}'
+ tour_id = f"{input.name}|{case_id}"
if tour_id in self._test.inputs.keys():
if case_id in self._data_inputs.keys():
hid = self._hids[case_id]
dataset = self._test.inputs[tour_id][0]
- step_msg = 'Select dataset: %s: %s' % \
- (hid, dataset)
+ step_msg = "Select dataset: %s: %s" % (hid, dataset)
else:
- case_params = ', '.join(
- self._test.inputs[tour_id])
- step_msg = 'Select parameter(s): ' + \
- '%s' % case_params
- cond_case_steps.append({
- 'title': case_title,
- 'element': '[tour_id="%s"]' % tour_id,
- 'placement': 'right',
- 'content': step_msg
- })
+ case_params = ", ".join(self._test.inputs[tour_id])
+ step_msg = "Select parameter(s): " + "%s" % case_params
+ cond_case_steps.append(
+ {
+ "title": case_title,
+ "element": '[tour_id="%s"]' % tour_id,
+ "placement": "right",
+ "content": step_msg,
+ }
+ )
if params:
- cond_param = ', '.join(params)
- step['content'] = 'Select parameter(s): %s' % \
- cond_param
+ cond_param = ", ".join(params)
+ step["content"] = "Select parameter(s): %s" % cond_param
else:
- step['content'] = 'Select a parameter'
+ step["content"] = "Select a parameter"
- elif input.type == 'data_column':
+ elif input.type == "data_column":
if name in test_inputs:
column_param = self._test.inputs[name][0]
- step['content'] = 'Select Column: %s' % column_param
+ step["content"] = "Select Column: %s" % column_param
else:
- step['content'] = 'Select a column'
+ step["content"] = "Select a column"
else:
- step['content'] = 'Select a parameter'
+ step["content"] = "Select a parameter"
steps.append(step)
if cond_case_steps:
steps.extend(cond_case_steps) # add conditional input steps
# Add the last step
- steps.append({
- 'title': 'Execute tool',
- 'content': 'Click Execute button to run the tool.',
- 'element': '#execute',
- 'placement': 'bottom',
- # 'postclick': ['#execute']
- })
+ steps.append(
+ {
+ "title": "Execute tool",
+ "content": "Click Execute button to run the tool.",
+ "element": "#execute",
+ "placement": "bottom",
+ # 'postclick': ['#execute']
+ }
+ )
self._tour = {
- 'title_default': tour_name,
- 'name': tour_name,
- 'description': self._tool.name + ' ' + self._tool.description,
- 'steps': steps
+ "title_default": tour_name,
+ "name": tour_name,
+ "description": self._tool.name + " " + self._tool.description,
+ "steps": steps,
}
def get_data(self):
@@ -264,26 +248,22 @@ class TourGenerator:
Return a dictionary with the uploaded datasets' history ids and
the generated tour.
"""
- return {
- 'useDatasets': self._use_datasets,
- 'hids': list(self._hids.values()),
- 'tour': self._tour
- }
+ return {"useDatasets": self._use_datasets, "hids": list(self._hids.values()), "tour": self._tour}
def main(trans, webhook, params):
- error = ''
+ error = ""
data = {}
try:
- if not params or 'tool_id' not in params.keys():
- raise KeyError('Tool id is missing.')
+ if not params or "tool_id" not in params.keys():
+ raise KeyError("Tool id is missing.")
- if not params or 'tool_version' not in params.keys():
- raise KeyError('Tool version is missing.')
+ if not params or "tool_version" not in params.keys():
+ raise KeyError("Tool version is missing.")
- tool_id = params['tool_id']
- tool_version = params['tool_version']
+ tool_id = params["tool_id"]
+ tool_version = params["tool_version"]
tour_generator = TourGenerator(trans, tool_id, tool_version)
data = tour_generator.get_data()
@@ -291,4 +271,4 @@ def main(trans, webhook, params):
error = str(e)
log.exception(e)
- return {'success': not error, 'error': error, 'data': data}
+ return {"success": not error, "error": error, "data": data}
diff --git a/contrib/galaxy_config_merger.py b/contrib/galaxy_config_merger.py
index 887e43434f4..f7d60f183aa 100644
--- a/contrib/galaxy_config_merger.py
+++ b/contrib/galaxy_config_merger.py
@@ -1,5 +1,5 @@
#! /usr/bin/env python
-'''
+"""
galaxy_config_merger.py
Created by Anne Pajon on 31 Jan 2012
@@ -22,7 +22,7 @@ FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY OF
THE ORIGINAL WORK IS WITH YOU.
Script for merging specific local Galaxy config galaxy.ini.cri with default Galaxy galaxy.ini.sample
-'''
+"""
import configparser
import logging
@@ -32,16 +32,18 @@ import sys
def main():
# logging configuration
- logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
+ logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO)
# get the options
parser = optparse.OptionParser()
parser.add_option("-s", "--sample", dest="sample", action="store", help="path to Galaxy galaxy.ini.sample file")
parser.add_option("-c", "--config", dest="config", action="store", help="path to your own galaxy.ini file")
- parser.add_option("-o", "--output", dest="output", action="store", help="path to the new merged galaxy.ini.new file")
+ parser.add_option(
+ "-o", "--output", dest="output", action="store", help="path to the new merged galaxy.ini.new file"
+ )
(options, args) = parser.parse_args()
- for option in ['sample', 'config']:
+ for option in ["sample", "config"]:
if getattr(options, option) is None:
print("Please supply a --%s parameter.\n" % (option))
parser.print_help()
@@ -63,18 +65,24 @@ def main():
for (name, value) in config.items(section):
if not config_sample.has_option(section, name):
if not "#%s" % name in config_sample_content:
- logging.warning(f"-MISSING- section [{section}] option '{name}' not found in sample file. It will be ignored.")
+ logging.warning(
+ f"-MISSING- section [{section}] option '{name}' not found in sample file. It will be ignored."
+ )
else:
- logging.info(f"-notset- section [{section}] option '{name}' not set in sample file. It will be added.")
+ logging.info(
+ f"-notset- section [{section}] option '{name}' not set in sample file. It will be added."
+ )
config_sample.set(section, name, value)
else:
if not config_sample.get(section, name) == value:
- logging.info(f"- diff - section [{section}] option '{name}' has different value ('{config_sample.get(section, name)}':'{value}'). It will be modified.")
+ logging.info(
+ f"- diff - section [{section}] option '{name}' has different value ('{config_sample.get(section, name)}':'{value}'). It will be modified."
+ )
config_sample.set(section, name, value)
logging.info("---------- DIFFERENCE ANALYSIS END ----------")
if options.output:
- outputfile = open(options.output, 'w')
+ outputfile = open(options.output, "w")
config_sample.write(outputfile)
outputfile.close()
else:
@@ -83,5 +91,5 @@ def main():
logging.info("read Galaxy galaxy.ini.sample for detailed information.")
-if __name__ == '__main__':
+if __name__ == "__main__":
main()
diff --git a/cron/add_manual_builds.py b/cron/add_manual_builds.py
index 8a4197eef5b..5138442e145 100644
--- a/cron/add_manual_builds.py
+++ b/cron/add_manual_builds.py
@@ -21,7 +21,7 @@ def add_manual_builds(input_file, build_file, chr_dir):
existing_builds.append(line.replace("\n", "").replace("\r", "").split("\t")[0])
except Exception:
continue
- build_file_out = open(build_file, 'a')
+ build_file_out = open(build_file, "a")
for line in open(input_file):
try:
fields = line.replace("\n", "").replace("\r", "").split("\t")
@@ -35,7 +35,7 @@ def add_manual_builds(input_file, build_file, chr_dir):
chrs = []
print(build + "\t" + name + " (" + build + ")", file=build_file_out)
if chrs: # create len file if provided chrom lens
- chr_len_out = open(os.path.join(chr_dir, build + ".len"), 'w')
+ chr_len_out = open(os.path.join(chr_dir, build + ".len"), "w")
for chr in chrs:
print(chr.replace("=", "\t"), file=chr_len_out)
chr_len_out.close()
diff --git a/cron/build_chrom_db.py b/cron/build_chrom_db.py
index 7289dc72202..615e791e5c2 100644
--- a/cron/build_chrom_db.py
+++ b/cron/build_chrom_db.py
@@ -17,24 +17,26 @@ import os
import sys
from urllib.parse import urlencode
-import requests
-
import parse_builds # noqa: I100,I202
+import requests
def getchrominfo(url, db):
tableURL = "http://genome-test.gi.ucsc.edu/cgi-bin/hgTables?"
- URL = tableURL + urlencode({
- "clade": "",
- "org": "",
- "db": db,
- "hgta_outputType": "primaryTable",
- "hgta_group": "allTables",
- "hgta_table": "chromInfo",
- "hgta_track": db,
- "hgta_regionType": "",
- "position": "",
- "hgta_doTopSubmit": "get info"})
+ URL = tableURL + urlencode(
+ {
+ "clade": "",
+ "org": "",
+ "db": db,
+ "hgta_outputType": "primaryTable",
+ "hgta_group": "allTables",
+ "hgta_table": "chromInfo",
+ "hgta_track": db,
+ "hgta_regionType": "",
+ "position": "",
+ "hgta_doTopSubmit": "get info",
+ }
+ )
page = requests.get(URL).text
for i, line in enumerate(page.splitlines()):
line = line.rstrip("\r\n")
diff --git a/cron/cleanup_datasets.py b/cron/cleanup_datasets.py
index 184be129957..f20362d6b36 100644
--- a/cron/cleanup_datasets.py
+++ b/cron/cleanup_datasets.py
@@ -2,5 +2,7 @@
import sys
-sys.exit("This script has been deprecated, replaced by the set of scripts in /scripts/cleanup_datsets/."
- "See https://wiki.galaxyproject.org/Admin/Config/Performance/Purge%20Histories%20and%20Datasets for more information.")
+sys.exit(
+ "This script has been deprecated, replaced by the set of scripts in /scripts/cleanup_datsets/."
+ "See https://wiki.galaxyproject.org/Admin/Config/Performance/Purge%20Histories%20and%20Datasets for more information."
+)
diff --git a/cron/parse_builds.py b/cron/parse_builds.py
index 45361c3aeeb..dcd2005c537 100644
--- a/cron/parse_builds.py
+++ b/cron/parse_builds.py
@@ -29,7 +29,7 @@ def getbuilds(url):
print("#Harvested from " + url)
print("?\tunspecified (?)")
for dsn in tree:
- build = dsn.find("SOURCE").attrib['id']
+ build = dsn.find("SOURCE").attrib["id"]
description = dsn.find("DESCRIPTION").text.replace(" - Genome at UCSC", "").replace(" Genome at UCSC", "")
fields = description.split(" ")
diff --git a/cron/parse_builds_3_sites.py b/cron/parse_builds_3_sites.py
index c1eb4d0a147..52ad1bd6acd 100644
--- a/cron/parse_builds_3_sites.py
+++ b/cron/parse_builds_3_sites.py
@@ -7,10 +7,12 @@ import xml.etree.ElementTree as ElementTree
import requests
-sites = ['http://genome.ucsc.edu/cgi-bin/',
- 'http://archaea.ucsc.edu/cgi-bin/',
- 'http://genome-test.gi.ucsc.edu/cgi-bin/']
-names = ['main', 'archaea', 'test']
+sites = [
+ "http://genome.ucsc.edu/cgi-bin/",
+ "http://archaea.ucsc.edu/cgi-bin/",
+ "http://genome-test.gi.ucsc.edu/cgi-bin/",
+]
+names = ["main", "archaea", "test"]
def main():
@@ -32,7 +34,7 @@ def main():
print("#Harvested from", site)
for dsn in tree:
- build = dsn.find("SOURCE").attrib['id']
+ build = dsn.find("SOURCE").attrib["id"]
builds.append(build)
build_dict = {}
for build in builds:
diff --git a/doc/parse_gx_xsd.py b/doc/parse_gx_xsd.py
index 382c7eee8ce..a74557fdf20 100644
--- a/doc/parse_gx_xsd.py
+++ b/doc/parse_gx_xsd.py
@@ -28,7 +28,7 @@ def main():
toc_list.append(tag.build_toc_entry())
content_list.append(tag.build_help())
elif not found_tag:
- print(line, end='')
+ print(line, end="")
else:
raise Exception("No normal text allowed after the first $tag")
print("## Contents\n")
@@ -36,11 +36,10 @@ def main():
print(el)
print("\n")
for el in content_list:
- print(el, end='')
+ print(el, end="")
class Tag:
-
def __init__(self, line):
assert line.startswith("$tag:")
line_parts = line.split(" ")
@@ -60,9 +59,9 @@ class Tag:
@property
def _anchor(self):
anchor = self.title
- for _ in ['|', '_']:
- anchor = anchor.replace(_, '-')
- return '#' + anchor
+ for _ in ["|", "_"]:
+ anchor = anchor.replace(_, "-")
+ return "#" + anchor
@property
def _pretty_title(self):
@@ -93,18 +92,24 @@ def _build_tag(tag, hide_attributes):
text = _replace_attribute_list(tag, text, attributes)
for line in text.splitlines():
if line.startswith("$assertions"):
- assertions_tag = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}complexType[@name='TestAssertions']")
+ assertions_tag = xmlschema_doc.find(
+ "//{http://www.w3.org/2001/XMLSchema}complexType[@name='TestAssertions']"
+ )
assertions_buffer = StringIO()
assertions_buffer.write(_doc_or_none(assertions_tag))
assertions_buffer.write("\n\n")
- assertion_groups = assertions_tag.xpath("xs:choice/xs:group", namespaces={'xs': 'http://www.w3.org/2001/XMLSchema'})
+ assertion_groups = assertions_tag.xpath(
+ "xs:choice/xs:group", namespaces={"xs": "http://www.w3.org/2001/XMLSchema"}
+ )
for group in assertion_groups:
- ref = group.attrib['ref']
+ ref = group.attrib["ref"]
assertion_tag = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}group[@name='" + ref + "']")
doc = _doc_or_none(assertion_tag)
assertions_buffer.write(f"### {doc}\n\n")
- elements = assertion_tag.findall("{http://www.w3.org/2001/XMLSchema}choice/{http://www.w3.org/2001/XMLSchema}element")
+ elements = assertion_tag.findall(
+ "{http://www.w3.org/2001/XMLSchema}choice/{http://www.w3.org/2001/XMLSchema}element"
+ )
for element in elements:
doc = _doc_or_none(element)
if doc is None:
@@ -121,9 +126,12 @@ def _build_tag(tag, hide_attributes):
best_practices = _get_bp_link(annotation_el)
if best_practices:
tag_help.write("\n\n### Best Practices\n")
- tag_help.write("""
+ tag_help.write(
+ """
Find the Intergalactic Utilities Commision suggested best practices for this
-element [here](%s).""" % best_practices)
+element [here](%s)."""
+ % best_practices
+ )
tag_help.write(_build_attributes_table(tag, attributes, hide_attributes))
return tag_help.getvalue()
@@ -139,7 +147,9 @@ def _replace_attribute_list(tag, text, attributes):
else:
attribute_names = attributes_str.split(",")
header_level = int(header_level)
- text = text.replace(line, _build_attributes_table(tag, attributes, attribute_names=attribute_names, header_level=header_level))
+ text = text.replace(
+ line, _build_attributes_table(tag, attributes, attribute_names=attribute_names, header_level=header_level)
+ )
return text
@@ -155,7 +165,7 @@ def _build_attributes_table(tag, attributes, hide_attributes=False, attribute_na
attribute_table = StringIO()
attribute_table.write("\n\n")
if attributes and not hide_attributes:
- header_prefix = '#' * header_level
+ header_prefix = "#" * header_level
attribute_table.write("\n%s Attributes\n\n" % header_prefix)
attribute_table.write("Attribute | Details | Required\n")
attribute_table.write("--- | --- | ---\n")
@@ -176,25 +186,42 @@ def _build_attributes_table(tag, attributes, hide_attributes=False, attribute_na
details = details.replace("\n", " ").strip()
best_practices = _get_bp_link(annotation_el)
if best_practices:
- details += """ Find the Intergalactic Utilities Commision suggested best practices for this element [here](%s).""" % best_practices
+ details += (
+ """ Find the Intergalactic Utilities Commision suggested best practices for this element [here](%s)."""
+ % best_practices
+ )
attribute_table.write(f"``{name}`` | {details} | {use}\n")
return attribute_table.getvalue()
def _find_attributes(tag):
- raw_attributes = tag.findall("{http://www.w3.org/2001/XMLSchema}attribute") or \
- tag.findall("{http://www.w3.org/2001/XMLSchema}complexType/{http://www.w3.org/2001/XMLSchema}attribute") or \
- tag.findall("{http://www.w3.org/2001/XMLSchema}complexContent/{http://www.w3.org/2001/XMLSchema}extension/{http://www.w3.org/2001/XMLSchema}attribute") or \
- tag.findall("{http://www.w3.org/2001/XMLSchema}simpleContent/{http://www.w3.org/2001/XMLSchema}extension/{http://www.w3.org/2001/XMLSchema}attribute")
- attribute_groups = tag.findall("{http://www.w3.org/2001/XMLSchema}attributeGroup") or \
- tag.findall("{http://www.w3.org/2001/XMLSchema}complexType/{http://www.w3.org/2001/XMLSchema}attributeGroup") or \
- tag.findall("{http://www.w3.org/2001/XMLSchema}complexContent/{http://www.w3.org/2001/XMLSchema}extension/{http://www.w3.org/2001/XMLSchema}attributeGroup") or \
- tag.findall("{http://www.w3.org/2001/XMLSchema}simpleContent/{http://www.w3.org/2001/XMLSchema}extension/{http://www.w3.org/2001/XMLSchema}attributeGroup")
+ raw_attributes = (
+ tag.findall("{http://www.w3.org/2001/XMLSchema}attribute")
+ or tag.findall("{http://www.w3.org/2001/XMLSchema}complexType/{http://www.w3.org/2001/XMLSchema}attribute")
+ or tag.findall(
+ "{http://www.w3.org/2001/XMLSchema}complexContent/{http://www.w3.org/2001/XMLSchema}extension/{http://www.w3.org/2001/XMLSchema}attribute"
+ )
+ or tag.findall(
+ "{http://www.w3.org/2001/XMLSchema}simpleContent/{http://www.w3.org/2001/XMLSchema}extension/{http://www.w3.org/2001/XMLSchema}attribute"
+ )
+ )
+ attribute_groups = (
+ tag.findall("{http://www.w3.org/2001/XMLSchema}attributeGroup")
+ or tag.findall("{http://www.w3.org/2001/XMLSchema}complexType/{http://www.w3.org/2001/XMLSchema}attributeGroup")
+ or tag.findall(
+ "{http://www.w3.org/2001/XMLSchema}complexContent/{http://www.w3.org/2001/XMLSchema}extension/{http://www.w3.org/2001/XMLSchema}attributeGroup"
+ )
+ or tag.findall(
+ "{http://www.w3.org/2001/XMLSchema}simpleContent/{http://www.w3.org/2001/XMLSchema}extension/{http://www.w3.org/2001/XMLSchema}attributeGroup"
+ )
+ )
attributes = list(raw_attributes)
for attribute_group in attribute_groups:
attribute_group_name = attribute_group.get("ref")
- attribute_group_def = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}attributeGroup/[@name='%s']" % attribute_group_name)
+ attribute_group_def = xmlschema_doc.find(
+ "//{http://www.w3.org/2001/XMLSchema}attributeGroup/[@name='%s']" % attribute_group_name
+ )
attributes.extend(_find_attributes(attribute_group_def))
return attributes
@@ -222,5 +249,5 @@ def _doc_or_none(tag):
return doc_el.text
-if __name__ == '__main__':
+if __name__ == "__main__":
main()
diff --git a/doc/source/conf.py b/doc/source/conf.py
index c983beef6a1..8e364417cb7 100644
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -15,6 +15,7 @@ import os
import sys
import sphinx_rtd_theme
+
# Library to make .md to slideshow
from recommonmark.transform import AutoStructify
@@ -30,35 +31,35 @@ SKIP_RELEASES = os.environ.get("GALAXY_DOCS_SKIP_RELEASES", False) == "1"
# REQUIRED GALAXY INCLUDES
-sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib')))
+sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "lib")))
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
-#sys.path.insert(0, os.path.abspath('.'))
+# sys.path.insert(0, os.path.abspath('.'))
# -- General configuration -----------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
-#needs_sphinx = '1.0'
+# needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['recommonmark', 'sphinx.ext.intersphinx', 'sphinx_markdown_tables']
+extensions = ["recommonmark", "sphinx.ext.intersphinx", "sphinx_markdown_tables"]
if not SKIP_SOURCE:
# TODO: Add https://pypi.org/project/sphinx-autodoc-typehints
- extensions += ['sphinx.ext.doctest', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.autodoc']
+ extensions += ["sphinx.ext.doctest", "sphinx.ext.todo", "sphinx.ext.coverage", "sphinx.ext.autodoc"]
if not SKIP_VIEW_CODE:
- extensions.append('sphinx.ext.viewcode')
+ extensions.append("sphinx.ext.viewcode")
# Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
# Configure default autodoc's action
-autodoc_default_flags = ['members', 'undoc-members']
+autodoc_default_flags = ["members", "undoc-members"]
# Prevent alphabetical reordering of module members.
-autodoc_member_order = 'bysource'
+autodoc_member_order = "bysource"
def dont_skip_init(app, what, name, obj, skip, options):
@@ -70,26 +71,30 @@ def dont_skip_init(app, what, name, obj, skip, options):
def setup(app):
if not SKIP_SOURCE:
app.connect("autodoc-skip-member", dont_skip_init)
- app.add_config_value('recommonmark_config', {
- 'enable_auto_doc_ref': False,
- 'enable_auto_toc_tree': False,
- 'enable_inline_math': False, # https://github.com/rtfd/recommonmark/pull/124
- }, True)
+ app.add_config_value(
+ "recommonmark_config",
+ {
+ "enable_auto_doc_ref": False,
+ "enable_auto_toc_tree": False,
+ "enable_inline_math": False, # https://github.com/rtfd/recommonmark/pull/124
+ },
+ True,
+ )
app.add_transform(AutoStructify)
# The suffix of source filenames.
-source_suffix = ['.rst', '.md']
+source_suffix = [".rst", ".md"]
# The encoding of source files.
-#source_encoding = 'utf-8-sig'
+# source_encoding = 'utf-8-sig'
# The master toctree document.
-master_doc = 'index'
+master_doc = "index"
# General information about the project.
-project = 'Galaxy Project'
-copyright = str(datetime.datetime.now().year) + ', Galaxy Committers'
+project = "Galaxy Project"
+copyright = str(datetime.datetime.now().year) + ", Galaxy Committers"
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
@@ -97,33 +102,34 @@ copyright = str(datetime.datetime.now().year) + ', Galaxy Committers'
#
# The short X.Y version.
from galaxy.version import VERSION, VERSION_MAJOR
+
version = VERSION_MAJOR
# The full version, including alpha/beta/rc tags.
release = VERSION
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
-#language = None
+# language = None
# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
-#today = ''
+# today = ''
# Else, today_fmt is used as the format for a strftime call.
-#today_fmt = '%B %d, %Y'
+# today_fmt = '%B %d, %Y'
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
-exclude_patterns = ['**/_*.rst']
+exclude_patterns = ["**/_*.rst"]
if SKIP_SOURCE:
- exclude_patterns.extend(['lib'])
+ exclude_patterns.extend(["lib"])
if SKIP_RELEASES:
- exclude_patterns.extend(['releases'])
+ exclude_patterns.extend(["releases"])
# The reST default role (used for this markup: `text`) to use for all documents.
-#default_role = None
+# default_role = None
# If true, '()' will be appended to :func: etc. cross-reference text.
-#add_function_parentheses = True
+# add_function_parentheses = True
# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
@@ -131,34 +137,34 @@ add_module_names = True
# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
-#show_authors = False
+# show_authors = False
# The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = "sphinx"
# A list of ignored prefixes for module index sorting.
-#modindex_common_prefix = []
+# modindex_common_prefix = []
# Intersphinx mapping to Python documentation
intersphinx_mapping = {
- 'python': ('https://docs.python.org/3', None),
- 'requests': ("https://requests.readthedocs.io/en/master/", None),
+ "python": ("https://docs.python.org/3", None),
+ "requests": ("https://requests.readthedocs.io/en/master/", None),
}
# -- Options for HTML output ---------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
-html_theme = 'sphinx_rtd_theme'
+html_theme = "sphinx_rtd_theme"
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
html_theme_options = {
- 'collapse_navigation': False,
- 'display_version': True,
- 'navigation_depth': 2,
- 'canonical_url': 'https://docs.galaxyproject.org/en/master/',
+ "collapse_navigation": False,
+ "display_version": True,
+ "navigation_depth": 2,
+ "canonical_url": "https://docs.galaxyproject.org/en/master/",
}
# Add any paths that contain custom themes here, relative to this directory.
@@ -166,68 +172,68 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
# The name for this set of Sphinx documents. If None, it defaults to
# " v documentation".
-#html_title = None
+# html_title = None
# A shorter title for the navigation bar. Default is the same as html_title.
-#html_short_title = None
+# html_short_title = None
# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
-#html_logo = None
+# html_logo = None
# The name of an image file (within the static path) to use as favicon of the
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
-#html_favicon = None
+# html_favicon = None
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
-#html_last_updated_fmt = '%b %d, %Y'
+# html_last_updated_fmt = '%b %d, %Y'
# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
-#html_use_smartypants = True
+# html_use_smartypants = True
# Custom sidebar templates, maps document names to template names.
-#html_sidebars = {}
+# html_sidebars = {}
# Additional templates that should be rendered to pages, maps page names to
# template names.
-#html_additional_pages = {}
+# html_additional_pages = {}
# If false, no module index is generated.
-#html_domain_indices = True
+# html_domain_indices = True
# If false, no index is generated.
-#html_use_index = True
+# html_use_index = True
# If true, the index is split into individual pages for each letter.
-#html_split_index = False
+# html_split_index = False
# If true, links to the reST sources are added to the pages.
-#html_show_sourcelink = True
+# html_show_sourcelink = True
# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
-#html_show_sphinx = True
+# html_show_sphinx = True
# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
-#html_show_copyright = True
+# html_show_copyright = True
# If true, an OpenSearch description file will be output, and all pages will
# contain a tag referring to it. The value of this option must be the
# base URL from which the finished HTML is served.
-#html_use_opensearch = ''
+# html_use_opensearch = ''
# This is the file name suffix for HTML files (e.g. ".xhtml").
-#html_file_suffix = None
+# html_file_suffix = None
# Output file base name for HTML help builder.
-htmlhelp_basename = 'Galaxydoc'
+htmlhelp_basename = "Galaxydoc"
# -- Options for LaTeX output --------------------------------------------------
@@ -235,10 +241,8 @@ htmlhelp_basename = 'Galaxydoc'
latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',
-
# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',
-
# Additional stuff for the LaTeX preamble.
#'preamble': '',
}
@@ -246,42 +250,38 @@ latex_elements = {
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
- ('index', 'Galaxy.tex', 'Galaxy Code Documentation',
- 'Galaxy Team', 'manual'),
+ ("index", "Galaxy.tex", "Galaxy Code Documentation", "Galaxy Team", "manual"),
]
# The name of an image file (relative to this directory) to place at the top of
# the title page.
-#latex_logo = None
+# latex_logo = None
# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
-#latex_use_parts = False
+# latex_use_parts = False
# If true, show page references after internal links.
-#latex_show_pagerefs = False
+# latex_show_pagerefs = False
# If true, show URL addresses after external links.
-#latex_show_urls = False
+# latex_show_urls = False
# Documents to append as an appendix to all manuals.
-#latex_appendices = []
+# latex_appendices = []
# If false, no module index is generated.
-#latex_domain_indices = True
+# latex_domain_indices = True
# -- Options for manual page output --------------------------------------------
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
-man_pages = [
- ('index', 'galaxy', 'Galaxy Documentation',
- ['Galaxy Team'], 1)
-]
+man_pages = [("index", "galaxy", "Galaxy Documentation", ["Galaxy Team"], 1)]
# If true, show URL addresses after external links.
-#man_show_urls = False
+# man_show_urls = False
# -- Options for Texinfo output ------------------------------------------------
@@ -290,19 +290,25 @@ man_pages = [
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
- ('index', 'Galaxy', 'Galaxy Documentation',
- 'Galaxy Team', 'Galaxy', 'Data intensive biology for everyone.',
- 'Miscellaneous'),
+ (
+ "index",
+ "Galaxy",
+ "Galaxy Documentation",
+ "Galaxy Team",
+ "Galaxy",
+ "Data intensive biology for everyone.",
+ "Miscellaneous",
+ ),
]
# Documents to append as an appendix to all manuals.
-#texinfo_appendices = []
+# texinfo_appendices = []
# If false, no module index is generated.
-#texinfo_domain_indices = True
+# texinfo_domain_indices = True
# How to display URL addresses: 'footnote', 'no', or 'inline'.
-#texinfo_show_urls = 'footnote'
+# texinfo_show_urls = 'footnote'
# -- ReadTheDocs.org Settings ------------------------------------------------
@@ -315,8 +321,8 @@ class Mock:
@classmethod
def __getattr__(cls, name):
- if name in ('__file__', '__path__'):
- return '/dev/null'
+ if name in ("__file__", "__path__"):
+ return "/dev/null"
elif name[0] == name[0].upper():
mockType = type(name, (), {})
mockType.__module__ = __name__
@@ -326,6 +332,6 @@ class Mock:
# adding pbs_python, DRMAA_python, markupsafe, and drmaa here had no effect.
-MOCK_MODULES = ['tables', 'decorator']
+MOCK_MODULES = ["tables", "decorator"]
for mod_name in MOCK_MODULES:
sys.modules[mod_name] = Mock()
diff --git a/doc/source/conf.versioning.py b/doc/source/conf.versioning.py
index fb8c7eb94b1..3e45f6db296 100644
--- a/doc/source/conf.versioning.py
+++ b/doc/source/conf.versioning.py
@@ -7,7 +7,7 @@ from distutils.version import LooseVersion
from subprocess import check_output
# This is set in the Jenkins matrix config
-TARGET_GIT_BRANCH = os.environ.get('TARGET_BRANCH', 'dev') # noqa: F821
+TARGET_GIT_BRANCH = os.environ.get("TARGET_BRANCH", "dev") # noqa: F821
# Version message templates
OLD_BANNER = """This document is for an old release of Galaxy."""
@@ -17,58 +17,51 @@ BANNER_APPEND = """ You can alternatively view this pa
exists or view the top of the latest release's documentation."""
# Minimum version for linking to docs
-MIN_DOC_VERSION = LooseVersion('17.05')
+MIN_DOC_VERSION = LooseVersion("17.05")
# Enable simpleversioning
-extensions += ['sphinxcontrib.simpleversioning'] # noqa: F821
+extensions += ["sphinxcontrib.simpleversioning"] # noqa: F821
# -- sphinxcontrib-simpleversioning Settings ---------------------------------
-simpleversioning_path_template = '/en/{version}/{pagename}'
-simpleversioning_stable_version = 'master'
+simpleversioning_path_template = "/en/{version}/{pagename}"
+simpleversioning_stable_version = "master"
simpleversioning_current_version = TARGET_GIT_BRANCH
simpleversioning_versions = [
- {'id': 'latest', 'name': 'dev'},
- {'id': 'master', 'name': 'stable'},
+ {"id": "latest", "name": "dev"},
+ {"id": "master", "name": "stable"},
# Additional versions added below
]
# Used for determining the latest stable release so the banner can be added to older releases.
_stable = None
# Use tags to determine versions - a stable version will have a branch before it's released, but not a tag.
-tags = check_output(('git', 'tag')).decode().splitlines()
+tags = check_output(("git", "tag")).decode().splitlines()
for _tag in reversed(tags):
- if _tag.startswith('v') and _tag.count('.') == 1:
+ if _tag.startswith("v") and _tag.count(".") == 1:
# this version is released
_ver = _tag[1:]
if not _stable:
_stable = _ver
if LooseVersion(_ver) >= MIN_DOC_VERSION:
- simpleversioning_versions.append(
- {'id': 'release_%s' % _ver, 'name': _ver}
- )
+ simpleversioning_versions.append({"id": "release_%s" % _ver, "name": _ver})
-if re.fullmatch(r'release_\d{2}\.\d{2}', TARGET_GIT_BRANCH):
+if re.fullmatch(r"release_\d{2}\.\d{2}", TARGET_GIT_BRANCH):
if _stable:
# The current stable release will go here but fail the next conditional, avoiding either banner.
- if TARGET_GIT_BRANCH != 'release_%s' % _stable:
+ if TARGET_GIT_BRANCH != "release_%s" % _stable:
simpleversioning_show_banner = True
- _target_ver = TARGET_GIT_BRANCH[len('release_'):]
+ _target_ver = TARGET_GIT_BRANCH[len("release_") :]
if LooseVersion(_target_ver) > LooseVersion(_stable):
# Pre-release
# Insert it between master and _stable
- simpleversioning_versions.insert(
- 2,
- {'id': TARGET_GIT_BRANCH, 'name': _target_ver}
- )
+ simpleversioning_versions.insert(2, {"id": TARGET_GIT_BRANCH, "name": _target_ver})
simpleversioning_banner_message = PRE_BANNER + BANNER_APPEND
else:
simpleversioning_banner_message = OLD_BANNER + BANNER_APPEND
-elif TARGET_GIT_BRANCH != 'master':
- if TARGET_GIT_BRANCH != 'dev':
+elif TARGET_GIT_BRANCH != "master":
+ if TARGET_GIT_BRANCH != "dev":
# Feature branch
- simpleversioning_versions.append(
- {'id': TARGET_GIT_BRANCH, 'name': TARGET_GIT_BRANCH}
- )
+ simpleversioning_versions.append({"id": TARGET_GIT_BRANCH, "name": TARGET_GIT_BRANCH})
simpleversioning_show_banner = True
simpleversioning_banner_message = DEV_BANNER + BANNER_APPEND
diff --git a/lib/galaxy/__init__.py b/lib/galaxy/__init__.py
index 8c478621de6..70b3b112b71 100644
--- a/lib/galaxy/__init__.py
+++ b/lib/galaxy/__init__.py
@@ -3,4 +3,5 @@ Galaxy root package -- this is a namespace package.
"""
from pkgutil import extend_path
+
__path__ = extend_path(__path__, __name__) # type: ignore[has-type]
diff --git a/lib/galaxy/actions/library.py b/lib/galaxy/actions/library.py
index 04be31f36ab..fdd365f0db2 100644
--- a/lib/galaxy/actions/library.py
+++ b/lib/galaxy/actions/library.py
@@ -22,19 +22,19 @@ from galaxy.tools.parameters import populate_state
from galaxy.util.path import (
safe_contains,
safe_relpath,
- unsafe_walk
+ unsafe_walk,
)
log = logging.getLogger(__name__)
def validate_server_directory_upload(trans, server_dir):
- if server_dir in [None, 'None', '']:
+ if server_dir in [None, "None", ""]:
raise RequestParameterInvalidException("Invalid or unspecified server_dir parameter")
if trans.user_is_admin:
import_dir = trans.app.config.library_import_dir
- import_dir_desc = 'library_import_dir'
+ import_dir_desc = "library_import_dir"
if not import_dir:
raise ConfigDoesNotAllowException('"library_import_dir" is not set in the Galaxy configuration')
else:
@@ -43,17 +43,31 @@ def validate_server_directory_upload(trans, server_dir):
raise ConfigDoesNotAllowException('"user_library_import_dir" is not set in the Galaxy configuration')
if server_dir != trans.user.email:
import_dir = os.path.join(import_dir, trans.user.email)
- import_dir_desc = 'user_library_import_dir'
+ import_dir_desc = "user_library_import_dir"
full_dir = os.path.join(import_dir, server_dir)
unsafe = None
if safe_relpath(server_dir):
username = trans.user.username if trans.app.config.user_library_import_check_permissions else None
- if import_dir_desc == 'user_library_import_dir' and safe_contains(import_dir, full_dir, allowlist=trans.app.config.user_library_import_symlink_allowlist):
- for unsafe in unsafe_walk(full_dir, allowlist=[import_dir] + trans.app.config.user_library_import_symlink_allowlist, username=username):
- log.error('User attempted to import a path that resolves to a path outside of their import dir: %s -> %s', unsafe, os.path.realpath(unsafe))
+ if import_dir_desc == "user_library_import_dir" and safe_contains(
+ import_dir, full_dir, allowlist=trans.app.config.user_library_import_symlink_allowlist
+ ):
+ for unsafe in unsafe_walk(
+ full_dir,
+ allowlist=[import_dir] + trans.app.config.user_library_import_symlink_allowlist,
+ username=username,
+ ):
+ log.error(
+ "User attempted to import a path that resolves to a path outside of their import dir: %s -> %s",
+ unsafe,
+ os.path.realpath(unsafe),
+ )
else:
- log.error('User attempted to import a directory path that resolves to a path outside of their import dir: %s -> %s', server_dir, os.path.realpath(full_dir))
+ log.error(
+ "User attempted to import a directory path that resolves to a path outside of their import dir: %s -> %s",
+ server_dir,
+ os.path.realpath(full_dir),
+ )
unsafe = True
if unsafe:
raise RequestParameterInvalidException("Invalid server_dir specified")
@@ -66,7 +80,7 @@ def validate_path_upload(trans):
raise ConfigDoesNotAllowException('"allow_path_paste" is not set to True in the Galaxy configuration file')
if not trans.user_is_admin:
- raise AdminRequiredException('Uploading files via filesystem paths can only be performed by administrators')
+ raise AdminRequiredException("Uploading files via filesystem paths can only be performed by administrators")
class LibraryActions:
@@ -76,10 +90,10 @@ class LibraryActions:
def _upload_dataset(self, trans, folder_id: str, replace_dataset: Optional[LibraryDataset] = None, **kwd):
# Set up the traditional tool state/params
- cntrller = 'api'
- tool_id = 'upload1'
+ cntrller = "api"
+ tool_id = "upload1"
message = None
- file_type = kwd.get('file_type')
+ file_type = kwd.get("file_type")
try:
upload_common.validate_datatype_extension(datatypes_registry=trans.app.datatypes_registry, ext=file_type)
except RequestParameterInvalidException as e:
@@ -93,13 +107,13 @@ class LibraryActions:
if input.type == "upload_dataset":
dataset_upload_inputs.append(input)
# Library-specific params
- server_dir = kwd.get('server_dir', '')
- upload_option = kwd.get('upload_option', 'upload_file')
+ server_dir = kwd.get("server_dir", "")
+ upload_option = kwd.get("upload_option", "upload_file")
response_code = 200
- if upload_option == 'upload_directory':
+ if upload_option == "upload_directory":
full_dir, import_dir_desc = validate_server_directory_upload(trans, server_dir)
- message = 'Select a directory'
- elif upload_option == 'upload_paths':
+ message = "Select a directory"
+ elif upload_option == "upload_paths":
# Library API already checked this - following check isn't actually needed.
validate_path_upload(trans)
# Some error handling should be added to this method.
@@ -112,28 +126,38 @@ class LibraryActions:
message = "Unable to parse upload parameters, please report this error."
# Proceed with (mostly) regular upload processing if we're still errorless
if response_code == 200:
- if upload_option == 'upload_file':
+ if upload_option == "upload_file":
tool_params = upload_common.persist_uploads(tool_params, trans)
- uploaded_datasets = upload_common.get_uploaded_datasets(trans, cntrller, tool_params, dataset_upload_inputs, library_bunch=library_bunch)
- elif upload_option == 'upload_directory':
- uploaded_datasets, response_code, message = self._get_server_dir_uploaded_datasets(trans, kwd, full_dir, import_dir_desc, library_bunch, response_code, message)
- elif upload_option == 'upload_paths':
- uploaded_datasets, response_code, message = self._get_path_paste_uploaded_datasets(trans, kwd, library_bunch, response_code, message)
- if upload_option == 'upload_file' and not uploaded_datasets:
+ uploaded_datasets = upload_common.get_uploaded_datasets(
+ trans, cntrller, tool_params, dataset_upload_inputs, library_bunch=library_bunch
+ )
+ elif upload_option == "upload_directory":
+ uploaded_datasets, response_code, message = self._get_server_dir_uploaded_datasets(
+ trans, kwd, full_dir, import_dir_desc, library_bunch, response_code, message
+ )
+ elif upload_option == "upload_paths":
+ uploaded_datasets, response_code, message = self._get_path_paste_uploaded_datasets(
+ trans, kwd, library_bunch, response_code, message
+ )
+ if upload_option == "upload_file" and not uploaded_datasets:
response_code = 400
- message = 'Select a file, enter a URL or enter text'
+ message = "Select a file, enter a URL or enter text"
if response_code != 200:
return (response_code, message)
json_file_path = upload_common.create_paramfile(trans, uploaded_datasets)
data_list = [ud.data for ud in uploaded_datasets]
job_params = {}
- job_params['link_data_only'] = json.dumps(kwd.get('link_data_only', 'copy_files'))
- job_params['uuid'] = json.dumps(kwd.get('uuid', None))
- job, output = upload_common.create_job(trans, tool_params, tool, json_file_path, data_list, folder=library_bunch.folder, job_params=job_params)
+ job_params["link_data_only"] = json.dumps(kwd.get("link_data_only", "copy_files"))
+ job_params["uuid"] = json.dumps(kwd.get("uuid", None))
+ job, output = upload_common.create_job(
+ trans, tool_params, tool, json_file_path, data_list, folder=library_bunch.folder, job_params=job_params
+ )
trans.app.job_manager.enqueue(job, tool=tool)
return output
- def _get_server_dir_uploaded_datasets(self, trans, params, full_dir, import_dir_desc, library_bunch, response_code, message):
+ def _get_server_dir_uploaded_datasets(
+ self, trans, params, full_dir, import_dir_desc, library_bunch, response_code, message
+ ):
dir_response = self._get_server_dir_files(params, full_dir, import_dir_desc)
files = dir_response[0]
if not files:
@@ -141,7 +165,9 @@ class LibraryActions:
uploaded_datasets = []
for file in files:
name = os.path.basename(file)
- uploaded_datasets.append(self._make_library_uploaded_dataset(trans, params, name, file, 'server_dir', library_bunch))
+ uploaded_datasets.append(
+ self._make_library_uploaded_dataset(trans, params, name, file, "server_dir", library_bunch)
+ )
return uploaded_datasets, 200, None
def _get_server_dir_files(self, params, full_dir, import_dir_desc):
@@ -150,8 +176,8 @@ class LibraryActions:
for entry in os.listdir(full_dir):
# Only import regular files
path = os.path.join(full_dir, entry)
- link_data_only = params.get('link_data_only', 'copy_files')
- if os.path.islink(full_dir) and link_data_only == 'link_to_files':
+ link_data_only = params.get("link_data_only", "copy_files")
+ if os.path.islink(full_dir) and link_data_only == "link_to_files":
# If we're linking instead of copying and the
# sub-"directory" in the import dir is actually a symlink,
# dereference the symlink, but not any of its contents.
@@ -160,7 +186,7 @@ class LibraryActions:
path = os.path.join(link_path, entry)
else:
path = os.path.abspath(os.path.join(link_path, entry))
- elif os.path.islink(path) and os.path.isfile(path) and link_data_only == 'link_to_files':
+ elif os.path.islink(path) and os.path.isfile(path) and link_data_only == "link_to_files":
# If we're linking instead of copying and the "file" in the
# sub-directory of the import dir is actually a symlink,
# dereference the symlink (one dereference only, Vasili).
@@ -182,13 +208,15 @@ class LibraryActions:
return files, None, None
def _get_path_paste_uploaded_datasets(self, trans, params, library_bunch, response_code, message):
- preserve_dirs = util.string_as_bool(params.get('preserve_dirs', False))
+ preserve_dirs = util.string_as_bool(params.get("preserve_dirs", False))
uploaded_datasets = []
(files_and_folders, _response_code, _message) = self._get_path_files_and_folders(params, preserve_dirs)
if _response_code:
return (uploaded_datasets, _response_code, _message)
for (path, name, folder) in files_and_folders:
- uploaded_datasets.append(self._make_library_uploaded_dataset(trans, params, name, path, 'path_paste', library_bunch, folder))
+ uploaded_datasets.append(
+ self._make_library_uploaded_dataset(trans, params, name, path, "path_paste", library_bunch, folder)
+ )
return uploaded_datasets, 200, None
def _get_path_files_and_folders(self, params, preserve_dirs):
@@ -210,17 +238,21 @@ class LibraryActions:
for file in files:
file_path = os.path.abspath(os.path.join(basedir, file))
if preserve_dirs:
- in_folder = os.path.dirname(file_path.replace(path, '', 1).lstrip('/'))
+ in_folder = os.path.dirname(file_path.replace(path, "", 1).lstrip("/"))
else:
in_folder = None
files_and_folders.append((file_path, file, in_folder))
return files_and_folders
def _paths_list(self, params):
- return [(line.strip(), os.path.abspath(line.strip())) for line in params.get('filesystem_paths', '').splitlines() if line.strip()]
+ return [
+ (line.strip(), os.path.abspath(line.strip()))
+ for line in params.get("filesystem_paths", "").splitlines()
+ if line.strip()
+ ]
def _check_path_paste_params(self, params):
- if params.get('filesystem_paths', '') == '':
+ if params.get("filesystem_paths", "") == "":
message = "No paths entered in the upload form"
response_code = 400
return None, response_code, message
@@ -235,36 +267,36 @@ class LibraryActions:
return None
def _make_library_uploaded_dataset(self, trans, params, name, path, type, library_bunch, in_folder=None):
- link_data_only = params.get('link_data_only', 'copy_files')
- uuid_str = params.get('uuid', None)
- file_type = params.get('file_type', None)
+ link_data_only = params.get("link_data_only", "copy_files")
+ uuid_str = params.get("uuid", None)
+ file_type = params.get("file_type", None)
library_bunch.replace_dataset = None # not valid for these types of upload
uploaded_dataset = util.bunch.Bunch()
new_name = name
# Remove compressed file extensions, if any, but only if
# we're copying files into Galaxy's file space.
- if link_data_only == 'copy_files':
- if new_name.endswith('.gz'):
- new_name = new_name.rstrip('.gz')
- elif new_name.endswith('.zip'):
- new_name = new_name.rstrip('.zip')
+ if link_data_only == "copy_files":
+ if new_name.endswith(".gz"):
+ new_name = new_name.rstrip(".gz")
+ elif new_name.endswith(".zip"):
+ new_name = new_name.rstrip(".zip")
uploaded_dataset.name = new_name
uploaded_dataset.path = path
uploaded_dataset.type = type
uploaded_dataset.ext = None
uploaded_dataset.file_type = file_type
- uploaded_dataset.dbkey = params.get('dbkey', None)
- uploaded_dataset.to_posix_lines = params.get('to_posix_lines', None)
- uploaded_dataset.space_to_tab = params.get('space_to_tab', None)
- uploaded_dataset.tag_using_filenames = params.get('tag_using_filenames', False)
- uploaded_dataset.tags = params.get('tags', None)
- uploaded_dataset.purge_source = getattr(trans.app.config, 'ftp_upload_purge', True)
+ uploaded_dataset.dbkey = params.get("dbkey", None)
+ uploaded_dataset.to_posix_lines = params.get("to_posix_lines", None)
+ uploaded_dataset.space_to_tab = params.get("space_to_tab", None)
+ uploaded_dataset.tag_using_filenames = params.get("tag_using_filenames", False)
+ uploaded_dataset.tags = params.get("tags", None)
+ uploaded_dataset.purge_source = getattr(trans.app.config, "ftp_upload_purge", True)
if in_folder:
uploaded_dataset.in_folder = in_folder
- uploaded_dataset.data = upload_common.new_upload(trans, 'api', uploaded_dataset, library_bunch)
+ uploaded_dataset.data = upload_common.new_upload(trans, "api", uploaded_dataset, library_bunch)
uploaded_dataset.link_data_only = link_data_only
uploaded_dataset.uuid = uuid_str
- if link_data_only == 'link_to_files':
+ if link_data_only == "link_to_files":
uploaded_dataset.data.link_to(path)
trans.sa_session.add_all((uploaded_dataset.data, uploaded_dataset.data.dataset))
trans.sa_session.flush()
@@ -274,15 +306,16 @@ class LibraryActions:
is_admin = trans.user_is_admin
current_user_roles = trans.get_current_user_roles()
try:
- parent_folder = trans.sa_session.query(trans.app.model.LibraryFolder).get(trans.security.decode_id(parent_id))
+ parent_folder = trans.sa_session.query(trans.app.model.LibraryFolder).get(
+ trans.security.decode_id(parent_id)
+ )
except Exception:
parent_folder = None
# Check the library which actually contains the user-supplied parent folder, not the user-supplied
# library, which could be anything.
self._check_access(trans, is_admin, parent_folder, current_user_roles)
self._check_add(trans, is_admin, parent_folder, current_user_roles)
- new_folder = trans.app.model.LibraryFolder(name=kwd.get('name', ''),
- description=kwd.get('description', ''))
+ new_folder = trans.app.model.LibraryFolder(name=kwd.get("name", ""), description=kwd.get("description", ""))
# We are associating the last used genome build with folders, so we will always
# initialize a new folder with the first dbkey in genome builds list which is currently
# ? unspecified (?)
@@ -300,7 +333,10 @@ class LibraryActions:
if not item:
message = f"Invalid history dataset ({escape(str(item))}) specified."
raise ObjectNotFound(message)
- elif not trans.app.security_agent.can_access_dataset(current_user_roles, item.dataset) and item.history.user == trans.user:
+ elif (
+ not trans.app.security_agent.can_access_dataset(current_user_roles, item.dataset)
+ and item.history.user == trans.user
+ ):
message = f"You do not have permission to access the history dataset with id ({str(item.id)})."
raise ItemAccessibilityException(message)
else:
@@ -308,13 +344,15 @@ class LibraryActions:
if not item:
message = f"Invalid library item ({escape(str(item))}) specified."
raise ObjectNotFound(message)
- elif not (is_admin or trans.app.security_agent.can_access_library_item(current_user_roles, item, trans.user)):
+ elif not (
+ is_admin or trans.app.security_agent.can_access_library_item(current_user_roles, item, trans.user)
+ ):
if isinstance(item, trans.model.Library):
- item_type = 'data library'
+ item_type = "data library"
elif isinstance(item, trans.model.LibraryFolder):
- item_type = 'folder'
+ item_type = "folder"
else:
- item_type = '(unknown item type)'
+ item_type = "(unknown item type)"
message = f"You do not have permission to access the {escape(item_type)} with id ({str(item.id)})."
raise ItemAccessibilityException(message)
diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py
index 0b131b4b3dc..f0647767282 100644
--- a/lib/galaxy/app.py
+++ b/lib/galaxy/app.py
@@ -2,13 +2,22 @@ import logging
import signal
import sys
import time
-from typing import Any, Callable, List, Tuple
+from typing import (
+ Any,
+ Callable,
+ List,
+ Tuple,
+)
import galaxy.model
import galaxy.model.security
import galaxy.queues
import galaxy.security
-from galaxy import auth, config, jobs
+from galaxy import (
+ auth,
+ config,
+ jobs,
+)
from galaxy.config_watchers import ConfigWatchers
from galaxy.containers import build_container_interfaces
from galaxy.datatypes.registry import Registry
@@ -43,11 +52,14 @@ from galaxy.queue_worker import (
GalaxyQueueWorker,
send_local_control_task,
)
-from galaxy.quota import get_quota_agent, QuotaAgent
+from galaxy.quota import (
+ get_quota_agent,
+ QuotaAgent,
+)
from galaxy.security.idencoding import IdEncodingHelper
from galaxy.security.vault import (
Vault,
- VaultFactory
+ VaultFactory,
)
from galaxy.tool_shed.galaxy_install.installed_repository_manager import InstalledRepositoryManager
from galaxy.tool_shed.galaxy_install.update_repository_manager import UpdateRepositoryManager
@@ -55,12 +67,15 @@ from galaxy.tool_util.deps.views import DependencyResolversView
from galaxy.tool_util.verify.test_data import TestDataResolver
from galaxy.tools.cache import (
ToolCache,
- ToolShedRepositoryCache
+ ToolShedRepositoryCache,
)
from galaxy.tools.data_manager.manager import DataManagers
from galaxy.tools.error_reports import ErrorReports
from galaxy.tools.special_tools import load_lib_tools
-from galaxy.tours import build_tours_registry, ToursRegistry
+from galaxy.tours import (
+ build_tours_registry,
+ ToursRegistry,
+)
from galaxy.util import (
ExecutionTimer,
heartbeat,
@@ -72,11 +87,18 @@ from galaxy.visualization.genomes import Genomes
from galaxy.visualization.plugins.registry import VisualizationsRegistry
from galaxy.web import url_for
from galaxy.web.proxy import ProxyManager
-from galaxy.web_stack import application_stack_instance, ApplicationStack
+from galaxy.web_stack import (
+ application_stack_instance,
+ ApplicationStack,
+)
from galaxy.webhooks import WebhooksRegistry
from galaxy.workflow.trs_proxy import TrsProxy
from .di import Container
-from .structured_app import BasicSharedApp, MinimalManagerApp, StructuredApp
+from .structured_app import (
+ BasicSharedApp,
+ MinimalManagerApp,
+ StructuredApp,
+)
log = logging.getLogger(__name__)
app = None
@@ -106,7 +128,13 @@ class SentryClientMixin:
self.sentry_client = None
if self.config.sentry_dsn:
event_level = self.config.sentry_event_level.upper()
- assert event_level in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], f"Invalid sentry event level '{self.config.sentry.event_level}'"
+ assert event_level in [
+ "DEBUG",
+ "INFO",
+ "WARNING",
+ "ERROR",
+ "CRITICAL",
+ ], f"Invalid sentry event level '{self.config.sentry.event_level}'"
def postfork_sentry_client():
import sentry_sdk
@@ -114,12 +142,12 @@ class SentryClientMixin:
sentry_logging = LoggingIntegration(
level=logging.INFO, # Capture info and above as breadcrumbs
- event_level=getattr(logging, event_level) # Send errors as events
+ event_level=getattr(logging, event_level), # Send errors as events
)
self.sentry_client = sentry_sdk.init(
self.config.sentry_dsn,
release=f"{self.config.version_major}.{self.config.version_minor}",
- integrations=[sentry_logging]
+ integrations=[sentry_logging],
)
self.application_stack.register_postfork_function(postfork_sentry_client)
@@ -141,7 +169,7 @@ class MinimalGalaxyApplication(BasicSharedApp, config.ConfiguresGalaxyMixin, Hal
# an appropriately configured logger in configure_logging below.
logging.basicConfig(level=logging.DEBUG)
log.debug("python path is: %s", ", ".join(sys.path))
- self.name = 'galaxy'
+ self.name = "galaxy"
self.is_webapp = False
self.new_installation = False
# Read config file and check for errors
@@ -150,7 +178,7 @@ class MinimalGalaxyApplication(BasicSharedApp, config.ConfiguresGalaxyMixin, Hal
if configure_logging:
config.configure_logging(self.config)
self._configure_object_store(fsmon=True)
- config_file = kwargs.get('global_conf', {}).get('__file__', None)
+ config_file = kwargs.get("global_conf", {}).get("__file__", None)
if config_file:
log.debug('Using "galaxy.ini" config file: %s', config_file)
self._configure_models(check_migrate_databases=self.config.check_migrate_databases, config_file=config_file)
@@ -165,7 +193,8 @@ class MinimalGalaxyApplication(BasicSharedApp, config.ConfiguresGalaxyMixin, Hal
def configure_fluent_log(self):
if self.config.fluent_log:
from galaxy.util.custom_logging.fluent_log import FluentTraceLogger
- self.trace_logger = FluentTraceLogger('galaxy', self.config.fluent_host, self.config.fluent_port)
+
+ self.trace_logger = FluentTraceLogger("galaxy", self.config.fluent_host, self.config.fluent_port)
else:
self.trace_logger = None
@@ -182,12 +211,16 @@ class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._register_singleton(MinimalManagerApp, self)
- self.execution_timer_factory = self._register_singleton(ExecutionTimerFactory, ExecutionTimerFactory(self.config))
+ self.execution_timer_factory = self._register_singleton(
+ ExecutionTimerFactory, ExecutionTimerFactory(self.config)
+ )
self.configure_fluent_log()
self.application_stack = self._register_singleton(ApplicationStack, application_stack_instance(app=self))
# Initialize job metrics manager, needs to be in place before
# config so per-destination modifications can be made.
- self.job_metrics = self._register_singleton(JobMetrics, JobMetrics(self.config.job_metrics_config_file, app=self))
+ self.job_metrics = self._register_singleton(
+ JobMetrics, JobMetrics(self.config.job_metrics_config_file, app=self)
+ )
# Initialize the job management configuration
self.job_config = self._register_singleton(jobs.JobConfiguration)
@@ -206,17 +239,22 @@ class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication):
self.library_datasets_manager = self._register_singleton(LibraryDatasetsManager)
self.role_manager = self._register_singleton(RoleManager)
from galaxy.jobs.manager import JobManager
+
self.job_manager = self._register_singleton(JobManager)
# ConfiguredFileSources
- self.file_sources = self._register_singleton(ConfiguredFileSources, ConfiguredFileSources.from_app_config(self.config))
+ self.file_sources = self._register_singleton(
+ ConfiguredFileSources, ConfiguredFileSources.from_app_config(self.config)
+ )
self.vault = self._register_singleton(Vault, VaultFactory.from_app(self))
# We need the datatype registry for running certain tasks that modify HDAs, and to build the registry we need
# to setup the installed repositories ... this is not ideal
self._configure_tool_config_files()
- self.installed_repository_manager = self._register_singleton(InstalledRepositoryManager, InstalledRepositoryManager(self))
+ self.installed_repository_manager = self._register_singleton(
+ InstalledRepositoryManager, InstalledRepositoryManager(self)
+ )
self._configure_datatypes_registry(self.installed_repository_manager)
self._register_singleton(Registry, self.datatypes_registry)
galaxy.model.set_datatypes_registry(self.datatypes_registry)
@@ -244,7 +282,7 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication):
self._register_singleton(StructuredApp, self)
# A lot of postfork initialization depends on the server name, ensure it is set immediately after forking before other postfork functions
self.application_stack.register_postfork_function(self.application_stack.set_postfork_server_name, self)
- self.config.reload_sanitize_allowlist(explicit='sanitize_allowlist_file' in kwargs)
+ self.config.reload_sanitize_allowlist(explicit="sanitize_allowlist_file" in kwargs)
self.amqp_internal_connection_obj = galaxy.queues.connection_from_config(self.config)
# queue_worker *can* be initialized with a queue, but here we don't
# want to and we'll allow postfork to bind and start it.
@@ -252,8 +290,12 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication):
self._configure_tool_shed_registry()
- self.dependency_resolvers_view = self._register_singleton(DependencyResolversView, DependencyResolversView(self))
- self.test_data_resolver = self._register_singleton(TestDataResolver, TestDataResolver(file_dirs=self.config.tool_test_data_directories))
+ self.dependency_resolvers_view = self._register_singleton(
+ DependencyResolversView, DependencyResolversView(self)
+ )
+ self.test_data_resolver = self._register_singleton(
+ TestDataResolver, TestDataResolver(file_dirs=self.config.tool_test_data_directories)
+ )
self.dynamic_tool_manager = self._register_singleton(DynamicToolManager)
self.api_keys_manager = self._register_singleton(ApiKeyManager)
@@ -268,7 +310,9 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication):
self.data_provider_registry = self._register_singleton(DataProviderRegistry)
# Initialize error report plugins.
- self.error_reports = self._register_singleton(ErrorReports, ErrorReports(self.config.error_report_file, app=self))
+ self.error_reports = self._register_singleton(
+ ErrorReports, ErrorReports(self.config.error_report_file, app=self)
+ )
# Setup a Tool Cache
self.tool_cache = self._register_singleton(ToolCache)
@@ -279,7 +323,9 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication):
# Load Data Manager
self.data_managers = self._register_singleton(DataManagers)
# Load the update repository manager.
- self.update_repository_manager = self._register_singleton(UpdateRepositoryManager, UpdateRepositoryManager(self))
+ self.update_repository_manager = self._register_singleton(
+ UpdateRepositoryManager, UpdateRepositoryManager(self)
+ )
# Load proprietary datatype converters and display applications.
self.installed_repository_manager.load_proprietary_converters_and_display_applications()
# Load datatype display applications defined in local datatypes_conf.xml
@@ -292,10 +338,14 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication):
load_lib_tools(self.toolbox)
self.toolbox.persist_cache(register_postfork=True)
# visualizations registry: associates resources with visualizations, controls how to render
- self.visualizations_registry = self._register_singleton(VisualizationsRegistry, VisualizationsRegistry(
- self,
- directories_setting=self.config.visualization_plugins_directory,
- template_cache_dir=self.config.template_cache_path))
+ self.visualizations_registry = self._register_singleton(
+ VisualizationsRegistry,
+ VisualizationsRegistry(
+ self,
+ directories_setting=self.config.visualization_plugins_directory,
+ template_cache_dir=self.config.template_cache_path,
+ ),
+ )
# Tours registry
tour_registry = build_tours_registry(self.config.tour_config_dir)
self.tour_registry = tour_registry
@@ -305,8 +355,8 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication):
# Load security policy.
self.security_agent = self.model.security_agent
self.host_security_agent = galaxy.model.security.HostAgent(
- model=self.security_agent.model,
- permitted_actions=self.security_agent.permitted_actions)
+ model=self.security_agent.model, permitted_actions=self.security_agent.permitted_actions
+ )
# Load quota management.
self.quota_agent = self._register_singleton(QuotaAgent, get_quota_agent(self.config, self.model))
# Heartbeat for thread profiling
@@ -317,9 +367,7 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication):
if self.config.use_heartbeat:
if heartbeat.Heartbeat:
self.heartbeat = heartbeat.Heartbeat(
- self.config,
- period=self.config.heartbeat_interval,
- fname=self.config.heartbeat_log
+ self.config, period=self.config.heartbeat_interval, fname=self.config.heartbeat_log
)
self.heartbeat.daemon = True
self.application_stack.register_postfork_function(self.heartbeat.start)
@@ -327,15 +375,15 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication):
self.authnz_manager = None
if self.config.enable_oidc:
from galaxy.authnz import managers
- self.authnz_manager = managers.AuthnzManager(self,
- self.config.oidc_config_file,
- self.config.oidc_backends_config_file)
+
+ self.authnz_manager = managers.AuthnzManager(
+ self, self.config.oidc_config_file, self.config.oidc_backends_config_file
+ )
self.containers = {}
if self.config.enable_beta_containers_interface:
self.containers = build_container_interfaces(
- self.config.containers_config_file,
- containers_conf=self.config.containers_conf
+ self.config.containers_config_file, containers_conf=self.config.containers_conf
)
if not self.config.enable_celery_tasks and self.config.history_audit_table_prune_interval > 0:
@@ -344,7 +392,8 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication):
name="HistoryAuditTablePruneTask",
interval=self.config.history_audit_table_prune_interval,
immediate_start=False,
- time_execution=True)
+ time_execution=True,
+ )
self.application_stack.register_postfork_function(self.prune_history_audit_task.start)
self.haltables.append(("HistoryAuditTablePruneTask", self.prune_history_audit_task.shutdown))
# Start the job manager
@@ -355,6 +404,7 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication):
self.proxy_manager = ProxyManager(self.config)
from galaxy.workflow import scheduling_manager
+
# Must be initialized after job_config.
self.workflow_scheduling_manager = scheduling_manager.WorkflowSchedulingManager(self)
@@ -372,9 +422,7 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication):
handlers[signal.SIGUSR1] = self.heartbeat.dump_signal_handler
self._configure_signal_handlers(handlers)
- self.database_heartbeat = DatabaseHeartbeat(
- application_stack=self.application_stack
- )
+ self.database_heartbeat = DatabaseHeartbeat(application_stack=self.application_stack)
self.database_heartbeat.add_change_callback(self.watchers.change_state)
self.application_stack.register_postfork_function(self.database_heartbeat.start)
@@ -382,7 +430,9 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication):
self.application_stack.register_postfork_function(self.application_stack.start)
self.application_stack.register_postfork_function(self.queue_worker.bind_and_start)
# Delay toolbox index until after startup
- self.application_stack.register_postfork_function(lambda: send_local_control_task(self, 'rebuild_toolbox_search_index'))
+ self.application_stack.register_postfork_function(
+ lambda: send_local_control_task(self, "rebuild_toolbox_search_index")
+ )
# Inject url_for for components to more easily optionally depend
# on url_for.
@@ -420,32 +470,33 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication):
@property
def is_job_handler(self) -> bool:
- return (self.config.track_jobs_in_database and self.job_config.is_handler) or not self.config.track_jobs_in_database
+ return (
+ self.config.track_jobs_in_database and self.job_config.is_handler
+ ) or not self.config.track_jobs_in_database
class StatsdStructuredExecutionTimer(StructuredExecutionTimer):
-
def __init__(self, galaxy_statsd_client, *args, **kwds):
self.galaxy_statsd_client = galaxy_statsd_client
super().__init__(*args, **kwds)
def to_str(self, **kwd):
- self.galaxy_statsd_client.timing(self.timer_id, self.elapsed * 1000., kwd)
+ self.galaxy_statsd_client.timing(self.timer_id, self.elapsed * 1000.0, kwd)
return super().to_str(**kwd)
class ExecutionTimerFactory:
-
def __init__(self, config):
statsd_host = getattr(config, "statsd_host", None)
if statsd_host:
from galaxy.web.framework.middleware.statsd import GalaxyStatsdClient
+
self.galaxy_statsd_client = GalaxyStatsdClient(
statsd_host,
- getattr(config, 'statsd_port', 8125),
- getattr(config, 'statsd_prefix', 'galaxy'),
- getattr(config, 'statsd_influxdb', False),
- getattr(config, 'statsd_mock_calls', False),
+ getattr(config, "statsd_port", 8125),
+ getattr(config, "statsd_prefix", "galaxy"),
+ getattr(config, "statsd_influxdb", False),
+ getattr(config, "statsd_mock_calls", False),
)
else:
self.galaxy_statsd_client = None
diff --git a/lib/galaxy/app_unittest_utils/celery_helper.py b/lib/galaxy/app_unittest_utils/celery_helper.py
index fa4657ef065..d70e0eafb94 100644
--- a/lib/galaxy/app_unittest_utils/celery_helper.py
+++ b/lib/galaxy/app_unittest_utils/celery_helper.py
@@ -3,6 +3,7 @@ from functools import wraps
def rebind_container_to_task(app):
import galaxy.app
+
galaxy.app.app = app
from galaxy.celery import (
CELERY_TASKS,
@@ -15,6 +16,6 @@ def rebind_container_to_task(app):
for task in CELERY_TASKS:
task_fn = getattr(tasks, task, None)
if task_fn:
- task_fn = getattr(task_fn, '__wrapped__', task_fn)
+ task_fn = getattr(task_fn, "__wrapped__", task_fn)
container_bound_task = magic_bind_dynamic(task_fn)
setattr(tasks, task, container_bound_task)
diff --git a/lib/galaxy/app_unittest_utils/galaxy_mock.py b/lib/galaxy/app_unittest_utils/galaxy_mock.py
index 98e3574b028..53ece3ff6fa 100644
--- a/lib/galaxy/app_unittest_utils/galaxy_mock.py
+++ b/lib/galaxy/app_unittest_utils/galaxy_mock.py
@@ -18,9 +18,16 @@ from galaxy.model import tags
from galaxy.model.base import SharedModelMapping
from galaxy.model.mapping import GalaxyModelMapping
from galaxy.model.scoped_session import galaxy_scoped_session
-from galaxy.model.unittest_utils import GalaxyDataTestApp, GalaxyDataTestConfig
+from galaxy.model.unittest_utils import (
+ GalaxyDataTestApp,
+ GalaxyDataTestConfig,
+)
from galaxy.security import idencoding
-from galaxy.structured_app import BasicSharedApp, MinimalManagerApp, StructuredApp
+from galaxy.structured_app import (
+ BasicSharedApp,
+ MinimalManagerApp,
+ StructuredApp,
+)
from galaxy.tool_util.deps.containers import NullContainerFinder
from galaxy.tools.data import ToolDataTableManager
from galaxy.util import StructuredExecutionTimer
@@ -33,34 +40,34 @@ from .celery_helper import rebind_container_to_task
# =============================================================================
def buildMockEnviron(**kwargs):
environ = {
- 'CONTENT_LENGTH': '0',
- 'CONTENT_TYPE': '',
- 'HTTP_ACCEPT': '*/*',
- 'HTTP_ACCEPT_ENCODING': 'gzip, deflate',
- 'HTTP_ACCEPT_LANGUAGE': 'en-US,en;q=0.8,zh;q=0.5,ja;q=0.3',
- 'HTTP_CACHE_CONTROL': 'no-cache',
- 'HTTP_CONNECTION': 'keep-alive',
- 'HTTP_DNT': '1',
- 'HTTP_HOST': 'localhost:8000',
- 'HTTP_ORIGIN': 'http://localhost:8000',
- 'HTTP_PRAGMA': 'no-cache',
- 'HTTP_REFERER': 'http://localhost:8000',
- 'HTTP_USER_AGENT': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:43.0) Gecko/20100101 Firefox/43.0',
- 'PATH_INFO': '/',
- 'QUERY_STRING': '',
- 'REMOTE_ADDR': '127.0.0.1',
- 'REQUEST_METHOD': 'GET',
- 'SCRIPT_NAME': '',
- 'SERVER_NAME': '127.0.0.1',
- 'SERVER_PORT': '8080',
- 'SERVER_PROTOCOL': 'HTTP/1.1'
+ "CONTENT_LENGTH": "0",
+ "CONTENT_TYPE": "",
+ "HTTP_ACCEPT": "*/*",
+ "HTTP_ACCEPT_ENCODING": "gzip, deflate",
+ "HTTP_ACCEPT_LANGUAGE": "en-US,en;q=0.8,zh;q=0.5,ja;q=0.3",
+ "HTTP_CACHE_CONTROL": "no-cache",
+ "HTTP_CONNECTION": "keep-alive",
+ "HTTP_DNT": "1",
+ "HTTP_HOST": "localhost:8000",
+ "HTTP_ORIGIN": "http://localhost:8000",
+ "HTTP_PRAGMA": "no-cache",
+ "HTTP_REFERER": "http://localhost:8000",
+ "HTTP_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:43.0) Gecko/20100101 Firefox/43.0",
+ "PATH_INFO": "/",
+ "QUERY_STRING": "",
+ "REMOTE_ADDR": "127.0.0.1",
+ "REQUEST_METHOD": "GET",
+ "SCRIPT_NAME": "",
+ "SERVER_NAME": "127.0.0.1",
+ "SERVER_PORT": "8080",
+ "SERVER_PROTOCOL": "HTTP/1.1",
}
environ.update(**kwargs)
return environ
class MockApp(di.Container, GalaxyDataTestApp):
- config: 'MockAppConfig'
+ config: "MockAppConfig"
def __init__(self, config=None, **kwargs):
super().__init__()
@@ -70,7 +77,7 @@ class MockApp(di.Container, GalaxyDataTestApp):
self[MinimalManagerApp] = self
self[StructuredApp] = self
self[idencoding.IdEncodingHelper] = self.security
- self.name = kwargs.get('name', 'galaxy')
+ self.name = kwargs.get("name", "galaxy")
self[SharedModelMapping] = self.model
self[GalaxyModelMapping] = self.model
self[galaxy_scoped_session] = self.model.context
@@ -79,10 +86,7 @@ class MockApp(di.Container, GalaxyDataTestApp):
self[tags.GalaxyTagHandler] = self.tag_handler
self.quota_agent = quota.DatabaseQuotaAgent(self.model)
self.job_config = Bunch(
- dynamic_params=None,
- destinations={},
- use_messaging=False,
- assign_handler=lambda *args, **kwargs: None
+ dynamic_params=None, destinations={}, use_messaging=False, assign_handler=lambda *args, **kwargs: None
)
self.tool_data_tables = ToolDataTableManager(tool_data_path=self.config.tool_data_path)
self.dataset_collections_service = None
@@ -103,6 +107,7 @@ class MockApp(di.Container, GalaxyDataTestApp):
def url_for(*args, **kwds):
return "/mock/url"
+
self.url_for = url_for
def wait_for_toolbox_reload(self, toolbox):
@@ -120,19 +125,18 @@ class MockLock:
class MockAppConfig(GalaxyDataTestConfig, CommonConfigurationMixin):
-
class MockSchema(Bunch):
pass
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.schema = self.MockSchema()
- self.use_remote_user = kwargs.get('use_remote_user', False)
+ self.use_remote_user = kwargs.get("use_remote_user", False)
self.enable_celery_tasks = False
- self.tool_data_path = os.path.join(self.root, 'tool-data')
+ self.tool_data_path = os.path.join(self.root, "tool-data")
self.galaxy_data_manager_data_path = self.tool_data_path
self.tool_dependency_dir = None
- self.metadata_strategy = 'directory'
+ self.metadata_strategy = "directory"
self.user_activation_on = False
self.new_user_dataset_access_role_default_private = False
@@ -152,8 +156,8 @@ class MockAppConfig(GalaxyDataTestConfig, CommonConfigurationMixin):
self.redact_email_in_job_name = False
# Follow two required by GenomeBuilds
- self.len_file_path = os.path.join('tool-data', 'shared', 'ucsc', 'chrom')
- self.builds_file_path = os.path.join('tool-data', 'shared', 'ucsc', 'builds.txt.sample')
+ self.len_file_path = os.path.join("tool-data", "shared", "ucsc", "chrom")
+ self.builds_file_path = os.path.join("tool-data", "shared", "ucsc", "builds.txt.sample")
self.shed_tool_config_file = "config/shed_tool_conf.xml"
self.shed_tool_config_file_set = False
@@ -165,7 +169,7 @@ class MockAppConfig(GalaxyDataTestConfig, CommonConfigurationMixin):
# set by MockDir
self.enable_tool_document_cache = False
- self.tool_cache_data_dir = os.path.join(self.root, 'tool_cache')
+ self.tool_cache_data_dir = os.path.join(self.root, "tool_cache")
self.delay_tool_initialization = True
self.external_chown_script = None
self.check_job_script_integrity = False
@@ -173,9 +177,9 @@ class MockAppConfig(GalaxyDataTestConfig, CommonConfigurationMixin):
self.check_job_script_integrity_sleep = 0
self.default_panel_view = "default"
- self.panel_views_dir = ''
+ self.panel_views_dir = ""
self.panel_views = {}
- self.edam_panel_views = ''
+ self.edam_panel_views = ""
self.config_file = None
@@ -189,7 +193,7 @@ class MockAppConfig(GalaxyDataTestConfig, CommonConfigurationMixin):
self.enable_tool_shed_check = False
self.monitor_thread_join_timeout = 1
self.integrated_tool_panel_config = None
- self.vault_config_file = kwargs.get('vault_config_file')
+ self.vault_config_file = kwargs.get("vault_config_file")
self.max_discovered_files = 10000
@property
@@ -198,23 +202,21 @@ class MockAppConfig(GalaxyDataTestConfig, CommonConfigurationMixin):
def __getattr__(self, name):
# Handle the automatic [option]_set options: for tests, assume none are set
- if name == 'is_set':
+ if name == "is_set":
return lambda x: False
# Handle the automatic config file _set options
- if name.endswith('_file_set'):
+ if name.endswith("_file_set"):
return False
raise AttributeError(name)
class MockWebapp:
-
def __init__(self, security: idencoding.IdEncodingHelper, **kwargs):
- self.name = kwargs.get('name', 'galaxy')
+ self.name = kwargs.get("name", "galaxy")
self.security = security
class MockTrans:
-
def __init__(self, app=None, user=None, history=None, **kwargs):
self.app = app or MockApp(**kwargs)
self.model = self.app.model
@@ -273,14 +275,13 @@ class MockTrans:
class MockVisualizationsRegistry:
- BUILT_IN_VISUALIZATIONS = ['trackster']
+ BUILT_IN_VISUALIZATIONS = ["trackster"]
def get_visualizations(self, trans, target):
return []
class MockDir:
-
def __init__(self, structure_dict, where=None):
self.structure_dict = structure_dict
self.create_root(structure_dict, where)
@@ -301,7 +302,7 @@ class MockDir:
self.create_structure(subdir_path, v)
def create_file(self, path, contents):
- with open(path, 'w') as newfile:
+ with open(path, "w") as newfile:
newfile.write(contents)
def remove(self):
diff --git a/lib/galaxy/app_unittest_utils/tools_support.py b/lib/galaxy/app_unittest_utils/tools_support.py
index 349bc291350..c36ac90911b 100644
--- a/lib/galaxy/app_unittest_utils/tools_support.py
+++ b/lib/galaxy/app_unittest_utils/tools_support.py
@@ -17,14 +17,12 @@ from galaxy.tool_util.parser import get_tool_source
from galaxy.tools import create_tool_from_source
from galaxy.util.bunch import Bunch
-
datatypes_registry = galaxy.datatypes.registry.Registry()
datatypes_registry.load_datatypes()
galaxy.model.set_datatypes_registry(datatypes_registry)
class UsesApp:
-
def setup_app(self):
self.test_directory = tempfile.mkdtemp()
self.app = MockApp()
@@ -37,7 +35,7 @@ class UsesApp:
# Simple tool with just one text parameter and output.
-SIMPLE_TOOL_CONTENTS = '''
+SIMPLE_TOOL_CONTENTS = """
echo "$param1" < $out1
@@ -46,11 +44,11 @@ SIMPLE_TOOL_CONTENTS = '''
-'''
+"""
# A tool with data parameters (kind of like cat1) my favorite test tool :)
-SIMPLE_CAT_TOOL_CONTENTS = '''
+SIMPLE_CAT_TOOL_CONTENTS = """
cat "$param1" #for $r in $repeat# "$r.param2" #end for# < $out1
@@ -62,11 +60,10 @@ SIMPLE_CAT_TOOL_CONTENTS = '''
-'''
+"""
class MockActionI:
-
def execute(self, tool, trans, **kwds):
pass
@@ -122,7 +119,6 @@ class UsesTools(UsesApp):
class MockContext:
-
def __init__(self, model_objects=None):
self.expunged_all = False
self.flushed = False
@@ -144,7 +140,6 @@ class MockContext:
class MockQuery:
-
def __init__(self, class_objects):
self.class_objects = class_objects
@@ -155,4 +150,4 @@ class MockQuery:
return self.class_objects.get(id, None)
-__all__ = ('UsesApp', )
+__all__ = ("UsesApp",)
diff --git a/lib/galaxy/auth/__init__.py b/lib/galaxy/auth/__init__.py
index 064ae8af2af..1f4aa9986c6 100644
--- a/lib/galaxy/auth/__init__.py
+++ b/lib/galaxy/auth/__init__.py
@@ -3,7 +3,10 @@ Contains implementations of the authentication logic.
"""
import logging
-from galaxy.auth.util import get_authenticators, parse_auth_results
+from galaxy.auth.util import (
+ get_authenticators,
+ parse_auth_results,
+)
from galaxy.exceptions import Conflict
from galaxy.util import string_as_bool
@@ -11,30 +14,29 @@ log = logging.getLogger(__name__)
class AuthManager:
-
def __init__(self, config):
self.redact_username_in_logs = config.redact_username_in_logs
- self.authenticators = get_authenticators(config.auth_config_file, config.is_set('auth_config_file'))
+ self.authenticators = get_authenticators(config.auth_config_file, config.is_set("auth_config_file"))
def check_registration_allowed(self, email, username, password):
"""Checks if the provided email/username is allowed to register."""
- message = ''
- status = 'done'
+ message = ""
+ status = "done"
for provider, options in self.active_authenticators(email, username, password):
allow_reg = _get_allow_register(options)
- if allow_reg == 'challenge':
+ if allow_reg == "challenge":
auth_results = provider.authenticate(email, username, password, options)
if auth_results[0] is True:
break
if auth_results[0] is None:
- message = 'Invalid email address/username or password.'
- status = 'error'
+ message = "Invalid email address/username or password."
+ status = "error"
break
elif allow_reg is True:
break
elif allow_reg is False:
- message = 'Account registration not required for your account. Please simply login.'
- status = 'error'
+ message = "Account registration not required for your account. Please simply login."
+ status = "error"
break
return message, status
@@ -43,22 +45,18 @@ class AuthManager:
Checks the username/email & password using auth providers in order.
If a match is found, returns the 'auto-register' option for that provider.
"""
- if '@' in login:
+ if "@" in login:
email = login
username = None
else:
email = None
username = login
- auth_return = {
- "auto_reg": False,
- "email": "",
- "username": ""
- }
+ auth_return = {"auto_reg": False, "email": "", "username": ""}
for provider, options in self.active_authenticators(email, username, password):
if provider is None:
log.debug(f"Unable to find module: {options}")
else:
- options['no_password_check'] = no_password_check
+ options["no_password_check"] = no_password_check
auth_results = provider.authenticate(email, username, password, options)
if auth_results[0] is True:
try:
@@ -98,10 +96,10 @@ class AuthManager:
if string_as_bool(options.get("allow-password-change", False)):
return
else:
- return 'Password change not supported.'
+ return "Password change not supported."
elif auth_result is None:
break # end authentication (skip rest)
- return 'Invalid current password.'
+ return "Invalid current password."
def active_authenticators(self, email, username, password):
"""Yields AuthProvider instances for the provided configfile that match the
@@ -112,11 +110,11 @@ class AuthManager:
filter_template = authenticator.filter_template
if filter_template:
filter_str = filter_template.format(email=email, username=username, password=password)
- passed_filter = eval(filter_str, {"__builtins__": None}, {'str': str})
+ passed_filter = eval(filter_str, {"__builtins__": None}, {"str": str})
if not passed_filter:
continue # skip to next
options = authenticator.options
- options['redact_username_in_logs'] = self.redact_username_in_logs
+ options["redact_username_in_logs"] = self.redact_username_in_logs
yield authenticator.plugin, options
except Exception:
log.exception("Active Authenticators Failure")
@@ -124,9 +122,9 @@ class AuthManager:
def _get_allow_register(d):
- s = d.get('allow-register', True)
+ s = d.get("allow-register", True)
lower_s = str(s).lower()
- if lower_s == 'challenge':
+ if lower_s == "challenge":
return lower_s
else:
return string_as_bool(s)
diff --git a/lib/galaxy/auth/providers/__init__.py b/lib/galaxy/auth/providers/__init__.py
index 220fdf24898..f85051dabcb 100644
--- a/lib/galaxy/auth/providers/__init__.py
+++ b/lib/galaxy/auth/providers/__init__.py
@@ -11,7 +11,7 @@ class AuthProvider(metaclass=abc.ABCMeta):
@abc.abstractproperty
def plugin_type(self):
- """ Short string providing labelling this plugin """
+ """Short string providing labelling this plugin"""
@abc.abstractmethod
def authenticate(self, email, username, password, options):
diff --git a/lib/galaxy/auth/providers/alwaysreject.py b/lib/galaxy/auth/providers/alwaysreject.py
index e39990ae977..b4ffa043fbe 100644
--- a/lib/galaxy/auth/providers/alwaysreject.py
+++ b/lib/galaxy/auth/providers/alwaysreject.py
@@ -14,13 +14,14 @@ class AlwaysReject(AuthProvider):
"""A simple authenticator that just accepts users (does not care about their
password).
"""
- plugin_type = 'alwaysreject'
+
+ plugin_type = "alwaysreject"
def authenticate(self, email, username, password, options):
"""
See abstract method documentation.
"""
- return (None, '', '')
+ return (None, "", "")
def authenticate_user(self, user, password, options):
"""
@@ -30,4 +31,4 @@ class AlwaysReject(AuthProvider):
return None
-__all__ = ('AlwaysReject', )
+__all__ = ("AlwaysReject",)
diff --git a/lib/galaxy/auth/providers/ldap_ad.py b/lib/galaxy/auth/providers/ldap_ad.py
index dac936f7386..fc7eef2af5b 100644
--- a/lib/galaxy/auth/providers/ldap_ad.py
+++ b/lib/galaxy/auth/providers/ldap_ad.py
@@ -43,11 +43,16 @@ def _parse_ldap_options(options_unparsed):
try:
key, value = opt.split("=")
except ValueError:
- log.warning("LDAP authenticate: Invalid syntax '%s' inside element. Syntax should be option1=value1,option2=value2", opt)
+ log.warning(
+ "LDAP authenticate: Invalid syntax '%s' inside element. Syntax should be option1=value1,option2=value2",
+ opt,
+ )
continue
if not key.startswith(prefix):
- log.warning("LDAP authenticate: Invalid LDAP option '%s'. '%s' doesn't start with prefix '%s'", opt, key, prefix)
+ log.warning(
+ "LDAP authenticate: Invalid LDAP option '%s'. '%s' doesn't start with prefix '%s'", opt, key, prefix
+ )
continue
try:
key = getattr(ldap, key)
@@ -58,7 +63,9 @@ def _parse_ldap_options(options_unparsed):
try:
value = getattr(ldap, value)
except AttributeError:
- log.warning("LDAP authenticate: Invalid LDAP option '%s'. '%s' is not available in module ldap", opt, value)
+ log.warning(
+ "LDAP authenticate: Invalid LDAP option '%s'. '%s' is not available in module ldap", opt, value
+ )
continue
pair = (key, value)
log.debug("LDAP authenticate: Valid LDAP option pair '%s' -> '%s=%s'", opt, *pair)
@@ -76,8 +83,9 @@ class LDAP(AuthProvider):
those fields first. After that it will bind to LDAP with the username
(formatted as specified).
"""
- plugin_type = 'ldap'
- role_search_option = 'auto-register-roles'
+
+ plugin_type = "ldap"
+ role_search_option = "auto-register-roles"
def __init__(self):
super().__init__()
@@ -86,32 +94,36 @@ class LDAP(AuthProvider):
def check_config(self, username, email, options):
ok = True
- if options.get('continue-on-failure', 'False') == 'False':
+ if options.get("continue-on-failure", "False") == "False":
failure_mode = None # reject and do not continue
else:
failure_mode = False # reject but continue
- if string_as_bool(options.get('login-use-username', False)):
+ if string_as_bool(options.get("login-use-username", False)):
if not username:
- log.debug('LDAP authenticate: username must be used to login, cannot be None')
+ log.debug("LDAP authenticate: username must be used to login, cannot be None")
return ok, failure_mode
else:
if not email:
- log.debug('LDAP authenticate: email must be used to login, cannot be None')
+ log.debug("LDAP authenticate: email must be used to login, cannot be None")
return ok, failure_mode
- auto_create_roles = string_as_bool(options.get('auto-create-roles', False))
- auto_create_groups = string_as_bool(options.get('auto-create-groups', False))
+ auto_create_roles = string_as_bool(options.get("auto-create-roles", False))
+ auto_create_groups = string_as_bool(options.get("auto-create-groups", False))
self.auto_create_roles_or_groups = auto_create_roles or auto_create_groups
- auto_assign_roles_to_groups_only = string_as_bool(options.get('auto-assign-roles-to-groups-only', False))
+ auto_assign_roles_to_groups_only = string_as_bool(options.get("auto-assign-roles-to-groups-only", False))
if auto_assign_roles_to_groups_only and not (auto_create_roles and auto_create_groups):
- raise ConfigurationError("If 'auto-assign-roles-to-groups-only' is True, auto-create-roles and "
- "auto-create-groups have to be True as well.")
+ raise ConfigurationError(
+ "If 'auto-assign-roles-to-groups-only' is True, auto-create-roles and "
+ "auto-create-groups have to be True as well."
+ )
self.role_search_attribute = options.get(self.role_search_option)
if self.auto_create_roles_or_groups and self.role_search_attribute is None:
- raise ConfigurationError("If 'auto-create-roles' or 'auto-create-groups' is True, a '%s' attribute has to"
- " be provided." % self.role_search_option)
+ raise ConfigurationError(
+ "If 'auto-create-roles' or 'auto-create-groups' is True, a '%s' attribute has to"
+ " be provided." % self.role_search_option
+ )
return ok, failure_mode
@@ -123,10 +135,10 @@ class LDAP(AuthProvider):
if not config_ok:
return failure_mode, None
- params = {'email': email, 'username': username}
+ params = {"email": email, "username": username}
try:
- ldap_options_raw = _get_subs(options, 'ldap-options', params)
+ ldap_options_raw = _get_subs(options, "ldap-options", params)
except ConfigurationError:
ldap_options = ()
else:
@@ -139,34 +151,37 @@ class LDAP(AuthProvider):
for opt in ldap_options:
ldap.set_option(*opt)
except Exception:
- log.exception('LDAP authenticate: set_option exception')
+ log.exception("LDAP authenticate: set_option exception")
return (failure_mode, None)
- if 'search-fields' in options:
+ if "search-fields" in options:
try:
- conn = ldap.initialize(_get_subs(options, 'server', params))
+ conn = ldap.initialize(_get_subs(options, "server", params))
conn.protocol_version = 3
- if 'search-user' in options:
+ if "search-user" in options:
conn.simple_bind_s(
- _get_subs(options, 'search-user', params),
- _get_subs(options, 'search-password', params))
+ _get_subs(options, "search-user", params), _get_subs(options, "search-password", params)
+ )
else:
conn.simple_bind_s()
# setup search
- attributes = {_.strip().format(**params) for _ in options['search-fields'].split(',')}
- if 'search-memberof-filter' in options:
- attributes.add('memberOf')
+ attributes = {_.strip().format(**params) for _ in options["search-fields"].split(",")}
+ if "search-memberof-filter" in options:
+ attributes.add("memberOf")
suser = conn.search_ext_s(
- _get_subs(options, 'search-base', params),
+ _get_subs(options, "search-base", params),
ldap.SCOPE_SUBTREE,
- _get_subs(options, 'search-filter', params), attributes,
- timeout=60, sizelimit=1)
+ _get_subs(options, "search-filter", params),
+ attributes,
+ timeout=60,
+ sizelimit=1,
+ )
# parse results
if suser is None or len(suser) == 0:
- log.warning('LDAP authenticate: search returned no results')
+ log.warning("LDAP authenticate: search returned no results")
return (failure_mode, None)
dn, attrs = suser[0]
log.debug("LDAP authenticate: dn is %s", dn)
@@ -175,7 +190,7 @@ class LDAP(AuthProvider):
if self.role_search_attribute and attr == self.role_search_attribute[1:-1]: # strip curly brackets
# keep role names as list
params[self.role_search_option] = [unicodify(_) for _ in attrs[attr]]
- elif attr == 'memberOf':
+ elif attr == "memberOf":
params[attr] = [unicodify(_) for _ in attrs[attr]]
elif attr in attrs:
params[attr] = unicodify(attrs[attr][0])
@@ -183,12 +198,13 @@ class LDAP(AuthProvider):
params[attr] = ""
if self.auto_create_roles_or_groups and self.role_search_option not in params:
- raise ConfigurationError("Missing or mismatching LDAP parameters for %s. Make sure the %s is "
- "included in the 'search-fields'." %
- (self.role_search_option, self.role_search_attribute))
- params['dn'] = dn
+ raise ConfigurationError(
+ "Missing or mismatching LDAP parameters for %s. Make sure the %s is "
+ "included in the 'search-fields'." % (self.role_search_option, self.role_search_attribute)
+ )
+ params["dn"] = dn
except Exception:
- log.exception('LDAP authenticate: search exception')
+ log.exception("LDAP authenticate: search exception")
return (failure_mode, None)
return failure_mode, params
@@ -197,7 +213,7 @@ class LDAP(AuthProvider):
"""
See abstract method documentation.
"""
- if not options['redact_username_in_logs']:
+ if not options["redact_username_in_logs"]:
log.debug("LDAP authenticate: email is %s", email)
log.debug("LDAP authenticate: username is %s", username)
@@ -205,39 +221,41 @@ class LDAP(AuthProvider):
failure_mode, params = self.ldap_search(email, username, options)
if not params:
- return failure_mode, '', ''
+ return failure_mode, "", ""
# allow to skip authentication to allow for pre-populating users
- if not options.get('no_password_check', False):
- params['password'] = password
+ if not options.get("no_password_check", False):
+ params["password"] = password
if not self._authenticate(params, options):
- return failure_mode, '', ''
+ return failure_mode, "", ""
# check whether the user is a member of a specified group/domain/...
- if 'search-memberof-filter' in options:
- search_filter = _get_subs(options, 'search-memberof-filter', params)
- if not any(search_filter in ad_node_name for ad_node_name in params['memberOf']):
- return failure_mode, '', ''
+ if "search-memberof-filter" in options:
+ search_filter = _get_subs(options, "search-memberof-filter", params)
+ if not any(search_filter in ad_node_name for ad_node_name in params["memberOf"]):
+ return failure_mode, "", ""
attributes = {}
if self.auto_create_roles_or_groups:
- attributes['roles'] = params[self.role_search_option]
- return (True,
- _get_subs(options, 'auto-register-email', params),
- transform_publicname(_get_subs(options, 'auto-register-username', params)),
- attributes)
+ attributes["roles"] = params[self.role_search_option]
+ return (
+ True,
+ _get_subs(options, "auto-register-email", params),
+ transform_publicname(_get_subs(options, "auto-register-username", params)),
+ attributes,
+ )
def _authenticate(self, params, options):
"""
Do the actual authentication by binding as the user to check their credentials
"""
try:
- conn = ldap.initialize(_get_subs(options, 'server', params))
+ conn = ldap.initialize(_get_subs(options, "server", params))
conn.protocol_version = 3
- bind_user = _get_subs(options, 'bind-user', params)
- bind_password = _get_subs(options, 'bind-password', params)
+ bind_user = _get_subs(options, "bind-user", params)
+ bind_password = _get_subs(options, "bind-password", params)
except Exception:
- log.exception('LDAP authenticate: initialize exception')
+ log.exception("LDAP authenticate: initialize exception")
return False
try:
conn.simple_bind_s(bind_user, bind_password)
@@ -248,13 +266,13 @@ class LDAP(AuthProvider):
pass
else:
if whoami is None:
- raise RuntimeError('LDAP authenticate: anonymous bind')
- if not options['redact_username_in_logs']:
+ raise RuntimeError("LDAP authenticate: anonymous bind")
+ if not options["redact_username_in_logs"]:
log.debug("LDAP authenticate: whoami is %s", whoami)
except Exception as e:
- log.info('LDAP authenticate: bind exception: %s', unicodify(e))
+ log.info("LDAP authenticate: bind exception: %s", unicodify(e))
return False
- log.debug('LDAP authentication successful')
+ log.debug("LDAP authentication successful")
return True
def authenticate_user(self, user, password, options):
@@ -265,9 +283,10 @@ class LDAP(AuthProvider):
class ActiveDirectory(LDAP):
- """ Effectively just an alias for LDAP auth, but may contain active directory specific
- logic in the future. """
- plugin_type = 'activedirectory'
+ """Effectively just an alias for LDAP auth, but may contain active directory specific
+ logic in the future."""
+
+ plugin_type = "activedirectory"
-__all__ = ('LDAP', 'ActiveDirectory')
+__all__ = ("LDAP", "ActiveDirectory")
diff --git a/lib/galaxy/auth/providers/localdb.py b/lib/galaxy/auth/providers/localdb.py
index d17d865a7dc..168800a3e06 100644
--- a/lib/galaxy/auth/providers/localdb.py
+++ b/lib/galaxy/auth/providers/localdb.py
@@ -12,13 +12,14 @@ log = logging.getLogger(__name__)
class LocalDB(AuthProvider):
"""Authenticate users against the local Galaxy database (as per usual)."""
- plugin_type = 'localdb'
+
+ plugin_type = "localdb"
def authenticate(self, email, username, password, options):
"""
See abstract method documentation.
"""
- return (False, '', '') # it can never auto-create based of localdb (chicken-egg)
+ return (False, "", "") # it can never auto-create based of localdb (chicken-egg)
def authenticate_user(self, user, password, options):
"""
@@ -29,4 +30,4 @@ class LocalDB(AuthProvider):
return user_ok
-__all__ = ('LocalDB', )
+__all__ = ("LocalDB",)
diff --git a/lib/galaxy/auth/providers/pam_auth.py b/lib/galaxy/auth/providers/pam_auth.py
index 69a56186935..48794157bd2 100644
--- a/lib/galaxy/auth/providers/pam_auth.py
+++ b/lib/galaxy/auth/providers/pam_auth.py
@@ -8,7 +8,7 @@ import shlex
from galaxy.util import (
commands,
- string_as_bool
+ string_as_bool,
)
from ..providers import AuthProvider
@@ -56,35 +56,37 @@ Configuration example (for internal authentication, use email for user details):
class PAM(AuthProvider):
- plugin_type = 'PAM'
+ plugin_type = "PAM"
def authenticate(self, email, username, password, options):
pam_username = None
auto_register_username = None
auto_register_email = None
force_fail = False
- if not options['redact_username_in_logs']:
- log.debug(f"use username: {options.get('login-use-username')} use email {options.get('login-use-email', False)} email {email} username {username}")
+ if not options["redact_username_in_logs"]:
+ log.debug(
+ f"use username: {options.get('login-use-username')} use email {options.get('login-use-email', False)} email {email} username {username}"
+ )
# check email based login first because if email exists in Galaxy DB
# we will be given the "public name" as username
- if string_as_bool(options.get('login-use-email', False)) and email is not None:
- if '@' in email:
- (email_user, email_domain) = email.split('@')
+ if string_as_bool(options.get("login-use-email", False)) and email is not None:
+ if "@" in email:
+ (email_user, email_domain) = email.split("@")
pam_username = email_user
- if email_domain == options.get('maildomain', None):
+ if email_domain == options.get("maildomain", None):
auto_register_email = email
if username is not None:
auto_register_username = username
else:
auto_register_username = email_user
else:
- log.debug('PAM authenticate: warning: email does not match configured PAM maildomain')
+ log.debug("PAM authenticate: warning: email does not match configured PAM maildomain")
# no need to fail: if auto-register is not enabled, this
# might still be a valid user
else:
- log.debug('PAM authenticate: email must be used to login, but no valid email found')
+ log.debug("PAM authenticate: email must be used to login, but no valid email found")
force_fail = True
- elif string_as_bool(options.get('login-use-username', False)):
+ elif string_as_bool(options.get("login-use-username", False)):
# if we get here via authenticate_user then
# user will be "public name" and
# email address will be as per registered user
@@ -92,42 +94,44 @@ class PAM(AuthProvider):
pam_username = username
if email is not None:
auto_register_email = email
- elif options.get('maildomain', None) is not None:
+ elif options.get("maildomain", None) is not None:
# we can register a user with this username and mail domain
# if auto registration is enabled
auto_register_email = f"{username}@{options['maildomain']}"
auto_register_username = username
else:
- log.debug('PAM authenticate: username login selected but no username provided')
+ log.debug("PAM authenticate: username login selected but no username provided")
force_fail = True
else:
- log.debug('PAM authenticate: could not find username for PAM')
+ log.debug("PAM authenticate: could not find username for PAM")
force_fail = True
if force_fail:
- return None, '', ''
+ return None, "", ""
- pam_service = options.get('pam-service', 'galaxy')
- use_helper = string_as_bool(options.get('use-external-helper', False))
+ pam_service = options.get("pam-service", "galaxy")
+ use_helper = string_as_bool(options.get("use-external-helper", False))
log.debug(f"PAM auth: will use external helper: {use_helper}")
authenticated = False
if use_helper:
- authentication_helper = options.get('authentication-helper-script', '/bin/false').strip()
+ authentication_helper = options.get("authentication-helper-script", "/bin/false").strip()
log.debug(f"PAM auth: external helper script: {authentication_helper}")
- if not authentication_helper.startswith('/'):
+ if not authentication_helper.startswith("/"):
# don't accept relative path
authenticated = False
else:
- auth_cmd = shlex.split(f'/usr/bin/sudo -n {authentication_helper}')
+ auth_cmd = shlex.split(f"/usr/bin/sudo -n {authentication_helper}")
log.debug(f"PAM auth: external helper cmd: {auth_cmd}")
- message = f'{pam_service}\n{pam_username}\n{password}\n'
+ message = f"{pam_service}\n{pam_username}\n{password}\n"
try:
output = commands.execute(auth_cmd, input=message)
except commands.CommandLineException as e:
- if e.stderr != '':
- log.debug(f"PAM auth: external authentication script had errors: status {e.returncode} error {e.stderr}")
+ if e.stderr != "":
+ log.debug(
+ f"PAM auth: external authentication script had errors: status {e.returncode} error {e.stderr}"
+ )
output = e.stdout
- if output.strip() == 'True':
+ if output.strip() == "True":
authenticated = True
else:
authenticated = False
@@ -135,21 +139,25 @@ class PAM(AuthProvider):
try:
import pam
except ImportError:
- log.debug('PAM authenticate: could not load pam module, PAM authentication disabled')
- return None, '', ''
+ log.debug("PAM authenticate: could not load pam module, PAM authentication disabled")
+ return None, "", ""
p_auth = pam.pam()
authenticated = p_auth.authenticate(pam_username, password, service=pam_service)
if authenticated:
- log.debug(f"PAM authentication successful for {'redacted' if options['redact_username_in_logs'] else pam_username}")
+ log.debug(
+ f"PAM authentication successful for {'redacted' if options['redact_username_in_logs'] else pam_username}"
+ )
return True, auto_register_email, auto_register_username
else:
- log.debug(f"PAM authentication failed for {'redacted' if options['redact_username_in_logs'] else pam_username}")
- return False, '', ''
+ log.debug(
+ f"PAM authentication failed for {'redacted' if options['redact_username_in_logs'] else pam_username}"
+ )
+ return False, "", ""
def authenticate_user(self, user, password, options):
return self.authenticate(user.email, user.username, password, options)[0]
-__all__ = ('PAM', )
+__all__ = ("PAM",)
diff --git a/lib/galaxy/auth/util.py b/lib/galaxy/auth/util.py
index 7f9f0f2dd1b..78168fda7de 100644
--- a/lib/galaxy/auth/util.py
+++ b/lib/galaxy/auth/util.py
@@ -12,7 +12,6 @@ from galaxy.util import (
string_as_bool,
)
-
log = logging.getLogger(__name__)
AUTH_CONF_XML = """
@@ -26,11 +25,11 @@ AUTH_CONF_XML = """
"""
-Authenticator = namedtuple('Authenticator', ['plugin', 'filter_template', 'options'])
+Authenticator = namedtuple("Authenticator", ["plugin", "filter_template", "options"])
def get_authenticators(auth_config_file, auth_config_file_set):
- __plugins_dict = plugin_config.plugins_dict(galaxy.auth.providers, 'plugin_type')
+ __plugins_dict = plugin_config.plugins_dict(galaxy.auth.providers, "plugin_type")
# parse XML
try:
ct = parse_xml(auth_config_file)
@@ -44,21 +43,23 @@ def get_authenticators(auth_config_file, auth_config_file_set):
authenticators = []
# process authenticators
for auth_elem in conf_root:
- type_elem_text = auth_elem.find('type').text
+ type_elem_text = auth_elem.find("type").text
plugin_class = __plugins_dict.get(type_elem_text)
if not plugin_class:
- raise Exception(f"Authenticator type '{type_elem_text}' not recognized, should be one of {', '.join(__plugins_dict)}")
+ raise Exception(
+ f"Authenticator type '{type_elem_text}' not recognized, should be one of {', '.join(__plugins_dict)}"
+ )
plugin = plugin_class()
# check filterelem
- filter_elem = auth_elem.find('filter')
+ filter_elem = auth_elem.find("filter")
if filter_elem is not None:
filter_template = str(filter_elem.text)
else:
filter_template = None
# extract options
- options_elem = auth_elem.find('options')
+ options_elem = auth_elem.find("options")
options = {}
if options_elem is not None:
for opt in options_elem:
@@ -77,23 +78,24 @@ def parse_auth_results(trans, auth_results, options):
auth_result, auto_email, auto_username = auth_results[:3]
auto_username = str(auto_username).lower()
# make username unique
- if validate_publicname(trans, auto_username) != '':
+ if validate_publicname(trans, auto_username) != "":
i = 1
while i <= 10: # stop after 10 tries
- if validate_publicname(trans, "%s-%i" % (auto_username, i)) == '':
+ if validate_publicname(trans, "%s-%i" % (auto_username, i)) == "":
auto_username = "%s-%i" % (auto_username, i)
break
i += 1
else:
raise Conflict("Cannot make unique username")
log.debug(f"Email: {auto_email}, auto-register with username: {auto_username}")
- auth_return["auto_reg"] = string_as_bool(options.get('auto-register', False))
+ auth_return["auto_reg"] = string_as_bool(options.get("auto-register", False))
auth_return["email"] = auto_email
auth_return["username"] = auto_username
- auth_return["auto_create_roles"] = string_as_bool(options.get('auto-create-roles', False))
- auth_return["auto_create_groups"] = string_as_bool(options.get('auto-create-groups', False))
+ auth_return["auto_create_roles"] = string_as_bool(options.get("auto-create-roles", False))
+ auth_return["auto_create_groups"] = string_as_bool(options.get("auto-create-groups", False))
auth_return["auto_assign_roles_to_groups_only"] = string_as_bool(
- options.get('auto-assign-roles-to-groups-only', False))
+ options.get("auto-assign-roles-to-groups-only", False)
+ )
if len(auth_results) == 4:
auth_return["attributes"] = auth_results[3]
diff --git a/lib/galaxy/authnz/custos_authnz.py b/lib/galaxy/authnz/custos_authnz.py
index 579006246b2..f3aa897a7c1 100644
--- a/lib/galaxy/authnz/custos_authnz.py
+++ b/lib/galaxy/authnz/custos_authnz.py
@@ -3,7 +3,10 @@ import hashlib
import json
import logging
import os
-from datetime import datetime, timedelta
+from datetime import (
+ datetime,
+ timedelta,
+)
from urllib.parse import quote
import jwt
@@ -11,55 +14,61 @@ import requests
from oauthlib.common import generate_nonce
from requests_oauthlib import OAuth2Session
-from galaxy import exceptions
-from galaxy import util
-from galaxy.model import CustosAuthnzToken, User
+from galaxy import (
+ exceptions,
+ util,
+)
+from galaxy.model import (
+ CustosAuthnzToken,
+ User,
+)
from ..authnz import IdentityProvider
log = logging.getLogger(__name__)
-STATE_COOKIE_NAME = 'galaxy-oidc-state'
-NONCE_COOKIE_NAME = 'galaxy-oidc-nonce'
-KEYCLOAK_BACKENDS = {'custos', 'cilogon', 'keycloak'}
+STATE_COOKIE_NAME = "galaxy-oidc-state"
+NONCE_COOKIE_NAME = "galaxy-oidc-nonce"
+KEYCLOAK_BACKENDS = {"custos", "cilogon", "keycloak"}
class CustosAuthnz(IdentityProvider):
def __init__(self, provider, oidc_config, oidc_backend_config, idphint=None):
provider = provider.lower()
- self.config = {'provider': provider}
- self.config['verify_ssl'] = oidc_config['VERIFY_SSL']
- self.config['url'] = oidc_backend_config['url']
- self.config['client_id'] = oidc_backend_config['client_id']
- self.config['client_secret'] = oidc_backend_config['client_secret']
- self.config['redirect_uri'] = oidc_backend_config['redirect_uri']
- self.config['ca_bundle'] = oidc_backend_config.get('ca_bundle', None)
- self.config['extra_params'] = {
- 'kc_idp_hint': oidc_backend_config.get('idphint', 'oidc' if self.config['provider'] in ['custos', 'keycloak'] else 'cilogon')
+ self.config = {"provider": provider}
+ self.config["verify_ssl"] = oidc_config["VERIFY_SSL"]
+ self.config["url"] = oidc_backend_config["url"]
+ self.config["client_id"] = oidc_backend_config["client_id"]
+ self.config["client_secret"] = oidc_backend_config["client_secret"]
+ self.config["redirect_uri"] = oidc_backend_config["redirect_uri"]
+ self.config["ca_bundle"] = oidc_backend_config.get("ca_bundle", None)
+ self.config["extra_params"] = {
+ "kc_idp_hint": oidc_backend_config.get(
+ "idphint", "oidc" if self.config["provider"] in ["custos", "keycloak"] else "cilogon"
+ )
}
- if provider == 'cilogon':
+ if provider == "cilogon":
self._load_config_for_cilogon()
- elif provider == 'custos':
+ elif provider == "custos":
self._load_config_for_custos()
- elif provider == 'keycloak':
+ elif provider == "keycloak":
self._load_config_for_keycloak()
def _decode_token_no_signature(self, token):
- return jwt.decode(token, audience=self.config['client_id'], options={"verify_signature": False})
+ return jwt.decode(token, audience=self.config["client_id"], options={"verify_signature": False})
def authenticate(self, trans, idphint=None):
- base_authorize_url = self.config['authorization_endpoint']
- scopes = ['openid', 'email', 'profile']
- if self.config['provider'] in ['custos', 'cilogon']:
- scopes.append('org.cilogon.userinfo')
+ base_authorize_url = self.config["authorization_endpoint"]
+ scopes = ["openid", "email", "profile"]
+ if self.config["provider"] in ["custos", "cilogon"]:
+ scopes.append("org.cilogon.userinfo")
oauth2_session = self._create_oauth2_session(scope=scopes)
nonce = generate_nonce()
nonce_hash = self._hash_nonce(nonce)
extra_params = {"nonce": nonce_hash}
if idphint is not None:
- extra_params['idphint'] = idphint
+ extra_params["idphint"] = idphint
if "extra_params" in self.config:
- extra_params.update(self.config['extra_params'])
- authorization_url, state = oauth2_session.authorization_url(
- base_authorize_url, **extra_params)
+ extra_params.update(self.config["extra_params"])
+ authorization_url, state = oauth2_session.authorization_url(base_authorize_url, **extra_params)
trans.set_cookie(value=state, name=STATE_COOKIE_NAME)
trans.set_cookie(value=nonce, name=NONCE_COOKIE_NAME)
return authorization_url
@@ -71,29 +80,31 @@ class CustosAuthnz(IdentityProvider):
state_cookie = trans.get_cookie(name=STATE_COOKIE_NAME)
oauth2_session = self._create_oauth2_session(state=state_cookie)
token = self._fetch_token(oauth2_session, trans)
- access_token = token['access_token']
- id_token = token['id_token']
- refresh_token = token['refresh_token'] if 'refresh_token' in token else None
- expiration_time = datetime.now() + timedelta(seconds=token.get('expires_in', 3600))
- refresh_expiration_time = (datetime.now() + timedelta(seconds=token['refresh_expires_in'])) if 'refresh_expires_in' in token else None
+ access_token = token["access_token"]
+ id_token = token["id_token"]
+ refresh_token = token["refresh_token"] if "refresh_token" in token else None
+ expiration_time = datetime.now() + timedelta(seconds=token.get("expires_in", 3600))
+ refresh_expiration_time = (
+ (datetime.now() + timedelta(seconds=token["refresh_expires_in"])) if "refresh_expires_in" in token else None
+ )
# Get nonce from token['id_token'] and validate. 'nonce' in the
# id_token is a hash of the nonce stored in the NONCE_COOKIE_NAME
# cookie.
id_token_decoded = self._decode_token_no_signature(id_token)
- nonce_hash = id_token_decoded['nonce']
+ nonce_hash = id_token_decoded["nonce"]
self._validate_nonce(trans, nonce_hash)
# Get userinfo and lookup/create Galaxy user record
- if id_token_decoded.get('email', None):
+ if id_token_decoded.get("email", None):
userinfo = id_token_decoded
else:
userinfo = self._get_userinfo(oauth2_session)
- email = userinfo['email']
- user_id = userinfo['sub']
+ email = userinfo["email"]
+ user_id = userinfo["sub"]
# Create or update custos_authnz_token record
- custos_authnz_token = self._get_custos_authnz_token(trans.sa_session, user_id, self.config['provider'])
+ custos_authnz_token = self._get_custos_authnz_token(trans.sa_session, user_id, self.config["provider"])
if custos_authnz_token is None:
user = trans.user
if not user:
@@ -105,15 +116,17 @@ class CustosAuthnz(IdentityProvider):
# TODO: Future work will expand on this and provide an
# interface for when there are multiple auth providers
# allowing explicit authenticated association.
- if (trans.app.config.enable_oidc
- and len(trans.app.config.oidc) == 1
- and len(trans.app.auth_manager.authenticators) == 0):
+ if (
+ trans.app.config.enable_oidc
+ and len(trans.app.config.oidc) == 1
+ and len(trans.app.auth_manager.authenticators) == 0
+ ):
user = existing_user
else:
message = f"There already exists a user with email {email}. To associate this external login, you must first be logged in as that existing account."
log.exception(message)
raise exceptions.AuthenticationFailed(message)
- elif self.config['provider'] == 'custos':
+ elif self.config["provider"] == "custos":
login_redirect_url = f"{login_redirect_url}root/login?confirm=true&custos_token={json.dumps(token)}"
return login_redirect_url, None
else:
@@ -122,14 +135,16 @@ class CustosAuthnz(IdentityProvider):
if trans.app.config.user_activation_on:
trans.app.user_manager.send_activation_email(trans, email, username)
- custos_authnz_token = CustosAuthnzToken(user=user,
- external_user_id=user_id,
- provider=self.config['provider'],
- access_token=access_token,
- id_token=id_token,
- refresh_token=refresh_token,
- expiration_time=expiration_time,
- refresh_expiration_time=refresh_expiration_time)
+ custos_authnz_token = CustosAuthnzToken(
+ user=user,
+ external_user_id=user_id,
+ provider=self.config["provider"],
+ access_token=access_token,
+ id_token=id_token,
+ refresh_token=refresh_token,
+ expiration_time=expiration_time,
+ refresh_expiration_time=refresh_expiration_time,
+ )
else:
custos_authnz_token.access_token = access_token
custos_authnz_token.id_token = id_token
@@ -143,11 +158,17 @@ class CustosAuthnz(IdentityProvider):
def create_user(self, token, trans, login_redirect_url):
token_dict = json.loads(token)
- access_token = token_dict['access_token']
- id_token = token_dict['id_token']
- refresh_token = token_dict['refresh_token'] if 'refresh_token' in token_dict else None
- expiration_time = datetime.now() + timedelta(seconds=token_dict.get('expires_in', 3600)) # might be a problem cause times no long valid
- refresh_expiration_time = (datetime.now() + timedelta(seconds=token_dict['refresh_expires_in'])) if 'refresh_expires_in' in token_dict else None
+ access_token = token_dict["access_token"]
+ id_token = token_dict["id_token"]
+ refresh_token = token_dict["refresh_token"] if "refresh_token" in token_dict else None
+ expiration_time = datetime.now() + timedelta(
+ seconds=token_dict.get("expires_in", 3600)
+ ) # might be a problem cause times no long valid
+ refresh_expiration_time = (
+ (datetime.now() + timedelta(seconds=token_dict["refresh_expires_in"]))
+ if "refresh_expires_in" in token_dict
+ else None
+ )
# Get nonce from token['id_token'] and validate. 'nonce' in the
# id_token is a hash of the nonce stored in the NONCE_COOKIE_NAME
@@ -155,23 +176,25 @@ class CustosAuthnz(IdentityProvider):
userinfo = self._decode_token_no_signature(id_token)
# Get userinfo and create Galaxy user record
- email = userinfo['email']
+ email = userinfo["email"]
# Check if username if already taken
username = self._username_from_userinfo(trans, userinfo)
- user_id = userinfo['sub']
+ user_id = userinfo["sub"]
user = trans.app.user_manager.create(email=email, username=username)
if trans.app.config.user_activation_on:
trans.app.user_manager.send_activation_email(trans, email, username)
- custos_authnz_token = CustosAuthnzToken(user=user,
- external_user_id=user_id,
- provider=self.config['provider'],
- access_token=access_token,
- id_token=id_token,
- refresh_token=refresh_token,
- expiration_time=expiration_time,
- refresh_expiration_time=refresh_expiration_time)
+ custos_authnz_token = CustosAuthnzToken(
+ user=user,
+ external_user_id=user_id,
+ provider=self.config["provider"],
+ access_token=access_token,
+ id_token=id_token,
+ refresh_token=refresh_token,
+ expiration_time=expiration_time,
+ refresh_expiration_time=refresh_expiration_time,
+ )
trans.sa_session.add(user)
trans.sa_session.add(custos_authnz_token)
@@ -189,7 +212,7 @@ class CustosAuthnz(IdentityProvider):
if len(provider_tokens) > 1:
for idx, token in enumerate(provider_tokens):
id_token_decoded = self._decode_token_no_signature(token.id_token)
- if (id_token_decoded['email'] == email):
+ if id_token_decoded["email"] == email:
index = idx
trans.sa_session.delete(provider_tokens[index])
trans.sa_session.flush()
@@ -199,7 +222,7 @@ class CustosAuthnz(IdentityProvider):
def logout(self, trans, post_logout_redirect_url=None):
try:
- redirect_url = self.config['end_session_endpoint']
+ redirect_url = self.config["end_session_endpoint"]
if post_logout_redirect_url is not None:
redirect_url += f"?redirect_uri={quote(post_logout_redirect_url)}"
return redirect_url
@@ -208,43 +231,40 @@ class CustosAuthnz(IdentityProvider):
return None
def _create_oauth2_session(self, state=None, scope=None):
- client_id = self.config['client_id']
- redirect_uri = self.config['redirect_uri']
- if (redirect_uri.startswith('http://localhost')
- and os.environ.get("OAUTHLIB_INSECURE_TRANSPORT", None) != "1"):
+ client_id = self.config["client_id"]
+ redirect_uri = self.config["redirect_uri"]
+ if redirect_uri.startswith("http://localhost") and os.environ.get("OAUTHLIB_INSECURE_TRANSPORT", None) != "1":
log.warning("Setting OAUTHLIB_INSECURE_TRANSPORT to '1' to allow plain HTTP (non-SSL) callback")
- os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = "1"
- session = OAuth2Session(client_id,
- scope=scope,
- redirect_uri=redirect_uri,
- state=state)
+ os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1"
+ session = OAuth2Session(client_id, scope=scope, redirect_uri=redirect_uri, state=state)
session.verify = self._get_verify_param()
return session
def _fetch_token(self, oauth2_session, trans):
- if self.config.get('iam_client_secret'):
+ if self.config.get("iam_client_secret"):
# Custos uses the Keycloak client secret to get the token
- client_secret = self.config['iam_client_secret']
+ client_secret = self.config["iam_client_secret"]
else:
- client_secret = self.config['client_secret']
- token_endpoint = self.config['token_endpoint']
+ client_secret = self.config["client_secret"]
+ token_endpoint = self.config["token_endpoint"]
clientIdAndSec = f"{self.config['client_id']}:{self.config['client_secret']}" # for custos
return oauth2_session.fetch_token(
token_endpoint,
client_secret=client_secret,
authorization_response=trans.request.url,
- headers={"Authorization": f"Basic {util.unicodify(base64.b64encode(util.smart_str(clientIdAndSec)))}"}, # for custos
- verify=self._get_verify_param())
+ headers={
+ "Authorization": f"Basic {util.unicodify(base64.b64encode(util.smart_str(clientIdAndSec)))}"
+ }, # for custos
+ verify=self._get_verify_param(),
+ )
def _get_userinfo(self, oauth2_session):
- userinfo_endpoint = self.config['userinfo_endpoint']
- return oauth2_session.get(userinfo_endpoint,
- verify=self._get_verify_param()).json()
+ userinfo_endpoint = self.config["userinfo_endpoint"]
+ return oauth2_session.get(userinfo_endpoint, verify=self._get_verify_param()).json()
def _get_custos_authnz_token(self, sa_session, user_id, provider):
- return sa_session.query(CustosAuthnzToken).filter_by(
- external_user_id=user_id, provider=provider).one_or_none()
+ return sa_session.query(CustosAuthnzToken).filter_by(external_user_id=user_id, provider=provider).one_or_none()
def _hash_nonce(self, nonce):
return hashlib.sha256(util.smart_str(nonce)).hexdigest()
@@ -252,48 +272,53 @@ class CustosAuthnz(IdentityProvider):
def _validate_nonce(self, trans, nonce_hash):
nonce_cookie = trans.get_cookie(name=NONCE_COOKIE_NAME)
# Delete the nonce cookie
- trans.set_cookie('', name=NONCE_COOKIE_NAME, age=-1)
+ trans.set_cookie("", name=NONCE_COOKIE_NAME, age=-1)
nonce_cookie_hash = self._hash_nonce(nonce_cookie)
if nonce_hash != nonce_cookie_hash:
raise Exception("Nonce mismatch!")
def _load_config_for_cilogon(self):
# Set cilogon endpoints
- self.config['authorization_endpoint'] = "https://cilogon.org/authorize"
- self.config['token_endpoint'] = "https://cilogon.org/oauth2/token"
- self.config['userinfo_endpoint'] = "https://cilogon.org/oauth2/userinfo"
+ self.config["authorization_endpoint"] = "https://cilogon.org/authorize"
+ self.config["token_endpoint"] = "https://cilogon.org/oauth2/token"
+ self.config["userinfo_endpoint"] = "https://cilogon.org/oauth2/userinfo"
def _load_config_for_custos(self):
- self.config['well_known_oidc_config_uri'] = self._get_well_known_uri_from_url(self.config['provider'])
- self.config['credential_url'] = f"{self.config['url'].rstrip('/')}/credentials"
+ self.config["well_known_oidc_config_uri"] = self._get_well_known_uri_from_url(self.config["provider"])
+ self.config["credential_url"] = f"{self.config['url'].rstrip('/')}/credentials"
self._get_custos_credentials()
# Set custos endpoints
clientIdAndSec = f"{self.config['client_id']}:{self.config['client_secret']}"
- eps = requests.get(self.config['well_known_oidc_config_uri'],
- headers={"Authorization": f"Basic {util.unicodify(base64.b64encode(util.smart_str(clientIdAndSec)))}"},
- verify=False,
- params={'client_id': self.config['client_id']},
- timeout=util.DEFAULT_SOCKET_TIMEOUT)
+ eps = requests.get(
+ self.config["well_known_oidc_config_uri"],
+ headers={"Authorization": f"Basic {util.unicodify(base64.b64encode(util.smart_str(clientIdAndSec)))}"},
+ verify=False,
+ params={"client_id": self.config["client_id"]},
+ timeout=util.DEFAULT_SOCKET_TIMEOUT,
+ )
well_known_oidc_config = eps.json()
self._load_well_known_oidc_config(well_known_oidc_config)
def _load_config_for_keycloak(self):
- self.config['well_known_oidc_config_uri'] = self._get_well_known_uri_from_url(self.config['provider'])
- well_known_oidc_config = self._fetch_well_known_oidc_config(self.config['well_known_oidc_config_uri'])
+ self.config["well_known_oidc_config_uri"] = self._get_well_known_uri_from_url(self.config["provider"])
+ well_known_oidc_config = self._fetch_well_known_oidc_config(self.config["well_known_oidc_config_uri"])
self._load_well_known_oidc_config(well_known_oidc_config)
def _get_custos_credentials(self):
clientIdAndSec = f"{self.config['client_id']}:{self.config['client_secret']}"
- creds = requests.get(self.config['credential_url'],
- headers={"Authorization": f"Basic {util.unicodify(base64.b64encode(util.smart_str(clientIdAndSec)))}"},
- verify=False, params={'client_id': self.config['client_id']},
- timeout=util.DEFAULT_SOCKET_TIMEOUT)
+ creds = requests.get(
+ self.config["credential_url"],
+ headers={"Authorization": f"Basic {util.unicodify(base64.b64encode(util.smart_str(clientIdAndSec)))}"},
+ verify=False,
+ params={"client_id": self.config["client_id"]},
+ timeout=util.DEFAULT_SOCKET_TIMEOUT,
+ )
credentials = creds.json()
- self.config['iam_client_secret'] = credentials['iam_client_secret']
+ self.config["iam_client_secret"] = credentials["iam_client_secret"]
def _get_well_known_uri_from_url(self, provider):
# TODO: Look up this URL from a Python library
- if provider in ['custos', 'keycloak']:
+ if provider in ["custos", "keycloak"]:
base_url = self.config["url"]
# Remove potential trailing slash to avoid "//realms"
base_url = base_url if base_url[-1] != "/" else base_url[:-1]
@@ -303,35 +328,35 @@ class CustosAuthnz(IdentityProvider):
def _fetch_well_known_oidc_config(self, well_known_uri):
try:
- return requests.get(well_known_uri,
- verify=self._get_verify_param(),
- timeout=util.DEFAULT_SOCKET_TIMEOUT).json()
+ return requests.get(
+ well_known_uri, verify=self._get_verify_param(), timeout=util.DEFAULT_SOCKET_TIMEOUT
+ ).json()
except Exception:
log.error(f"Failed to load well-known OIDC config URI: {well_known_uri}")
raise
def _load_well_known_oidc_config(self, well_known_oidc_config):
- self.config['authorization_endpoint'] = well_known_oidc_config['authorization_endpoint']
- self.config['token_endpoint'] = well_known_oidc_config['token_endpoint']
- self.config['userinfo_endpoint'] = well_known_oidc_config['userinfo_endpoint']
- self.config['end_session_endpoint'] = well_known_oidc_config.get('end_session_endpoint')
+ self.config["authorization_endpoint"] = well_known_oidc_config["authorization_endpoint"]
+ self.config["token_endpoint"] = well_known_oidc_config["token_endpoint"]
+ self.config["userinfo_endpoint"] = well_known_oidc_config["userinfo_endpoint"]
+ self.config["end_session_endpoint"] = well_known_oidc_config.get("end_session_endpoint")
def _get_verify_param(self):
"""Return 'ca_bundle' if 'verify_ssl' is true and 'ca_bundle' is configured."""
# in requests_oauthlib, the verify param can either be a boolean or a CA bundle path
- if self.config['ca_bundle'] is not None and self.config['verify_ssl']:
- return self.config['ca_bundle']
+ if self.config["ca_bundle"] is not None and self.config["verify_ssl"]:
+ return self.config["ca_bundle"]
else:
- return self.config['verify_ssl']
+ return self.config["verify_ssl"]
def _username_from_userinfo(self, trans, userinfo):
- username = userinfo.get('preferred_username', userinfo['email'])
+ username = userinfo.get("preferred_username", userinfo["email"])
if "@" in username:
- username = username.split('@')[0] # username created from username portion of email
- if (trans.sa_session.query(trans.app.model.User).filter_by(username=username).first()):
+ username = username.split("@")[0] # username created from username portion of email
+ if trans.sa_session.query(trans.app.model.User).filter_by(username=username).first():
# if username already exists in database, append integer and iterate until unique username found
count = 0
- while (trans.sa_session.query(trans.app.model.User).filter_by(username=(f"{username}{count}")).first()):
+ while trans.sa_session.query(trans.app.model.User).filter_by(username=(f"{username}{count}")).first():
count += 1
return f"{username}{count}"
else:
diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py
index 479924f74fb..7049448c39f 100644
--- a/lib/galaxy/authnz/managers.py
+++ b/lib/galaxy/authnz/managers.py
@@ -8,12 +8,12 @@ import string
import requests
from cloudauthz import CloudAuthz
-from cloudauthz.exceptions import (
- CloudAuthzBaseException
-)
+from cloudauthz.exceptions import CloudAuthzBaseException
-from galaxy import exceptions
-from galaxy import model
+from galaxy import (
+ exceptions,
+ model,
+)
from galaxy.util import (
asbool,
etree,
@@ -31,22 +31,20 @@ from .psa_authnz import (
on_the_fly_config,
PSAAuthnz,
Storage,
- Strategy
+ Strategy,
)
-
log = logging.getLogger(__name__)
# Note: This if for backward compatibility. Icons can be specified in oidc_backends_config.xml.
DEFAULT_OIDC_IDP_ICONS = {
- 'google': 'https://developers.google.com/identity/images/btn_google_signin_light_normal_web.png',
- 'elixir': 'https://elixir-europe.org/sites/default/files/images/login-button-orange.png',
- 'okta': 'https://www.okta.com/sites/all/themes/Okta/images/blog/Logos/Okta_Logo_BrightBlue_Medium.png'
+ "google": "https://developers.google.com/identity/images/btn_google_signin_light_normal_web.png",
+ "elixir": "https://elixir-europe.org/sites/default/files/images/login-button-orange.png",
+ "okta": "https://www.okta.com/sites/all/themes/Okta/images/blog/Logos/Okta_Logo_BrightBlue_Medium.png",
}
class AuthnzManager:
-
def __init__(self, app, oidc_config_file, oidc_backends_config_file):
"""
:type app: galaxy.app.UniverseApplication
@@ -66,35 +64,42 @@ class AuthnzManager:
try:
tree = parse_xml(config_file)
root = tree.getroot()
- if root.tag != 'OIDC':
- raise etree.ParseError("The root element in OIDC_Config xml file is expected to be `OIDC`, "
- "found `{}` instead -- unable to continue.".format(root.tag))
+ if root.tag != "OIDC":
+ raise etree.ParseError(
+ "The root element in OIDC_Config xml file is expected to be `OIDC`, "
+ "found `{}` instead -- unable to continue.".format(root.tag)
+ )
for child in root:
- if child.tag != 'Setter':
- log.error("Expect a node with `Setter` tag, found a node with `{}` tag instead; "
- "skipping this node.".format(child.tag))
+ if child.tag != "Setter":
+ log.error(
+ "Expect a node with `Setter` tag, found a node with `{}` tag instead; "
+ "skipping this node.".format(child.tag)
+ )
continue
- if 'Property' not in child.attrib or 'Value' not in child.attrib or 'Type' not in child.attrib:
- log.error("Could not find the node attributes `Property` and/or `Value` and/or `Type`;"
- " found these attributes: `{}`; skipping this node.".format(child.attrib))
+ if "Property" not in child.attrib or "Value" not in child.attrib or "Type" not in child.attrib:
+ log.error(
+ "Could not find the node attributes `Property` and/or `Value` and/or `Type`;"
+ " found these attributes: `{}`; skipping this node.".format(child.attrib)
+ )
continue
try:
- if child.get('Type') == "bool":
+ if child.get("Type") == "bool":
func = string_as_bool
else:
- func = getattr(builtins, child.get('Type'))
+ func = getattr(builtins, child.get("Type"))
except AttributeError:
- log.error("The value of attribute `Type`, `{}`, is not a valid built-in type;"
- " skipping this node").format(child.get('Type'))
+ log.error(
+ "The value of attribute `Type`, `{}`, is not a valid built-in type;" " skipping this node"
+ ).format(child.get("Type"))
continue
- self.oidc_config[child.get('Property')] = func(child.get('Value'))
+ self.oidc_config[child.get("Property")] = func(child.get("Value"))
except ImportError:
raise
except etree.ParseError as e:
raise etree.ParseError(f"Invalid configuration at `{config_file}`: {e} -- unable to continue.")
def _get_idp_icon(self, idp):
- return self.oidc_backends_config[idp].get('icon') or DEFAULT_OIDC_IDP_ICONS.get(idp)
+ return self.oidc_backends_config[idp].get("icon") or DEFAULT_OIDC_IDP_ICONS.get(idp)
def _parse_oidc_backends_config(self, config_file):
self.oidc_backends_config = {}
@@ -102,26 +107,30 @@ class AuthnzManager:
try:
tree = parse_xml(config_file)
root = tree.getroot()
- if root.tag != 'OIDC':
- raise etree.ParseError("The root element in OIDC config xml file is expected to be `OIDC`, "
- "found `{}` instead -- unable to continue.".format(root.tag))
+ if root.tag != "OIDC":
+ raise etree.ParseError(
+ "The root element in OIDC config xml file is expected to be `OIDC`, "
+ "found `{}` instead -- unable to continue.".format(root.tag)
+ )
for child in root:
- if child.tag != 'provider':
- log.error("Expect a node with `provider` tag, found a node with `{}` tag instead; "
- "skipping the node.".format(child.tag))
+ if child.tag != "provider":
+ log.error(
+ "Expect a node with `provider` tag, found a node with `{}` tag instead; "
+ "skipping the node.".format(child.tag)
+ )
continue
- if 'name' not in child.attrib:
+ if "name" not in child.attrib:
log.error(f"Could not find a node attribute 'name'; skipping the node '{child.tag}'.")
continue
- idp = child.get('name').lower()
+ idp = child.get("name").lower()
if idp in BACKENDS_NAME:
self.oidc_backends_config[idp] = self._parse_idp_config(child)
- self.oidc_backends_implementation[idp] = 'psa'
- self.app.config.oidc[idp] = {'icon': self._get_idp_icon(idp)}
+ self.oidc_backends_implementation[idp] = "psa"
+ self.app.config.oidc[idp] = {"icon": self._get_idp_icon(idp)}
elif idp in KEYCLOAK_BACKENDS:
self.oidc_backends_config[idp] = self._parse_custos_config(child)
- self.oidc_backends_implementation[idp] = 'custos'
- self.app.config.oidc[idp] = {'icon': self._get_idp_icon(idp)}
+ self.oidc_backends_implementation[idp] = "custos"
+ self.app.config.oidc[idp] = {"icon": self._get_idp_icon(idp)}
else:
raise etree.ParseError("Unknown provider specified")
if len(self.oidc_backends_config) == 0:
@@ -133,40 +142,42 @@ class AuthnzManager:
def _parse_idp_config(self, config_xml):
rtv = {
- 'client_id': config_xml.find('client_id').text,
- 'client_secret': config_xml.find('client_secret').text,
- 'redirect_uri': config_xml.find('redirect_uri').text,
- 'enable_idp_logout': asbool(config_xml.findtext('enable_idp_logout', 'false'))}
- if config_xml.find('prompt') is not None:
- rtv['prompt'] = config_xml.find('prompt').text
- if config_xml.find('api_url') is not None:
- rtv['api_url'] = config_xml.find('api_url').text
- if config_xml.find('url') is not None:
- rtv['url'] = config_xml.find('url').text
- if config_xml.find('icon') is not None:
- rtv['icon'] = config_xml.find('icon').text
- if config_xml.find('extra_scopes') is not None:
- rtv['extra_scopes'] = listify(config_xml.find('extra_scopes').text)
+ "client_id": config_xml.find("client_id").text,
+ "client_secret": config_xml.find("client_secret").text,
+ "redirect_uri": config_xml.find("redirect_uri").text,
+ "enable_idp_logout": asbool(config_xml.findtext("enable_idp_logout", "false")),
+ }
+ if config_xml.find("prompt") is not None:
+ rtv["prompt"] = config_xml.find("prompt").text
+ if config_xml.find("api_url") is not None:
+ rtv["api_url"] = config_xml.find("api_url").text
+ if config_xml.find("url") is not None:
+ rtv["url"] = config_xml.find("url").text
+ if config_xml.find("icon") is not None:
+ rtv["icon"] = config_xml.find("icon").text
+ if config_xml.find("extra_scopes") is not None:
+ rtv["extra_scopes"] = listify(config_xml.find("extra_scopes").text)
return rtv
def _parse_custos_config(self, config_xml):
rtv = {
- 'url': config_xml.find('url').text,
- 'client_id': config_xml.find('client_id').text,
- 'client_secret': config_xml.find('client_secret').text,
- 'redirect_uri': config_xml.find('redirect_uri').text,
- 'enable_idp_logout': asbool(config_xml.findtext('enable_idp_logout', 'false'))}
- if config_xml.find('credential_url') is not None:
- rtv['credential_url'] = config_xml.find('credential_url').text
- if config_xml.find('well_known_oidc_config_uri') is not None:
- rtv['well_known_oidc_config_uri'] = config_xml.find('well_known_oidc_config_uri').text
- if config_xml.findall('allowed_idp') is not None:
- self.allowed_idps = list(map(lambda idp: idp.text, config_xml.findall('allowed_idp')))
- if config_xml.find('ca_bundle') is not None:
- rtv['ca_bundle'] = config_xml.find('ca_bundle').text
- if config_xml.find('icon') is not None:
- rtv['icon'] = config_xml.find('icon').text
+ "url": config_xml.find("url").text,
+ "client_id": config_xml.find("client_id").text,
+ "client_secret": config_xml.find("client_secret").text,
+ "redirect_uri": config_xml.find("redirect_uri").text,
+ "enable_idp_logout": asbool(config_xml.findtext("enable_idp_logout", "false")),
+ }
+ if config_xml.find("credential_url") is not None:
+ rtv["credential_url"] = config_xml.find("credential_url").text
+ if config_xml.find("well_known_oidc_config_uri") is not None:
+ rtv["well_known_oidc_config_uri"] = config_xml.find("well_known_oidc_config_uri").text
+ if config_xml.findall("allowed_idp") is not None:
+ self.allowed_idps = list(map(lambda idp: idp.text, config_xml.findall("allowed_idp")))
+ if config_xml.find("ca_bundle") is not None:
+ rtv["ca_bundle"] = config_xml.find("ca_bundle").text
+ if config_xml.find("icon") is not None:
+ rtv["icon"] = config_xml.find("icon").text
return rtv
def get_allowed_idps(self):
@@ -188,22 +199,37 @@ class AuthnzManager:
identity_provider_class = self._get_identity_provider_class(self.oidc_backends_implementation[provider])
try:
if provider in KEYCLOAK_BACKENDS:
- return True, "", identity_provider_class(unified_provider_name, self.oidc_config, self.oidc_backends_config[unified_provider_name], idphint=idphint)
+ return (
+ True,
+ "",
+ identity_provider_class(
+ unified_provider_name,
+ self.oidc_config,
+ self.oidc_backends_config[unified_provider_name],
+ idphint=idphint,
+ ),
+ )
else:
- return True, "", identity_provider_class(unified_provider_name, self.oidc_config, self.oidc_backends_config[unified_provider_name])
+ return (
+ True,
+ "",
+ identity_provider_class(
+ unified_provider_name, self.oidc_config, self.oidc_backends_config[unified_provider_name]
+ ),
+ )
except Exception as e:
- log.exception(f'An error occurred when loading {identity_provider_class.__name__}')
+ log.exception(f"An error occurred when loading {identity_provider_class.__name__}")
return False, unicodify(e), None
else:
- msg = f'The requested identity provider, `{provider}`, is not a recognized/expected provider.'
+ msg = f"The requested identity provider, `{provider}`, is not a recognized/expected provider."
log.debug(msg)
return False, msg, None
@staticmethod
def _get_identity_provider_class(implementation):
- if implementation == 'psa':
+ if implementation == "psa":
return PSAAuthnz
- elif implementation == 'custos':
+ elif implementation == "custos":
return CustosAuthnz
else:
return None
@@ -215,17 +241,22 @@ class AuthnzManager:
strategy = Strategy(request, None, Storage, backend.config)
on_the_fly_config(sa_session)
try:
- config['id_token'] = cloudauthz.authn.get_id_token(strategy)
+ config["id_token"] = cloudauthz.authn.get_id_token(strategy)
except requests.exceptions.HTTPError as e:
- msg = "Sign-out from Galaxy and remove its access from `{}`, then log back in using `{}` " \
- "account.".format(self._unify_provider_name(cloudauthz.authn.provider), cloudauthz.authn.uid)
- log.debug("Failed to get/refresh ID token for user with ID `{}` for assuming authz_id `{}`. "
- "User may not have a refresh token. If the problem persists, set the `prompt` key to "
- "`consent` in `oidc_backends_config.xml`, then restart Galaxy and ask user to: {}"
- "Error Message: `{}`".format(user_id, cloudauthz.id, msg, e.response.text))
+ msg = (
+ "Sign-out from Galaxy and remove its access from `{}`, then log back in using `{}` "
+ "account.".format(self._unify_provider_name(cloudauthz.authn.provider), cloudauthz.authn.uid)
+ )
+ log.debug(
+ "Failed to get/refresh ID token for user with ID `{}` for assuming authz_id `{}`. "
+ "User may not have a refresh token. If the problem persists, set the `prompt` key to "
+ "`consent` in `oidc_backends_config.xml`, then restart Galaxy and ask user to: {}"
+ "Error Message: `{}`".format(user_id, cloudauthz.id, msg, e.response.text)
+ )
raise exceptions.AuthenticationFailed(
err_msg="An error occurred getting your ID token. {}. If the problem persists, please "
- "contact Galaxy admin.".format(msg))
+ "contact Galaxy admin.".format(msg)
+ )
return config
@staticmethod
@@ -233,12 +264,14 @@ class AuthnzManager:
qres = trans.sa_session.query(model.UserAuthnzToken).get(authn_id)
if qres is None:
msg = "Authentication record with the given `authn_id` (`{}`) not found.".format(
- trans.security.encode_id(authn_id))
+ trans.security.encode_id(authn_id)
+ )
log.debug(msg)
raise exceptions.ObjectNotFound(msg)
if qres.user_id != trans.user.id:
- msg = "The request authentication with ID `{}` is not accessible to user with ID " \
- "`{}`.".format(trans.security.encode_id(authn_id), trans.security.encode_id(trans.user.id))
+ msg = "The request authentication with ID `{}` is not accessible to user with ID " "`{}`.".format(
+ trans.security.encode_id(authn_id), trans.security.encode_id(trans.user.id)
+ )
log.warning(msg)
raise exceptions.ItemAccessibilityException(msg)
@@ -264,8 +297,10 @@ class AuthnzManager:
if qres is None:
raise exceptions.ObjectNotFound("An authorization configuration with given ID not found.")
if user_id != qres.user_id:
- msg = "The request authorization configuration (with ID:`{}`) is not accessible for user with " \
- "ID:`{}`.".format(qres.id, user_id)
+ msg = (
+ "The request authorization configuration (with ID:`{}`) is not accessible for user with "
+ "ID:`{}`.".format(qres.id, user_id)
+ )
log.warning(msg)
raise exceptions.ItemAccessibilityException(msg)
return qres
@@ -284,14 +319,22 @@ class AuthnzManager:
if success is False:
return False, message, None
elif provider in KEYCLOAK_BACKENDS:
- if (self.allowed_idps and (idphint not in self.allowed_idps)):
- msg = f'An error occurred when authenticating a user. Invalid EntityID: `{idphint}`'
+ if self.allowed_idps and (idphint not in self.allowed_idps):
+ msg = f"An error occurred when authenticating a user. Invalid EntityID: `{idphint}`"
log.exception(msg)
return False, msg, None
- return True, f"Redirecting to the `{provider}` identity provider for authentication", backend.authenticate(trans, idphint)
- return True, f"Redirecting to the `{provider}` identity provider for authentication", backend.authenticate(trans)
+ return (
+ True,
+ f"Redirecting to the `{provider}` identity provider for authentication",
+ backend.authenticate(trans, idphint),
+ )
+ return (
+ True,
+ f"Redirecting to the `{provider}` identity provider for authentication",
+ backend.authenticate(trans),
+ )
except Exception:
- msg = f'An error occurred when authenticating a user on `{provider}` identity provider'
+ msg = f"An error occurred when authenticating a user on `{provider}` identity provider"
log.exception(msg)
return False, msg, None
@@ -304,7 +347,7 @@ class AuthnzManager:
except exceptions.AuthenticationFailed:
raise
except Exception:
- msg = f'An error occurred when handling callback from `{provider}` identity provider. Please contact an administrator for assistance.'
+ msg = f"An error occurred when handling callback from `{provider}` identity provider. Please contact an administrator for assistance."
log.exception(msg)
return False, msg, (None, None)
@@ -318,7 +361,7 @@ class AuthnzManager:
log.exception("Error creating user")
raise
except Exception:
- msg = f'An error occurred when creating a user with `{provider}` identity provider. Please contact an administrator for assistance.'
+ msg = f"An error occurred when creating a user with `{provider}` identity provider. Please contact an administrator for assistance."
log.exception(msg)
return False, msg, (None, None)
@@ -338,7 +381,7 @@ class AuthnzManager:
try:
# check if logout is enabled for this idp and return false if not
unified_provider_name = self._unify_provider_name(provider)
- if self.oidc_backends_config[unified_provider_name]['enable_idp_logout'] is False:
+ if self.oidc_backends_config[unified_provider_name]["enable_idp_logout"] is False:
return False, f"IDP logout is not enabled for {provider}", None
success, message, backend = self._get_authnz_backend(provider)
@@ -346,7 +389,7 @@ class AuthnzManager:
return False, message, None
return True, message, backend.logout(trans, post_logout_redirect_url)
except Exception:
- msg = f'An error occurred when logging out from `{provider}` identity provider. Please contact an administrator for assistance.'
+ msg = f"An error occurred when logging out from `{provider}` identity provider. Please contact an administrator for assistance."
log.exception(msg)
return False, msg, None
@@ -359,8 +402,11 @@ class AuthnzManager:
return backend.disconnect(provider, trans, email, disconnect_redirect_url)
return backend.disconnect(provider, trans, disconnect_redirect_url)
except Exception:
- msg = 'An error occurred when disconnecting authentication with `{}` identity provider for user `{}`' \
- .format(provider, trans.user.username)
+ msg = (
+ "An error occurred when disconnecting authentication with `{}` identity provider for user `{}`".format(
+ provider, trans.user.username
+ )
+ )
log.exception(msg)
return False, msg, None
@@ -399,8 +445,11 @@ class AuthnzManager:
config = self._extend_cloudauthz_config(cloudauthz, request, sa_session, user_id)
try:
ca = CloudAuthz()
- log.info("Requesting credentials using CloudAuthz with config id `{}` on be half of user `{}`.".format(
- cloudauthz.id, user_id))
+ log.info(
+ "Requesting credentials using CloudAuthz with config id `{}` on be half of user `{}`.".format(
+ cloudauthz.id, user_id
+ )
+ )
credentials = ca.authorize(cloudauthz.provider, config)
return credentials
except CloudAuthzBaseException as e:
@@ -439,12 +488,18 @@ class AuthnzManager:
:rtype: str
:return: The filename to which credentials are written.
"""
- filename = os.path.abspath(os.path.join(new_file_path,
- "cd_" + ''.join(random.SystemRandom().choice(
- string.ascii_uppercase + string.digits) for _ in range(11))))
+ filename = os.path.abspath(
+ os.path.join(
+ new_file_path,
+ "cd_"
+ + "".join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(11)),
+ )
+ )
credentials = self.get_cloud_access_credentials(cloudauthz, sa_session, user_id, request)
- log.info("Writting credentials generated using CloudAuthz with config id `{}` to the following file: `{}`"
- "".format(cloudauthz.id, filename))
+ log.info(
+ "Writting credentials generated using CloudAuthz with config id `{}` to the following file: `{}`"
+ "".format(cloudauthz.id, filename)
+ )
with open(filename, "w") as f:
f.write(json.dumps(credentials))
return filename
diff --git a/lib/galaxy/authnz/psa_authnz.py b/lib/galaxy/authnz/psa_authnz.py
index 2139ab698a8..de6a6b4d576 100644
--- a/lib/galaxy/authnz/psa_authnz.py
+++ b/lib/galaxy/authnz/psa_authnz.py
@@ -1,39 +1,48 @@
import json
import requests
-from social_core.actions import do_auth, do_complete, do_disconnect
+from social_core.actions import (
+ do_auth,
+ do_complete,
+ do_disconnect,
+)
from social_core.backends.utils import get_backend
from social_core.strategy import BaseStrategy
-from social_core.utils import module_member, setting_name
+from social_core.utils import (
+ module_member,
+ setting_name,
+)
from sqlalchemy.exc import IntegrityError
from galaxy.exceptions import MalformedContents
from galaxy.util import DEFAULT_SOCKET_TIMEOUT
from ..authnz import IdentityProvider
-from ..model import PSAAssociation, PSACode, PSANonce, PSAPartial, UserAuthnzToken
-
+from ..model import (
+ PSAAssociation,
+ PSACode,
+ PSANonce,
+ PSAPartial,
+ UserAuthnzToken,
+)
# key: a component name which PSA requests.
# value: is the name of a class associated with that key.
-DEFAULTS = {
- 'STRATEGY': 'Strategy',
- 'STORAGE': 'Storage'
-}
+DEFAULTS = {"STRATEGY": "Strategy", "STORAGE": "Storage"}
BACKENDS = {
- 'google': 'social_core.backends.google_openidconnect.GoogleOpenIdConnect',
- 'globus': 'social_core.backends.globus.GlobusOpenIdConnect',
- 'elixir': 'social_core.backends.elixir.ElixirOpenIdConnect',
- 'okta': 'social_core.backends.okta_openidconnect.OktaOpenIdConnect',
- 'azure': 'social_core.backends.azuread_tenant.AzureADTenantOAuth2'
+ "google": "social_core.backends.google_openidconnect.GoogleOpenIdConnect",
+ "globus": "social_core.backends.globus.GlobusOpenIdConnect",
+ "elixir": "social_core.backends.elixir.ElixirOpenIdConnect",
+ "okta": "social_core.backends.okta_openidconnect.OktaOpenIdConnect",
+ "azure": "social_core.backends.azuread_tenant.AzureADTenantOAuth2",
}
BACKENDS_NAME = {
- 'google': 'google-openidconnect',
- 'globus': 'globus',
- 'elixir': 'elixir',
- 'okta': 'okta-openidconnect',
- 'azure': 'azuread-tenant-oauth2'
+ "google": "google-openidconnect",
+ "globus": "globus",
+ "elixir": "elixir",
+ "okta": "okta-openidconnect",
+ "azure": "azuread-tenant-oauth2",
}
AUTH_PIPELINE = (
@@ -41,67 +50,52 @@ AUTH_PIPELINE = (
# format to create the user instance later. On some cases the details are
# already part of the auth response from the provider, but sometimes this
# could hit a provider API.
- 'social_core.pipeline.social_auth.social_details',
-
+ "social_core.pipeline.social_auth.social_details",
# Get the social uid from whichever service we're authing thru. The uid is
# the unique identifier of the given user in the provider.
- 'social_core.pipeline.social_auth.social_uid',
-
+ "social_core.pipeline.social_auth.social_uid",
# Verifies that the current auth process is valid within the current
# project, this is where emails and domains allowlists are applied (if
# defined).
- 'social_core.pipeline.social_auth.auth_allowed',
-
+ "social_core.pipeline.social_auth.auth_allowed",
# Checks if the decoded response contains all the required fields such
# as an ID token or a refresh token.
- 'galaxy.authnz.psa_authnz.contains_required_data',
-
- 'galaxy.authnz.psa_authnz.verify',
-
+ "galaxy.authnz.psa_authnz.contains_required_data",
+ "galaxy.authnz.psa_authnz.verify",
# Checks if the current social-account is already associated in the site.
- 'social_core.pipeline.social_auth.social_user',
-
+ "social_core.pipeline.social_auth.social_user",
# Make up a username for this person, appends a random string at the end if
# there's any collision.
- 'social_core.pipeline.user.get_username',
-
+ "social_core.pipeline.user.get_username",
# Send a validation email to the user to verify its email address.
# 'social_core.pipeline.mail.mail_validation',
-
# Associates the current social details with another user account with
# a similar email address.
- 'social_core.pipeline.social_auth.associate_by_email',
-
+ "social_core.pipeline.social_auth.associate_by_email",
# Create a user account if we haven't found one yet.
- 'social_core.pipeline.user.create_user',
-
+ "social_core.pipeline.user.create_user",
# Create the record that associated the social account with this user.
- 'social_core.pipeline.social_auth.associate_user',
-
+ "social_core.pipeline.social_auth.associate_user",
# Populate the extra_data field in the social record with the values
# specified by settings (and the default ones like access_token, etc).
- 'social_core.pipeline.social_auth.load_extra_data',
-
+ "social_core.pipeline.social_auth.load_extra_data",
# Update the user record with any changed info from the auth service.
- 'social_core.pipeline.user.user_details'
+ "social_core.pipeline.user.user_details",
)
-DISCONNECT_PIPELINE = (
- 'galaxy.authnz.psa_authnz.allowed_to_disconnect',
- 'galaxy.authnz.psa_authnz.disconnect'
-)
+DISCONNECT_PIPELINE = ("galaxy.authnz.psa_authnz.allowed_to_disconnect", "galaxy.authnz.psa_authnz.disconnect")
class PSAAuthnz(IdentityProvider):
def __init__(self, provider, oidc_config, oidc_backend_config):
- self.config = {'provider': provider.lower()}
+ self.config = {"provider": provider.lower()}
for key, value in oidc_config.items():
self.config[setting_name(key)] = value
- self.config[setting_name('USER_MODEL')] = 'models.User'
- self.config['SOCIAL_AUTH_PIPELINE'] = AUTH_PIPELINE
- self.config['DISCONNECT_PIPELINE'] = DISCONNECT_PIPELINE
- self.config[setting_name('AUTHENTICATION_BACKENDS')] = (BACKENDS[provider],)
+ self.config[setting_name("USER_MODEL")] = "models.User"
+ self.config["SOCIAL_AUTH_PIPELINE"] = AUTH_PIPELINE
+ self.config["DISCONNECT_PIPELINE"] = DISCONNECT_PIPELINE
+ self.config[setting_name("AUTHENTICATION_BACKENDS")] = (BACKENDS[provider],)
self.config["VERIFY_SSL"] = oidc_config.get("VERIFY_SSL")
self.config["REQUESTS_TIMEOUT"] = oidc_config.get("REQUESTS_TIMEOUT")
@@ -111,90 +105,95 @@ class PSAAuthnz(IdentityProvider):
# logging in a user. If this setting is set to false, the `_login_user`
# would not be called, and as a result Galaxy would not know who is
# the just logged-in user.
- self.config[setting_name('INACTIVE_USER_LOGIN')] = True
+ self.config[setting_name("INACTIVE_USER_LOGIN")] = True
if provider in BACKENDS_NAME:
self._setup_idp(oidc_backend_config)
# Secondary AuthZ with Google identities is currently supported
if provider != "google":
- if 'SOCIAL_AUTH_SECONDARY_AUTH_PROVIDER' in self.config:
+ if "SOCIAL_AUTH_SECONDARY_AUTH_PROVIDER" in self.config:
del self.config["SOCIAL_AUTH_SECONDARY_AUTH_PROVIDER"]
- if 'SOCIAL_AUTH_SECONDARY_AUTH_ENDPOINT' in self.config:
+ if "SOCIAL_AUTH_SECONDARY_AUTH_ENDPOINT" in self.config:
del self.config["SOCIAL_AUTH_SECONDARY_AUTH_ENDPOINT"]
def _setup_idp(self, oidc_backend_config):
- self.config[setting_name('AUTH_EXTRA_ARGUMENTS')] = {'access_type': 'offline'}
- self.config['KEY'] = oidc_backend_config.get('client_id')
- self.config['SECRET'] = oidc_backend_config.get('client_secret')
- self.config['redirect_uri'] = oidc_backend_config.get('redirect_uri')
- self.config['EXTRA_SCOPES'] = oidc_backend_config.get('extra_scopes')
- if oidc_backend_config.get('prompt') is not None:
- self.config[setting_name('AUTH_EXTRA_ARGUMENTS')]['prompt'] = oidc_backend_config.get('prompt')
- if oidc_backend_config.get('api_url') is not None:
- self.config[setting_name('API_URL')] = oidc_backend_config.get('api_url')
- if oidc_backend_config.get('url') is not None:
- self.config[setting_name('URL')] = oidc_backend_config.get('url')
+ self.config[setting_name("AUTH_EXTRA_ARGUMENTS")] = {"access_type": "offline"}
+ self.config["KEY"] = oidc_backend_config.get("client_id")
+ self.config["SECRET"] = oidc_backend_config.get("client_secret")
+ self.config["redirect_uri"] = oidc_backend_config.get("redirect_uri")
+ self.config["EXTRA_SCOPES"] = oidc_backend_config.get("extra_scopes")
+ if oidc_backend_config.get("prompt") is not None:
+ self.config[setting_name("AUTH_EXTRA_ARGUMENTS")]["prompt"] = oidc_backend_config.get("prompt")
+ if oidc_backend_config.get("api_url") is not None:
+ self.config[setting_name("API_URL")] = oidc_backend_config.get("api_url")
+ if oidc_backend_config.get("url") is not None:
+ self.config[setting_name("URL")] = oidc_backend_config.get("url")
def _get_helper(self, name, do_import=False):
this_config = self.config.get(setting_name(name), DEFAULTS.get(name, None))
return do_import and module_member(this_config) or this_config
def _load_backend(self, strategy, redirect_uri):
- backends = self._get_helper('AUTHENTICATION_BACKENDS')
- backend = get_backend(backends, BACKENDS_NAME[self.config['provider']])
+ backends = self._get_helper("AUTHENTICATION_BACKENDS")
+ backend = get_backend(backends, BACKENDS_NAME[self.config["provider"]])
return backend(strategy, redirect_uri)
def _login_user(self, backend, user, social_user):
- self.config['user'] = user
+ self.config["user"] = user
def authenticate(self, trans):
on_the_fly_config(trans.sa_session)
strategy = Strategy(trans.request, trans.session, Storage, self.config)
- backend = self._load_backend(strategy, self.config['redirect_uri'])
- if backend.name is BACKENDS_NAME["google"] and \
- "SOCIAL_AUTH_SECONDARY_AUTH_PROVIDER" in self.config and \
- "SOCIAL_AUTH_SECONDARY_AUTH_ENDPOINT" in self.config:
+ backend = self._load_backend(strategy, self.config["redirect_uri"])
+ if (
+ backend.name is BACKENDS_NAME["google"]
+ and "SOCIAL_AUTH_SECONDARY_AUTH_PROVIDER" in self.config
+ and "SOCIAL_AUTH_SECONDARY_AUTH_ENDPOINT" in self.config
+ ):
backend.DEFAULT_SCOPE.append("https://www.googleapis.com/auth/cloud-platform")
- if self.config['EXTRA_SCOPES'] is not None:
- backend.DEFAULT_SCOPE.extend(self.config['EXTRA_SCOPES'])
+ if self.config["EXTRA_SCOPES"] is not None:
+ backend.DEFAULT_SCOPE.extend(self.config["EXTRA_SCOPES"])
return do_auth(backend)
def callback(self, state_token, authz_code, trans, login_redirect_url):
on_the_fly_config(trans.sa_session)
- self.config[setting_name('LOGIN_REDIRECT_URL')] = login_redirect_url
+ self.config[setting_name("LOGIN_REDIRECT_URL")] = login_redirect_url
strategy = Strategy(trans.request, trans.session, Storage, self.config)
strategy.session_set(f"{BACKENDS_NAME[self.config['provider']]}_state", state_token)
- backend = self._load_backend(strategy, self.config['redirect_uri'])
+ backend = self._load_backend(strategy, self.config["redirect_uri"])
redirect_url = do_complete(
backend,
login=lambda backend, user, social_user: self._login_user(backend, user, social_user),
user=trans.user,
- state=state_token)
- return redirect_url, self.config.get('user', None)
+ state=state_token,
+ )
+ return redirect_url, self.config.get("user", None)
def disconnect(self, provider, trans, disconnect_redirect_url=None, association_id=None):
on_the_fly_config(trans.sa_session)
- self.config[setting_name('DISCONNECT_REDIRECT_URL')] =\
+ self.config[setting_name("DISCONNECT_REDIRECT_URL")] = (
disconnect_redirect_url if disconnect_redirect_url is not None else ()
+ )
strategy = Strategy(trans.request, trans.session, Storage, self.config)
- backend = self._load_backend(strategy, self.config['redirect_uri'])
+ backend = self._load_backend(strategy, self.config["redirect_uri"])
response = do_disconnect(backend, trans.user, association_id)
if isinstance(response, str):
return True, "", response
- return response.get('success', False), response.get('message', ""), ""
+ return response.get("success", False), response.get("message", ""), ""
class Strategy(BaseStrategy):
-
def __init__(self, request, session, storage, config, tpl=None):
self.request = request
self.session = session if session else {}
self.config = config
- self.config['SOCIAL_AUTH_REDIRECT_IS_HTTPS'] = True if self.request and self.request.host.startswith('https:') else False
- self.config['SOCIAL_AUTH_GOOGLE_OPENIDCONNECT_EXTRA_DATA'] = ['id_token']
+ self.config["SOCIAL_AUTH_REDIRECT_IS_HTTPS"] = (
+ True if self.request and self.request.host.startswith("https:") else False
+ )
+ self.config["SOCIAL_AUTH_GOOGLE_OPENIDCONNECT_EXTRA_DATA"] = ["id_token"]
super().__init__(storage, tpl)
def get_setting(self, name):
@@ -207,7 +206,7 @@ class Strategy(BaseStrategy):
self.session[name] = value
def session_pop(self, name):
- raise NotImplementedError('Not implemented.')
+ raise NotImplementedError("Not implemented.")
def request_data(self, merge=True):
if not self.request:
@@ -215,7 +214,7 @@ class Strategy(BaseStrategy):
if merge:
data = self.request.GET.copy()
data.update(self.request.POST)
- elif self.request.method == 'POST':
+ elif self.request.method == "POST":
data = self.request.POST
else:
data = self.request.GET
@@ -226,23 +225,25 @@ class Strategy(BaseStrategy):
return self.request.host
def build_absolute_uri(self, path=None):
- path = path or ''
- if path.startswith('http://') or path.startswith('https://'):
+ path = path or ""
+ if path.startswith("http://") or path.startswith("https://"):
return path
if self.request:
- return \
- self.request.host +\
- '/authnz' + ('/' + self.config.get('provider')) if self.config.get('provider', None) is not None else ''
+ return (
+ self.request.host + "/authnz" + ("/" + self.config.get("provider"))
+ if self.config.get("provider", None) is not None
+ else ""
+ )
return path
def redirect(self, url):
return url
def html(self, content):
- raise NotImplementedError('Not implemented.')
+ raise NotImplementedError("Not implemented.")
def render_html(self, tpl=None, html=None, context=None):
- raise NotImplementedError('Not implemented.')
+ raise NotImplementedError("Not implemented.")
def start(self):
self.clean_partial_pipeline()
@@ -316,11 +317,13 @@ def contains_required_data(response=None, is_new=False, **kwargs):
:rtype: void
:return: Raises an exception if any of the required arguments is missing, and pass if all are given.
"""
- hint_msg = "Visit the identity provider's permitted applications page " \
- "(e.g., visit `https://myaccount.google.com/u/0/permissions` " \
- "for Google), then revoke the access of this Galaxy instance, " \
- "and then retry to login. If the problem persists, contact " \
- "the Admin of this Galaxy instance."
+ hint_msg = (
+ "Visit the identity provider's permitted applications page "
+ "(e.g., visit `https://myaccount.google.com/u/0/permissions` "
+ "for Google), then revoke the access of this Galaxy instance, "
+ "and then retry to login. If the problem persists, contact "
+ "the Admin of this Galaxy instance."
+ )
if response is None or not isinstance(response, dict):
# This can happen only if PSA is not able to decode the `authnz code`
# sent back from the identity provider. PSA internally handles such
@@ -356,10 +359,10 @@ def verify(strategy=None, response=None, details=None, **kwargs):
result = requests.post(
f"https://iam.googleapis.com/v1/projects/-/serviceAccounts/{endpoint}:getIamPolicy",
headers={
- 'Authorization': f"Bearer {response.get('access_token')}",
- 'Accept': 'application/json',
+ "Authorization": f"Bearer {response.get('access_token')}",
+ "Accept": "application/json",
},
- timeout=DEFAULT_SOCKET_TIMEOUT
+ timeout=DEFAULT_SOCKET_TIMEOUT,
)
res = json.loads(result.content)
if result.status_code == requests.codes.ok:
@@ -380,8 +383,9 @@ def verify(strategy=None, response=None, details=None, **kwargs):
raise Exception(f"`{provider}` is an unsupported secondary authorization provider, contact admin.")
-def allowed_to_disconnect(name=None, user=None, user_storage=None, strategy=None,
- backend=None, request=None, details=None, **kwargs):
+def allowed_to_disconnect(
+ name=None, user=None, user_storage=None, strategy=None, backend=None, request=None, details=None, **kwargs
+):
"""
Disconnect is the process of disassociating a Galaxy user and a third-party authnz.
In other words, it is the process of removing any access and/or ID tokens of a user.
@@ -402,8 +406,9 @@ def allowed_to_disconnect(name=None, user=None, user_storage=None, strategy=None
"""
-def disconnect(name=None, user=None, user_storage=None, strategy=None,
- backend=None, request=None, details=None, **kwargs):
+def disconnect(
+ name=None, user=None, user_storage=None, strategy=None, backend=None, request=None, details=None, **kwargs
+):
"""
Disconnect is the process of disassociating a Galaxy user and a third-party authnz.
In other words, it is the process of removing any access and/or ID tokens of a user.
@@ -422,10 +427,13 @@ def disconnect(name=None, user=None, user_storage=None, strategy=None,
"""
sa_session = user_storage.sa_session
- user_authnz = sa_session.query(user_storage).filter(user_storage.table.c.user_id == user.id,
- user_storage.table.c.provider == name).first()
+ user_authnz = (
+ sa_session.query(user_storage)
+ .filter(user_storage.table.c.user_id == user.id, user_storage.table.c.provider == name)
+ .first()
+ )
if user_authnz is None:
- return {'success': False, 'message': 'Not authenticated by any identity providers.'}
+ return {"success": False, "message": "Not authenticated by any identity providers."}
# option A
sa_session.delete(user_authnz)
# option B
diff --git a/lib/galaxy/celery/__init__.py b/lib/galaxy/celery/__init__.py
index 5bc16fb0d96..6aa5a4c8535 100644
--- a/lib/galaxy/celery/__init__.py
+++ b/lib/galaxy/celery/__init__.py
@@ -15,7 +15,10 @@ from galaxy.config import Configuration
from galaxy.main_config import find_config
from galaxy.util.custom_logging import get_logger
from galaxy.util.properties import load_app_properties
-from ._serialization import schema_dumps, schema_loads
+from ._serialization import (
+ schema_dumps,
+ schema_loads,
+)
log = get_logger(__name__)
@@ -23,11 +26,12 @@ log = get_logger(__name__)
@lru_cache(maxsize=1)
def get_galaxy_app():
import galaxy.app
+
if galaxy.app.app:
return galaxy.app.app
kwargs = get_app_properties()
if kwargs:
- kwargs['check_migrate_databases'] = False
+ kwargs["check_migrate_databases"] = False
galaxy_app = galaxy.app.GalaxyManagerApplication(configure_logging=False, **kwargs)
return galaxy_app
@@ -35,16 +39,16 @@ def get_galaxy_app():
@lru_cache(maxsize=1)
def get_app_properties():
config_file = os.environ.get("GALAXY_CONFIG_FILE")
- galaxy_root_dir = os.environ.get('GALAXY_ROOT_DIR')
+ galaxy_root_dir = os.environ.get("GALAXY_ROOT_DIR")
if not config_file and galaxy_root_dir:
config_file = find_config(config_file, galaxy_root_dir)
if config_file:
properties = load_app_properties(
config_file=os.path.abspath(config_file),
- config_section='galaxy',
+ config_section="galaxy",
)
if galaxy_root_dir:
- properties['root_dir'] = galaxy_root_dir
+ properties["root_dir"] = galaxy_root_dir
return properties
@@ -52,7 +56,7 @@ def get_app_properties():
def get_config():
kwargs = get_app_properties()
if kwargs:
- kwargs['override_tempdir'] = False
+ kwargs["override_tempdir"] = False
return Configuration(**kwargs)
@@ -71,33 +75,30 @@ def get_history_audit_table_prune_interval():
broker = get_broker()
-celery_app = Celery('galaxy', broker=broker, include=['galaxy.celery.tasks'])
+celery_app = Celery("galaxy", broker=broker, include=["galaxy.celery.tasks"])
prune_interval = get_history_audit_table_prune_interval()
if prune_interval > 0:
celery_app.conf.beat_schedule = {
- 'prune-history-audit-table': {
- 'task': 'galaxy.celery.tasks.prune_history_audit_table',
- 'schedule': prune_interval,
+ "prune-history-audit-table": {
+ "task": "galaxy.celery.tasks.prune_history_audit_table",
+ "schedule": prune_interval,
},
}
-celery_app.conf.timezone = 'UTC'
+celery_app.conf.timezone = "UTC"
CELERY_TASKS = []
-PYDANTIC_AWARE_SERIALIER_NAME = 'pydantic-aware-json'
+PYDANTIC_AWARE_SERIALIER_NAME = "pydantic-aware-json"
serialization.register(
- PYDANTIC_AWARE_SERIALIER_NAME,
- encoder=schema_dumps,
- decoder=schema_loads,
- content_type='application/json'
+ PYDANTIC_AWARE_SERIALIER_NAME, encoder=schema_dumps, decoder=schema_loads, content_type="application/json"
)
def galaxy_task(*args, **celery_task_kwd):
- if 'serializer' not in celery_task_kwd:
- celery_task_kwd['serializer'] = PYDANTIC_AWARE_SERIALIER_NAME
+ if "serializer" not in celery_task_kwd:
+ celery_task_kwd["serializer"] = PYDANTIC_AWARE_SERIALIER_NAME
def decorate(func):
CELERY_TASKS.append(func.__name__)
@@ -117,5 +118,5 @@ def galaxy_task(*args, **celery_task_kwd):
return decorate
-if __name__ == '__main__':
+if __name__ == "__main__":
celery_app.start()
diff --git a/lib/galaxy/celery/_serialization.py b/lib/galaxy/celery/_serialization.py
index a7a6b3a4670..8accdb632d3 100644
--- a/lib/galaxy/celery/_serialization.py
+++ b/lib/galaxy/celery/_serialization.py
@@ -10,32 +10,34 @@ SCHEMA_MODELS_PACKAGE_BASE = "galaxy.schema."
def fullname(o):
klass = o.__class__
module = klass.__module__
- if module == 'builtins':
+ if module == "builtins":
return klass.__qualname__ # avoid outputs like 'builtins.str'
- return module + '.' + klass.__qualname__
+ return module + "." + klass.__qualname__
class SchemaEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, BaseModel):
return {
- '__type__': '__pydantic_object__',
- '__class__': fullname(obj),
- '__object__': obj.dict(),
+ "__type__": "__pydantic_object__",
+ "__class__": fullname(obj),
+ "__object__": obj.dict(),
}
else:
return json.JSONEncoder.default(self, obj)
def schema_decoder(obj):
- if '__type__' in obj:
- if obj['__type__'] == '__pydantic_object__':
- clazz_str = obj['__class__']
- assert clazz_str.startswith(SCHEMA_MODELS_PACKAGE_BASE) and ".." not in clazz_str, f"Invalid class str {clazz_str}"
- module_name, class_name = clazz_str.rsplit('.', 1)
+ if "__type__" in obj:
+ if obj["__type__"] == "__pydantic_object__":
+ clazz_str = obj["__class__"]
+ assert (
+ clazz_str.startswith(SCHEMA_MODELS_PACKAGE_BASE) and ".." not in clazz_str
+ ), f"Invalid class str {clazz_str}"
+ module_name, class_name = clazz_str.rsplit(".", 1)
module = import_module(module_name)
clazz = getattr(module, class_name, None)
- obj = clazz(**obj['__object__'])
+ obj = clazz(**obj["__object__"])
return obj
return obj
diff --git a/lib/galaxy/celery/tasks.py b/lib/galaxy/celery/tasks.py
index 8b1d4a97faa..8914cfaf193 100644
--- a/lib/galaxy/celery/tasks.py
+++ b/lib/galaxy/celery/tasks.py
@@ -31,24 +31,27 @@ def purge_hda(hda_manager: HDAManager, hda_id):
@galaxy_task
-def set_metadata(hda_manager: HDAManager, ldda_manager: LDDAManager, dataset_id, model_class='HistoryDatasetAssociation'):
- if model_class == 'HistoryDatasetAssociation':
+def set_metadata(
+ hda_manager: HDAManager, ldda_manager: LDDAManager, dataset_id, model_class="HistoryDatasetAssociation"
+):
+ if model_class == "HistoryDatasetAssociation":
dataset = hda_manager.by_id(dataset_id)
- elif model_class == 'LibraryDatasetDatasetAssociation':
+ elif model_class == "LibraryDatasetDatasetAssociation":
dataset = ldda_manager.by_id(dataset_id)
dataset.datatype.set_meta(dataset)
@galaxy_task(ignore_result=True)
def export_history(
- app: MinimalManagerApp,
- sa_session: galaxy_scoped_session,
- job_manager: JobManager,
- store_directory: str,
- history_id: int,
- job_id: int,
- include_hidden=False,
- include_deleted=False):
+ app: MinimalManagerApp,
+ sa_session: galaxy_scoped_session,
+ job_manager: JobManager,
+ store_directory: str,
+ history_id: int,
+ job_id: int,
+ include_hidden=False,
+ include_deleted=False,
+):
history = sa_session.query(model.History).get(history_id)
with model.store.DirectoryModelExportStore(store_directory, app=app, export_files="symlink") as export_store:
export_store.export_history(history, include_hidden=include_hidden, include_deleted=include_deleted)
diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py
index 0d76a4e39f6..15ac7dd8d16 100644
--- a/lib/galaxy/config/__init__.py
+++ b/lib/galaxy/config/__init__.py
@@ -61,7 +61,10 @@ from galaxy.util.properties import (
)
from galaxy.web.formatting import expand_pretty_datetime_format
from galaxy.web_stack import get_stack_facts
-from ..version import VERSION_MAJOR, VERSION_MINOR
+from ..version import (
+ VERSION_MAJOR,
+ VERSION_MINOR,
+)
try:
from importlib.resources import files # type: ignore[attr-defined]
@@ -78,60 +81,60 @@ if TYPE_CHECKING:
log = logging.getLogger(__name__)
-GALAXY_APP_NAME = 'galaxy'
-GALAXY_SCHEMAS_PATH = files('galaxy.config') / 'schemas'
-GALAXY_CONFIG_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / 'config_schema.yml'
-UWSGI_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / 'uwsgi_schema.yml'
+GALAXY_APP_NAME = "galaxy"
+GALAXY_SCHEMAS_PATH = files("galaxy.config") / "schemas"
+GALAXY_CONFIG_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / "config_schema.yml"
+UWSGI_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / "uwsgi_schema.yml"
LOGGING_CONFIG_DEFAULT: Dict[str, Any] = {
- 'disable_existing_loggers': False,
- 'version': 1,
- 'root': {
- 'handlers': ['console'],
- 'level': 'DEBUG',
+ "disable_existing_loggers": False,
+ "version": 1,
+ "root": {
+ "handlers": ["console"],
+ "level": "DEBUG",
},
- 'loggers': {
- 'paste.httpserver.ThreadPool': {
- 'level': 'WARN',
- 'qualname': 'paste.httpserver.ThreadPool',
+ "loggers": {
+ "paste.httpserver.ThreadPool": {
+ "level": "WARN",
+ "qualname": "paste.httpserver.ThreadPool",
},
- 'sqlalchemy_json.track': {
- 'level': 'WARN',
- 'qualname': 'sqlalchemy_json.track',
+ "sqlalchemy_json.track": {
+ "level": "WARN",
+ "qualname": "sqlalchemy_json.track",
},
- 'urllib3.connectionpool': {
- 'level': 'WARN',
- 'qualname': 'urllib3.connectionpool',
+ "urllib3.connectionpool": {
+ "level": "WARN",
+ "qualname": "urllib3.connectionpool",
},
- 'routes.middleware': {
- 'level': 'WARN',
- 'qualname': 'routes.middleware',
+ "routes.middleware": {
+ "level": "WARN",
+ "qualname": "routes.middleware",
},
- 'amqp': {
- 'level': 'INFO',
- 'qualname': 'amqp',
+ "amqp": {
+ "level": "INFO",
+ "qualname": "amqp",
},
- 'botocore': {
- 'level': 'INFO',
- 'qualname': 'botocore',
+ "botocore": {
+ "level": "INFO",
+ "qualname": "botocore",
},
},
- 'filters': {
- 'stack': {
- '()': 'galaxy.web_stack.application_stack_log_filter',
+ "filters": {
+ "stack": {
+ "()": "galaxy.web_stack.application_stack_log_filter",
},
},
- 'handlers': {
- 'console': {
- 'class': 'logging.StreamHandler',
- 'formatter': 'stack',
- 'level': 'DEBUG',
- 'stream': 'ext://sys.stderr',
- 'filters': ['stack'],
+ "handlers": {
+ "console": {
+ "class": "logging.StreamHandler",
+ "formatter": "stack",
+ "level": "DEBUG",
+ "stream": "ext://sys.stderr",
+ "filters": ["stack"],
},
},
- 'formatters': {
- 'stack': {
- '()': 'galaxy.web_stack.application_stack_log_formatter',
+ "formatters": {
+ "stack": {
+ "()": "galaxy.web_stack.application_stack_log_formatter",
},
},
}
@@ -139,7 +142,7 @@ LOGGING_CONFIG_DEFAULT: Dict[str, Any] = {
def find_root(kwargs):
- return os.path.abspath(kwargs.get('root_dir', '.'))
+ return os.path.abspath(kwargs.get("root_dir", "."))
OptStr = TypeVar("OptStr", None, str)
@@ -150,7 +153,9 @@ class BaseAppConfiguration(HasDynamicProperties):
# If VALUE == first directory in a user-supplied path that resolves to KEY, it will be stripped from that path
renamed_options: Optional[Dict[str, str]] = None
deprecated_dirs: Dict[str, str] = {}
- paths_to_check_against_root: Set[str] = set() # backward compatibility: if resolved path doesn't exist, try resolving w.r.t root
+ paths_to_check_against_root: Set[
+ str
+ ] = set() # backward compatibility: if resolved path doesn't exist, try resolving w.r.t root
add_sample_file_to_defaults: Set[str] = set() # for these options, add sample config files to their defaults
listify_options: Set[str] = set() # values for these options are processed as lists of values
object_store_store_by: str
@@ -188,19 +193,21 @@ class BaseAppConfiguration(HasDynamicProperties):
Fix deprecated database URLs (postgres... >> postgresql...)
https://docs.sqlalchemy.org/en/14/changelog/changelog_14.html#change-3687655465c25a39b968b4f5f6e9170b
"""
- old_dialect, new_dialect = 'postgres', 'postgresql'
- old_prefixes = (f'{old_dialect}:', f'{old_dialect}+') # check for postgres://foo and postgres+driver//foo
+ old_dialect, new_dialect = "postgres", "postgresql"
+ old_prefixes = (f"{old_dialect}:", f"{old_dialect}+") # check for postgres://foo and postgres+driver//foo
offset = len(old_dialect)
- keys = ('database_connection', 'install_database_connection')
+ keys = ("database_connection", "install_database_connection")
for key in keys:
if key in kwargs:
value = kwargs[key]
for prefix in old_prefixes:
if value.startswith(prefix):
- value = f'{new_dialect}{value[offset:]}'
+ value = f"{new_dialect}{value[offset:]}"
kwargs[key] = value
- log.warning('PostgreSQL database URLs of the form "postgres://" have been '
- 'deprecated. Please use "postgresql://".')
+ log.warning(
+ 'PostgreSQL database URLs of the form "postgres://" have been '
+ 'deprecated. Please use "postgresql://".'
+ )
def is_set(self, key):
"""Check if a configuration option has been explicitly set."""
@@ -215,13 +222,12 @@ class BaseAppConfiguration(HasDynamicProperties):
return self._in_root_dir(path)
def _set_config_base(self, config_kwargs):
-
def _set_global_conf():
- self.config_file = find_config_file('galaxy')
- self.global_conf = config_kwargs.get('global_conf')
+ self.config_file = find_config_file("galaxy")
+ self.global_conf = config_kwargs.get("global_conf")
self.global_conf_parser = configparser.ConfigParser()
if not self.config_file and self.global_conf and "__file__" in self.global_conf:
- self.config_file = os.path.join(self.root, self.global_conf['__file__'])
+ self.config_file = os.path.join(self.root, self.global_conf["__file__"])
if self.config_file is None:
log.warning("No Galaxy config file found, running from current working directory: %s", os.getcwd())
@@ -236,40 +242,40 @@ class BaseAppConfiguration(HasDynamicProperties):
def _set_config_directories():
# Set config_dir to value from kwargs OR dirname of config_file OR None
_config_dir = os.path.dirname(self.config_file) if self.config_file else None
- self.config_dir = config_kwargs.get('config_dir', _config_dir)
+ self.config_dir = config_kwargs.get("config_dir", _config_dir)
# Make path absolute before using it as base for other paths
if self.config_dir:
self.config_dir = os.path.abspath(self.config_dir)
- self.data_dir = config_kwargs.get('data_dir')
+ self.data_dir = config_kwargs.get("data_dir")
if self.data_dir:
self.data_dir = os.path.abspath(self.data_dir)
- self.sample_config_dir = os.path.join(os.path.dirname(__file__), 'sample')
+ self.sample_config_dir = os.path.join(os.path.dirname(__file__), "sample")
if self.sample_config_dir:
self.sample_config_dir = os.path.abspath(self.sample_config_dir)
- self.managed_config_dir = config_kwargs.get('managed_config_dir')
+ self.managed_config_dir = config_kwargs.get("managed_config_dir")
if self.managed_config_dir:
self.managed_config_dir = os.path.abspath(self.managed_config_dir)
if running_from_source:
if not self.config_dir:
- self.config_dir = os.path.join(self.root, 'config')
+ self.config_dir = os.path.join(self.root, "config")
if not self.data_dir:
- self.data_dir = os.path.join(self.root, 'database')
+ self.data_dir = os.path.join(self.root, "database")
if not self.managed_config_dir:
self.managed_config_dir = self.config_dir
else:
if not self.config_dir:
self.config_dir = os.getcwd()
if not self.data_dir:
- self.data_dir = self._in_config_dir('data')
+ self.data_dir = self._in_config_dir("data")
if not self.managed_config_dir:
- self.managed_config_dir = self._in_data_dir('config')
+ self.managed_config_dir = self._in_data_dir("config")
# TODO: do we still need to support ../shed_tools when running_from_source?
- self.shed_tools_dir = self._in_data_dir('shed_tools')
+ self.shed_tools_dir = self._in_data_dir("shed_tools")
log.debug("Configuration directory is %s", self.config_dir)
log.debug("Data directory is %s", self.data_dir)
@@ -280,7 +286,7 @@ class BaseAppConfiguration(HasDynamicProperties):
def _load_schema(self):
# Override in subclasses
- raise Exception('Not implemented')
+ raise Exception("Not implemented")
def _preprocess_paths_to_resolve(self):
# For these options, if option is not set, listify its defaults and add a sample config file.
@@ -288,17 +294,20 @@ class BaseAppConfiguration(HasDynamicProperties):
for key in self.add_sample_file_to_defaults:
if not self.is_set(key):
defaults = listify(getattr(self, key), do_strip=True)
- sample = f'{defaults[-1]}.sample' # if there are multiple defaults, use last as template
+ sample = f"{defaults[-1]}.sample" # if there are multiple defaults, use last as template
sample = self._in_sample_dir(sample) # resolve w.r.t sample_dir
defaults.append(sample)
setattr(self, key, defaults)
def _postprocess_paths_to_resolve(self):
-
def select_one_path_from_list():
# To consider: options with a sample file added to defaults except options that can have multiple values.
# If value is not set, check each path in list; set to first path that exists; if none exist, set to last path in list.
- keys = self.add_sample_file_to_defaults - self.listify_options if self.listify_options else self.add_sample_file_to_defaults
+ keys = (
+ self.add_sample_file_to_defaults - self.listify_options
+ if self.listify_options
+ else self.add_sample_file_to_defaults
+ )
for key in keys:
if not self.is_set(key):
paths = getattr(self, key)
@@ -307,7 +316,9 @@ class BaseAppConfiguration(HasDynamicProperties):
setattr(self, key, path)
break
else:
- setattr(self, key, paths[-1]) # TODO: we assume it exists; but we've already checked in the loop! Raise error instead?
+ setattr(
+ self, key, paths[-1]
+ ) # TODO: we assume it exists; but we've already checked in the loop! Raise error instead?
def select_one_or_all_paths_from_list():
# Values for these options are lists of paths. If value is not set, use defaults if all paths in list exist;
@@ -320,7 +331,9 @@ class BaseAppConfiguration(HasDynamicProperties):
setattr(self, key, [paths[-1]]) # value is a list
break
- if self.add_sample_file_to_defaults: # Currently, this is the ONLY case when we need to pick one file from a list
+ if (
+ self.add_sample_file_to_defaults
+ ): # Currently, this is the ONLY case when we need to pick one file from a list
select_one_path_from_list()
if self.listify_options:
select_one_or_all_paths_from_list()
@@ -346,7 +359,7 @@ class BaseAppConfiguration(HasDynamicProperties):
}
def convert_datatype(key, value):
- datatype = self.schema.app_schema[key].get('type')
+ datatype = self.schema.app_schema[key].get("type")
# check for `not None` explicitly (value can be falsy)
if value is not None and datatype in type_converters:
# convert value or each item in value to type `datatype`
@@ -367,14 +380,18 @@ class BaseAppConfiguration(HasDynamicProperties):
ignore = first_dir + os.sep
log.warning(
"Paths for the '%s' option are now relative to '%s', remove the leading '%s' "
- "to suppress this warning: %s", key, resolves_to, ignore, path
+ "to suppress this warning: %s",
+ key,
+ resolves_to,
+ ignore,
+ path,
)
- paths[i] = path[len(ignore):]
+ paths[i] = path[len(ignore) :]
# return list or string, depending on type of `value`
if isinstance(value, list):
return paths
- return ','.join(paths)
+ return ",".join(paths)
return value
for key, value in kwargs.items():
@@ -387,7 +404,7 @@ class BaseAppConfiguration(HasDynamicProperties):
def _create_attributes_from_raw_config(self):
# `base_configs` are a special case: these attributes have been created and will be ignored
# by the code below. Trying to overwrite any other existing attributes will raise an error.
- base_configs = {'config_dir', 'data_dir', 'managed_config_dir'}
+ base_configs = {"config_dir", "data_dir", "managed_config_dir"}
for key, value in self._raw_config.items():
if not hasattr(self, key):
setattr(self, key, value)
@@ -395,7 +412,6 @@ class BaseAppConfiguration(HasDynamicProperties):
raise ConfigurationError(f"Attempting to override existing attribute '{key}'")
def _resolve_paths(self):
-
def resolve(key):
if key in _cache: # resolve each path only once
return _cache[key]
@@ -420,7 +436,7 @@ class BaseAppConfiguration(HasDynamicProperties):
# Check if value is a list or should be listified; if so, listify and resolve each item separately.
if type(value) is list or (self.listify_options and key in self.listify_options):
saved_values = listify(getattr(self, key), do_strip=True) # listify and save original value
- setattr(self, key, '_') # replace value with temporary placeholder
+ setattr(self, key, "_") # replace value with temporary placeholder
resolve(key) # resolve temporary value (`_` becomes `parent-path/_`)
resolved_base = getattr(self, key)[:-1] # get rid of placeholder in resolved path
# apply resolved base to saved values
@@ -433,7 +449,6 @@ class BaseAppConfiguration(HasDynamicProperties):
self._check_against_root(key)
def _check_against_root(self, key):
-
def get_path(current_path, initial_path):
# if path does not exist and was set as relative:
if not self._path_exists(current_path) and not os.path.isabs(initial_path):
@@ -531,64 +546,71 @@ class CommonConfigurationMixin:
class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
- deprecated_options = ('database_file', 'track_jobs_in_database', 'blacklist_file', 'whitelist_file',
- 'sanitize_whitelist_file', 'user_library_import_symlink_whitelist', 'fetch_url_whitelist',
- 'containers_resolvers_config_file')
+ deprecated_options = (
+ "database_file",
+ "track_jobs_in_database",
+ "blacklist_file",
+ "whitelist_file",
+ "sanitize_whitelist_file",
+ "user_library_import_symlink_whitelist",
+ "fetch_url_whitelist",
+ "containers_resolvers_config_file",
+ )
renamed_options = {
- 'blacklist_file': 'email_domain_blocklist_file',
- 'whitelist_file': 'email_domain_allowlist_file',
- 'sanitize_whitelist_file': 'sanitize_allowlist_file',
- 'user_library_import_symlink_whitelist': 'user_library_import_symlink_allowlist',
- 'fetch_url_whitelist': 'fetch_url_allowlist',
- 'containers_resolvers_config_file': 'container_resolvers_config_file',
+ "blacklist_file": "email_domain_blocklist_file",
+ "whitelist_file": "email_domain_allowlist_file",
+ "sanitize_whitelist_file": "sanitize_allowlist_file",
+ "user_library_import_symlink_whitelist": "user_library_import_symlink_allowlist",
+ "fetch_url_whitelist": "fetch_url_allowlist",
+ "containers_resolvers_config_file": "container_resolvers_config_file",
}
- default_config_file_name = 'galaxy.yml'
- deprecated_dirs = {'config_dir': 'config', 'data_dir': 'database'}
+ default_config_file_name = "galaxy.yml"
+ deprecated_dirs = {"config_dir": "config", "data_dir": "database"}
paths_to_check_against_root = {
- 'auth_config_file',
- 'build_sites_config_file',
- 'containers_config_file',
- 'data_manager_config_file',
- 'datatypes_config_file',
- 'dependency_resolvers_config_file',
- 'error_report_file',
- 'job_config_file',
- 'job_metrics_config_file',
- 'job_resource_params_file',
- 'local_conda_mapping_file',
- 'migrated_tools_config',
- 'modules_mapping_files',
- 'object_store_config_file',
- 'oidc_backends_config_file',
- 'oidc_config_file',
- 'shed_data_manager_config_file',
- 'shed_tool_config_file',
- 'shed_tool_data_table_config',
- 'tool_destinations_config_file',
- 'tool_sheds_config_file',
- 'user_preferences_extra_conf_path',
- 'workflow_resource_params_file',
- 'workflow_schedulers_config_file',
- 'markdown_export_css',
- 'markdown_export_css_pages',
- 'markdown_export_css_invocation_reports',
- 'file_path',
- 'tool_data_table_config_path',
- 'tool_config_file',
+ "auth_config_file",
+ "build_sites_config_file",
+ "containers_config_file",
+ "data_manager_config_file",
+ "datatypes_config_file",
+ "dependency_resolvers_config_file",
+ "error_report_file",
+ "job_config_file",
+ "job_metrics_config_file",
+ "job_resource_params_file",
+ "local_conda_mapping_file",
+ "migrated_tools_config",
+ "modules_mapping_files",
+ "object_store_config_file",
+ "oidc_backends_config_file",
+ "oidc_config_file",
+ "shed_data_manager_config_file",
+ "shed_tool_config_file",
+ "shed_tool_data_table_config",
+ "tool_destinations_config_file",
+ "tool_sheds_config_file",
+ "user_preferences_extra_conf_path",
+ "workflow_resource_params_file",
+ "workflow_schedulers_config_file",
+ "markdown_export_css",
+ "markdown_export_css_pages",
+ "markdown_export_css_invocation_reports",
+ "file_path",
+ "tool_data_table_config_path",
+ "tool_config_file",
}
add_sample_file_to_defaults = {
- 'build_sites_config_file',
- 'datatypes_config_file',
- 'job_metrics_config_file',
- 'tool_data_table_config_path',
- 'tool_config_file',
+ "build_sites_config_file",
+ "datatypes_config_file",
+ "job_metrics_config_file",
+ "tool_data_table_config_path",
+ "tool_config_file",
}
listify_options = {
- 'tool_data_table_config_path',
- 'tool_config_file',
+ "tool_data_table_config_path",
+ "tool_config_file",
}
database_connection: str
tool_path: str
@@ -637,17 +659,20 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
This method should be deleted after migration to SQLAlchemy 2.0 is complete.
To enable warnings, set `GALAXY_CONFIG_SQLALCHEMY_WARN_20=1`,
"""
- warn = string_as_bool(kwargs.get('sqlalchemy_warn_20', False))
+ warn = string_as_bool(kwargs.get("sqlalchemy_warn_20", False))
if warn:
import sqlalchemy
+
sqlalchemy.util.deprecations.SQLALCHEMY_WARN_20 = True
self._setup_sqlalchemy20_warnings_filters()
def _setup_sqlalchemy20_warnings_filters(self):
import warnings
+
from sqlalchemy.exc import RemovedIn20Warning
+
# Always display RemovedIn20Warning warnings.
- warnings.filterwarnings('always', category=RemovedIn20Warning)
+ warnings.filterwarnings("always", category=RemovedIn20Warning)
# Optionally, enable filters for specific warnings (raise error, or log, etc.)
# messages = [
# r"replace with warning text to match",
@@ -693,13 +718,13 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
self.version_minor = VERSION_MINOR
# Database related configuration
- self.check_migrate_databases = kwargs.get('check_migrate_databases', True)
+ self.check_migrate_databases = kwargs.get("check_migrate_databases", True)
if not self.database_connection: # Provide default if not supplied by user
- db_path = self._in_data_dir('universe.sqlite')
- self.database_connection = f'sqlite:///{db_path}?isolation_level=IMMEDIATE'
+ db_path = self._in_data_dir("universe.sqlite")
+ self.database_connection = f"sqlite:///{db_path}?isolation_level=IMMEDIATE"
self.database_engine_options = get_database_engine_options(kwargs)
- self.database_create_tables = string_as_bool(kwargs.get('database_create_tables', 'True'))
- self.database_encoding = kwargs.get('database_encoding') # Create new databases with this encoding
+ self.database_create_tables = string_as_bool(kwargs.get("database_create_tables", "True"))
+ self.database_encoding = kwargs.get("database_encoding") # Create new databases with this encoding
self.thread_local_log = None
if self.enable_per_request_sql_debugging:
self.thread_local_log = threading.local()
@@ -710,12 +735,12 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
self.tool_path = self._in_root_dir(self.tool_path)
self.tool_data_path = self._in_root_dir(self.tool_data_path)
if not running_from_source and kwargs.get("tool_data_path") is None:
- self.tool_data_path = self._in_data_dir(self.schema.defaults['tool_data_path'])
+ self.tool_data_path = self._in_data_dir(self.schema.defaults["tool_data_path"])
self.builds_file_path = os.path.join(self.tool_data_path, self.builds_file_path)
self.len_file_path = os.path.join(self.tool_data_path, self.len_file_path)
self.oidc = {}
self.integrated_tool_panel_config = self._in_managed_config_dir(self.integrated_tool_panel_config)
- integrated_tool_panel_tracking_directory = kwargs.get('integrated_tool_panel_tracking_directory')
+ integrated_tool_panel_tracking_directory = kwargs.get("integrated_tool_panel_tracking_directory")
if integrated_tool_panel_tracking_directory:
self.integrated_tool_panel_tracking_directory = self._in_root_dir(integrated_tool_panel_tracking_directory)
else:
@@ -728,18 +753,18 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
self.user_tool_filters = listify(self.user_tool_filters, do_strip=True)
self.user_tool_label_filters = listify(self.user_tool_label_filters, do_strip=True)
self.user_tool_section_filters = listify(self.user_tool_section_filters, do_strip=True)
- self.has_user_tool_filters = bool(self.user_tool_filters or self.user_tool_label_filters or self.user_tool_section_filters)
-
- self.password_expiration_period = timedelta(
- days=int(cast(SupportsInt, self.password_expiration_period))
+ self.has_user_tool_filters = bool(
+ self.user_tool_filters or self.user_tool_label_filters or self.user_tool_section_filters
)
+ self.password_expiration_period = timedelta(days=int(cast(SupportsInt, self.password_expiration_period)))
+
if self.shed_tool_data_path:
self.shed_tool_data_path = self._in_root_dir(self.shed_tool_data_path)
else:
self.shed_tool_data_path = self.tool_data_path
- self.running_functional_tests = string_as_bool(kwargs.get('running_functional_tests', False))
+ self.running_functional_tests = string_as_bool(kwargs.get("running_functional_tests", False))
if isinstance(self.hours_between_check, str):
self.hours_between_check = float(self.hours_between_check)
try:
@@ -765,9 +790,9 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
self.use_remote_user = self.use_remote_user or self.single_user
self.fetch_url_allowlist_ips = [
ipaddress.ip_network(unicodify(ip.strip())) # If it has a slash, assume 127.0.0.1/24 notation
- if '/' in ip else
- ipaddress.ip_address(unicodify(ip.strip())) # Otherwise interpret it as an ip address.
- for ip in kwargs.get("fetch_url_allowlist", "").split(',')
+ if "/" in ip
+ else ipaddress.ip_address(unicodify(ip.strip())) # Otherwise interpret it as an ip address.
+ for ip in kwargs.get("fetch_url_allowlist", "").split(",")
if len(ip.strip()) > 0
]
self.job_queue_cleanup_interval = int(kwargs.get("job_queue_cleanup_interval", "5"))
@@ -780,36 +805,47 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
self.preserve_python_environment = "legacy_only"
self.nodejs_path = kwargs.get("nodejs_path")
self.container_image_cache_path = self._in_data_dir(kwargs.get("container_image_cache_path", "container_cache"))
- self.output_size_limit = int(kwargs.get('output_size_limit', 0))
+ self.output_size_limit = int(kwargs.get("output_size_limit", 0))
# activation_email was used until release_15.03
- activation_email = kwargs.get('activation_email')
+ activation_email = kwargs.get("activation_email")
self.email_from = self.email_from or activation_email
- self.email_domain_blocklist_content = self._load_list_from_file(self._in_config_dir(self.email_domain_blocklist_file)) if self.email_domain_blocklist_file else None
- self.email_domain_allowlist_content = self._load_list_from_file(self._in_config_dir(self.email_domain_allowlist_file)) if self.email_domain_allowlist_file else None
+ self.email_domain_blocklist_content = (
+ self._load_list_from_file(self._in_config_dir(self.email_domain_blocklist_file))
+ if self.email_domain_blocklist_file
+ else None
+ )
+ self.email_domain_allowlist_content = (
+ self._load_list_from_file(self._in_config_dir(self.email_domain_allowlist_file))
+ if self.email_domain_allowlist_file
+ else None
+ )
# These are not even beta - just experiments - don't use them unless
# you want yours tools to be broken in the future.
- self.enable_beta_tool_formats = string_as_bool(kwargs.get('enable_beta_tool_formats', 'False'))
+ self.enable_beta_tool_formats = string_as_bool(kwargs.get("enable_beta_tool_formats", "False"))
- if self.workflow_resource_params_mapper and ':' not in self.workflow_resource_params_mapper:
+ if self.workflow_resource_params_mapper and ":" not in self.workflow_resource_params_mapper:
# Assume it is not a Python function, so a file; else: a Python function
self.workflow_resource_params_mapper = self._in_root_dir(self.workflow_resource_params_mapper)
- self.pbs_application_server = kwargs.get('pbs_application_server', "")
- self.pbs_dataset_server = kwargs.get('pbs_dataset_server', "")
- self.pbs_dataset_path = kwargs.get('pbs_dataset_path', "")
- self.pbs_stage_path = kwargs.get('pbs_stage_path', "")
+ self.pbs_application_server = kwargs.get("pbs_application_server", "")
+ self.pbs_dataset_server = kwargs.get("pbs_dataset_server", "")
+ self.pbs_dataset_path = kwargs.get("pbs_dataset_path", "")
+ self.pbs_stage_path = kwargs.get("pbs_stage_path", "")
_sanitize_allowlist_path = self._in_managed_config_dir(self.sanitize_allowlist_file)
if not os.path.isfile(_sanitize_allowlist_path): # then check old default location
for deprecated in (
- self._in_managed_config_dir('sanitize_whitelist.txt'),
- self._in_root_dir('config/sanitize_whitelist.txt')):
+ self._in_managed_config_dir("sanitize_whitelist.txt"),
+ self._in_root_dir("config/sanitize_whitelist.txt"),
+ ):
if os.path.isfile(deprecated):
- log.warning("The path '%s' for the 'sanitize_allowlist_file' config option is "
+ log.warning(
+ "The path '%s' for the 'sanitize_allowlist_file' config option is "
"deprecated and will be no longer checked in a future release. Please consult "
- "the latest version of the sample configuration file." % deprecated)
+ "the latest version of the sample configuration file." % deprecated
+ )
_sanitize_allowlist_path = deprecated
break
self.sanitize_allowlist_file = _sanitize_allowlist_path
@@ -818,18 +854,24 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
if "trust_jupyter_notebook_conversion" not in kwargs:
# if option not set, check IPython-named alternative, falling back to schema default if not set either
_default = self.trust_jupyter_notebook_conversion
- self.trust_jupyter_notebook_conversion = string_as_bool(kwargs.get('trust_ipython_notebook_conversion', _default))
+ self.trust_jupyter_notebook_conversion = string_as_bool(
+ kwargs.get("trust_ipython_notebook_conversion", _default)
+ )
# Configuration for the message box directly below the masthead.
- self.blog_url = kwargs.get('blog_url')
+ self.blog_url = kwargs.get("blog_url")
self.user_library_import_symlink_allowlist = listify(self.user_library_import_symlink_allowlist, do_strip=True)
- self.user_library_import_dir_auto_creation = self.user_library_import_dir_auto_creation if self.user_library_import_dir else False
+ self.user_library_import_dir_auto_creation = (
+ self.user_library_import_dir_auto_creation if self.user_library_import_dir else False
+ )
# Searching data libraries
- self.ftp_upload_dir_template = kwargs.get('ftp_upload_dir_template', '${ftp_upload_dir}%s${ftp_upload_dir_identifier}' % os.path.sep)
+ self.ftp_upload_dir_template = kwargs.get(
+ "ftp_upload_dir_template", "${ftp_upload_dir}%s${ftp_upload_dir_identifier}" % os.path.sep
+ )
# Support older library-specific path paste option but just default to the new
# allow_path_paste value.
- self.allow_library_path_paste = string_as_bool(kwargs.get('allow_library_path_paste', self.allow_path_paste))
- self.disable_library_comptypes = kwargs.get('disable_library_comptypes', '').lower().split(',')
- self.check_upload_content = string_as_bool(kwargs.get('check_upload_content', True))
+ self.allow_library_path_paste = string_as_bool(kwargs.get("allow_library_path_paste", self.allow_path_paste))
+ self.disable_library_comptypes = kwargs.get("disable_library_comptypes", "").lower().split(",")
+ self.check_upload_content = string_as_bool(kwargs.get("check_upload_content", True))
# On can mildly speed up Galaxy startup time by disabling index of help,
# not needed on production systems but useful if running many functional tests.
self.index_tool_help = string_as_bool(kwargs.get("index_tool_help", True))
@@ -839,7 +881,9 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
# Deployers may either specify a complete list of mapping files or get the default for free and just
# specify a local mapping file to adapt and extend the default one.
if "conda_mapping_files" not in kwargs:
- _default_mapping = self._in_root_dir(os.path.join("lib", "galaxy", "tool_util", "deps", "resolvers", "default_conda_mapping.yml"))
+ _default_mapping = self._in_root_dir(
+ os.path.join("lib", "galaxy", "tool_util", "deps", "resolvers", "default_conda_mapping.yml")
+ )
# dependency resolution options are consumed via config_dict - so don't populate
# self, populate config_dict
self.config_dict["conda_mapping_files"] = [self.local_conda_mapping_file, _default_mapping]
@@ -848,16 +892,16 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
self.container_resolvers_config_file = self._in_config_dir(self.container_resolvers_config_file)
# tool_dependency_dir can be "none" (in old configs). If so, set it to None
- if self.tool_dependency_dir and self.tool_dependency_dir.lower() == 'none':
+ if self.tool_dependency_dir and self.tool_dependency_dir.lower() == "none":
self.tool_dependency_dir = None
if self.involucro_path is None:
- target_dir = self.tool_dependency_dir or self.schema.defaults['tool_dependency_dir']
+ target_dir = self.tool_dependency_dir or self.schema.defaults["tool_dependency_dir"]
self.involucro_path = self._in_data_dir(os.path.join(target_dir, "involucro"))
self.involucro_path = self._in_root_dir(self.involucro_path)
if self.mulled_channels:
self.mulled_channels = [c.strip() for c in self.mulled_channels.split(",")] # type: ignore[attr-defined]
- default_job_resubmission_condition = kwargs.get('default_job_resubmission_condition', '')
+ default_job_resubmission_condition = kwargs.get("default_job_resubmission_condition", "")
if not default_job_resubmission_condition.strip():
default_job_resubmission_condition = None
self.default_job_resubmission_condition = default_job_resubmission_condition
@@ -866,92 +910,100 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
if self.nginx_upload_store:
self.nginx_upload_store = os.path.abspath(self.nginx_upload_store)
- self.object_store = kwargs.get('object_store', 'disk')
- self.object_store_check_old_style = string_as_bool(kwargs.get('object_store_check_old_style', False))
- self.object_store_cache_path = self._in_root_dir(kwargs.get("object_store_cache_path", self._in_data_dir("object_store_cache")))
+ self.object_store = kwargs.get("object_store", "disk")
+ self.object_store_check_old_style = string_as_bool(kwargs.get("object_store_check_old_style", False))
+ self.object_store_cache_path = self._in_root_dir(
+ kwargs.get("object_store_cache_path", self._in_data_dir("object_store_cache"))
+ )
self._configure_dataset_storage()
# Handle AWS-specific config options for backward compatibility
- if kwargs.get('aws_access_key') is not None:
- self.os_access_key = kwargs.get('aws_access_key')
- self.os_secret_key = kwargs.get('aws_secret_key')
- self.os_bucket_name = kwargs.get('s3_bucket')
- self.os_use_reduced_redundancy = kwargs.get('use_reduced_redundancy', False)
+ if kwargs.get("aws_access_key") is not None:
+ self.os_access_key = kwargs.get("aws_access_key")
+ self.os_secret_key = kwargs.get("aws_secret_key")
+ self.os_bucket_name = kwargs.get("s3_bucket")
+ self.os_use_reduced_redundancy = kwargs.get("use_reduced_redundancy", False)
else:
- self.os_access_key = kwargs.get('os_access_key')
- self.os_secret_key = kwargs.get('os_secret_key')
- self.os_bucket_name = kwargs.get('os_bucket_name')
- self.os_use_reduced_redundancy = kwargs.get('os_use_reduced_redundancy', False)
- self.os_host = kwargs.get('os_host')
- self.os_port = kwargs.get('os_port')
- self.os_is_secure = string_as_bool(kwargs.get('os_is_secure', True))
- self.os_conn_path = kwargs.get('os_conn_path', '/')
- self.object_store_cache_size = float(kwargs.get('object_store_cache_size', -1))
- self.distributed_object_store_config_file = kwargs.get('distributed_object_store_config_file')
+ self.os_access_key = kwargs.get("os_access_key")
+ self.os_secret_key = kwargs.get("os_secret_key")
+ self.os_bucket_name = kwargs.get("os_bucket_name")
+ self.os_use_reduced_redundancy = kwargs.get("os_use_reduced_redundancy", False)
+ self.os_host = kwargs.get("os_host")
+ self.os_port = kwargs.get("os_port")
+ self.os_is_secure = string_as_bool(kwargs.get("os_is_secure", True))
+ self.os_conn_path = kwargs.get("os_conn_path", "/")
+ self.object_store_cache_size = float(kwargs.get("object_store_cache_size", -1))
+ self.distributed_object_store_config_file = kwargs.get("distributed_object_store_config_file")
if self.distributed_object_store_config_file is not None:
self.distributed_object_store_config_file = self._in_root_dir(self.distributed_object_store_config_file)
- self.irods_root_collection_path = kwargs.get('irods_root_collection_path')
- self.irods_default_resource = kwargs.get('irods_default_resource')
+ self.irods_root_collection_path = kwargs.get("irods_root_collection_path")
+ self.irods_default_resource = kwargs.get("irods_default_resource")
# Heartbeat log file name override
- if self.global_conf is not None and 'heartbeat_log' in self.global_conf:
- self.heartbeat_log = self.global_conf['heartbeat_log']
+ if self.global_conf is not None and "heartbeat_log" in self.global_conf:
+ self.heartbeat_log = self.global_conf["heartbeat_log"]
# Determine which 'server:' this is
- self.server_name = 'main'
+ self.server_name = "main"
for arg in sys.argv:
# Crummy, but PasteScript does not give you a way to determine this
- if arg.lower().startswith('--server-name='):
- self.server_name = arg.split('=', 1)[-1]
+ if arg.lower().startswith("--server-name="):
+ self.server_name = arg.split("=", 1)[-1]
# Allow explicit override of server name in config params
if "server_name" in kwargs:
self.server_name = kwargs.get("server_name")
# The application stack code may manipulate the server name. It also needs to be accessible via the get() method
# for galaxy.util.facts()
- self.config_dict['base_server_name'] = self.base_server_name = self.server_name
+ self.config_dict["base_server_name"] = self.base_server_name = self.server_name
# Store all configured server names for the message queue routing
self.server_names = []
for section in self.global_conf_parser.sections():
- if section.startswith('server:'):
- self.server_names.append(section.replace('server:', '', 1))
+ if section.startswith("server:"):
+ self.server_names.append(section.replace("server:", "", 1))
self._set_galaxy_infrastructure_url(kwargs)
# Asynchronous execution process pools - limited functionality for now, attach_to_pools is designed to allow
# webless Galaxy server processes to attach to arbitrary message queues (e.g. as job handlers) so they do not
# have to be explicitly defined as such in the job configuration.
- self.attach_to_pools = kwargs.get('attach_to_pools', []) or []
+ self.attach_to_pools = kwargs.get("attach_to_pools", []) or []
# Store advanced job management config
- self.job_handlers = [x.strip() for x in kwargs.get('job_handlers', self.server_name).split(',')]
- self.default_job_handlers = [x.strip() for x in kwargs.get('default_job_handlers', ','.join(self.job_handlers)).split(',')]
+ self.job_handlers = [x.strip() for x in kwargs.get("job_handlers", self.server_name).split(",")]
+ self.default_job_handlers = [
+ x.strip() for x in kwargs.get("default_job_handlers", ",".join(self.job_handlers)).split(",")
+ ]
# Galaxy internal control queue configuration.
# If specified in universe, use it, otherwise we use whatever 'real'
# database is specified. Lastly, we create and use new sqlite database
# (to minimize locking) as a final option.
- if 'amqp_internal_connection' in kwargs:
- self.amqp_internal_connection = kwargs.get('amqp_internal_connection')
+ if "amqp_internal_connection" in kwargs:
+ self.amqp_internal_connection = kwargs.get("amqp_internal_connection")
# TODO Get extra amqp args as necessary for ssl
- elif 'database_connection' in kwargs:
+ elif "database_connection" in kwargs:
self.amqp_internal_connection = f"sqlalchemy+{self.database_connection}"
else:
- self.amqp_internal_connection = f"sqlalchemy+sqlite:///{self._in_data_dir('control.sqlite')}?isolation_level=IMMEDIATE"
+ self.amqp_internal_connection = (
+ f"sqlalchemy+sqlite:///{self._in_data_dir('control.sqlite')}?isolation_level=IMMEDIATE"
+ )
self.pretty_datetime_format = expand_pretty_datetime_format(self.pretty_datetime_format)
try:
with open(self.user_preferences_extra_conf_path) as stream:
self.user_preferences_extra = yaml.safe_load(stream)
except Exception:
- if self.is_set('user_preferences_extra_conf_path'):
- log.warning(f'Config file ({self.user_preferences_extra_conf_path}) could not be found or is malformed.')
- self.user_preferences_extra = {'preferences': {}}
+ if self.is_set("user_preferences_extra_conf_path"):
+ log.warning(
+ f"Config file ({self.user_preferences_extra_conf_path}) could not be found or is malformed."
+ )
+ self.user_preferences_extra = {"preferences": {}}
# Experimental: This will not be enabled by default and will hide
# nonproduction code.
# The api_folders refers to whether the API exposes the /folders section.
- self.api_folders = string_as_bool(kwargs.get('api_folders', False))
+ self.api_folders = string_as_bool(kwargs.get("api_folders", False))
# This is for testing new library browsing capabilities.
- self.new_lib_browse = string_as_bool(kwargs.get('new_lib_browse', False))
+ self.new_lib_browse = string_as_bool(kwargs.get("new_lib_browse", False))
# Logging configuration with logging.config.configDict:
# Statistics and profiling with statsd
- self.statsd_host = kwargs.get('statsd_host', '')
+ self.statsd_host = kwargs.get("statsd_host", "")
ie_dirs = self.interactive_environment_plugins_directory
self.gie_dirs = [d.strip() for d in (ie_dirs.split(",") if ie_dirs else [])]
@@ -962,7 +1014,9 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
self.manage_dynamic_proxy = self.dynamic_proxy_manage # Set to false if being launched externally
# InteractiveTools propagator mapping file
- self.interactivetools_map = self._in_root_dir(kwargs.get("interactivetools_map", self._in_data_dir("interactivetools_map.sqlite")))
+ self.interactivetools_map = self._in_root_dir(
+ kwargs.get("interactivetools_map", self._in_data_dir("interactivetools_map.sqlite"))
+ )
self.containers_conf = parse_containers_config(self.containers_config_file)
@@ -989,58 +1043,56 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
self.redact_user_address_during_deletion = True
self.allow_user_deletion = True
- LOGGING_CONFIG_DEFAULT['formatters']['brief'] = {
- 'format': '%(asctime)s %(levelname)-8s %(name)-15s %(message)s'
+ LOGGING_CONFIG_DEFAULT["formatters"]["brief"] = {
+ "format": "%(asctime)s %(levelname)-8s %(name)-15s %(message)s"
}
- LOGGING_CONFIG_DEFAULT['handlers']['compliance_log'] = {
- 'class': 'logging.handlers.RotatingFileHandler',
- 'formatter': 'brief',
- 'filename': 'compliance.log',
- 'backupCount': 0,
+ LOGGING_CONFIG_DEFAULT["handlers"]["compliance_log"] = {
+ "class": "logging.handlers.RotatingFileHandler",
+ "formatter": "brief",
+ "filename": "compliance.log",
+ "backupCount": 0,
}
- LOGGING_CONFIG_DEFAULT['loggers']['COMPLIANCE'] = {
- 'handlers': ['compliance_log'],
- 'level': 'DEBUG',
- 'qualname': 'COMPLIANCE'
+ LOGGING_CONFIG_DEFAULT["loggers"]["COMPLIANCE"] = {
+ "handlers": ["compliance_log"],
+ "level": "DEBUG",
+ "qualname": "COMPLIANCE",
}
log_destination = kwargs.get("log_destination")
- galaxy_daemon_log_destination = os.environ.get('GALAXY_DAEMON_LOG')
+ galaxy_daemon_log_destination = os.environ.get("GALAXY_DAEMON_LOG")
if log_destination == "stdout":
- LOGGING_CONFIG_DEFAULT['handlers']['console'] = {
- 'class': 'logging.StreamHandler',
- 'formatter': 'stack',
- 'level': 'DEBUG',
- 'stream': 'ext://sys.stdout',
- 'filters': ['stack']
+ LOGGING_CONFIG_DEFAULT["handlers"]["console"] = {
+ "class": "logging.StreamHandler",
+ "formatter": "stack",
+ "level": "DEBUG",
+ "stream": "ext://sys.stdout",
+ "filters": ["stack"],
}
elif log_destination:
- LOGGING_CONFIG_DEFAULT['handlers']['console'] = {
- 'class': 'logging.FileHandler',
- 'formatter': 'stack',
- 'level': 'DEBUG',
- 'filename': log_destination,
- 'filters': ['stack']
+ LOGGING_CONFIG_DEFAULT["handlers"]["console"] = {
+ "class": "logging.FileHandler",
+ "formatter": "stack",
+ "level": "DEBUG",
+ "filename": log_destination,
+ "filters": ["stack"],
}
if galaxy_daemon_log_destination:
- LOGGING_CONFIG_DEFAULT['handlers']['files'] = {
- 'class': 'logging.FileHandler',
- 'formatter': 'stack',
- 'level': 'DEBUG',
- 'filename': galaxy_daemon_log_destination,
- 'filters': ['stack']
+ LOGGING_CONFIG_DEFAULT["handlers"]["files"] = {
+ "class": "logging.FileHandler",
+ "formatter": "stack",
+ "level": "DEBUG",
+ "filename": galaxy_daemon_log_destination,
+ "filters": ["stack"],
}
- LOGGING_CONFIG_DEFAULT['root']['handlers'].append('files')
+ LOGGING_CONFIG_DEFAULT["root"]["handlers"].append("files")
def _configure_dataset_storage(self):
# The default for `file_path` has changed in 20.05; we may need to fall back to the old default
- self._set_alt_paths('file_path', self._in_data_dir('files')) # this is called BEFORE guessing id/uuid
- ID, UUID = 'id', 'uuid'
- if self.is_set('object_store_store_by'):
+ self._set_alt_paths("file_path", self._in_data_dir("files")) # this is called BEFORE guessing id/uuid
+ ID, UUID = "id", "uuid"
+ if self.is_set("object_store_store_by"):
if self.object_store_store_by not in [ID, UUID]:
- raise Exception(
- f"Invalid value for object_store_store_by [{self.object_store_store_by}]"
- )
+ raise Exception(f"Invalid value for object_store_store_by [{self.object_store_store_by}]")
elif os.path.basename(self.file_path) == "objects":
self.object_store_store_by = UUID
else:
@@ -1053,35 +1105,41 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
def _set_galaxy_infrastructure_url(self, kwargs):
# indicate if this was not set explicitly, so dependending on the context a better default
# can be used (request url in a web thread, Docker parent in IE stuff, etc.)
- self.galaxy_infrastructure_url_set = kwargs.get('galaxy_infrastructure_url') is not None
+ self.galaxy_infrastructure_url_set = kwargs.get("galaxy_infrastructure_url") is not None
if "HOST_IP" in self.galaxy_infrastructure_url:
- self.galaxy_infrastructure_url = string.Template(self.galaxy_infrastructure_url).safe_substitute({
- 'HOST_IP': socket.gethostbyname(socket.gethostname())
- })
+ self.galaxy_infrastructure_url = string.Template(self.galaxy_infrastructure_url).safe_substitute(
+ {"HOST_IP": socket.gethostbyname(socket.gethostname())}
+ )
if "GALAXY_WEB_PORT" in self.galaxy_infrastructure_url:
- port = os.environ.get('GALAXY_WEB_PORT')
+ port = os.environ.get("GALAXY_WEB_PORT")
if not port:
- raise Exception('$GALAXY_WEB_PORT set in galaxy_infrastructure_url, but environment variable not set')
- self.galaxy_infrastructure_url = string.Template(self.galaxy_infrastructure_url).safe_substitute({
- 'GALAXY_WEB_PORT': port
- })
+ raise Exception("$GALAXY_WEB_PORT set in galaxy_infrastructure_url, but environment variable not set")
+ self.galaxy_infrastructure_url = string.Template(self.galaxy_infrastructure_url).safe_substitute(
+ {"GALAXY_WEB_PORT": port}
+ )
if "UWSGI_PORT" in self.galaxy_infrastructure_url:
import uwsgi
- http = unicodify(uwsgi.opt['http'])
+
+ http = unicodify(uwsgi.opt["http"])
host, port = http.split(":", 1)
assert port, "galaxy_infrastructure_url depends on dynamic PORT determination but port unknown"
- self.galaxy_infrastructure_url = string.Template(self.galaxy_infrastructure_url).safe_substitute({
- 'UWSGI_PORT': port
- })
+ self.galaxy_infrastructure_url = string.Template(self.galaxy_infrastructure_url).safe_substitute(
+ {"UWSGI_PORT": port}
+ )
def reload_sanitize_allowlist(self, explicit=True):
self.sanitize_allowlist = []
if not os.path.exists(self.sanitize_allowlist_file):
if explicit:
- log.warning("Sanitize log file explicitly specified as '%s' but does not exist, continuing with no tools allowlisted.", self.sanitize_allowlist_file)
+ log.warning(
+ "Sanitize log file explicitly specified as '%s' but does not exist, continuing with no tools allowlisted.",
+ self.sanitize_allowlist_file,
+ )
else:
with open(self.sanitize_allowlist_file) as f:
- self.sanitize_allowlist = sorted(line.strip() for line in f.readlines() if line.strip() and not line.startswith('#'))
+ self.sanitize_allowlist = sorted(
+ line.strip() for line in f.readlines() if line.strip() and not line.startswith("#")
+ )
def ensure_tempdir(self):
self._ensure_directory(self.new_file_path)
@@ -1113,7 +1171,9 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
# Check for deprecated options.
for key in self.config_dict.keys():
if key in self.deprecated_options:
- log.warning(f"Config option '{key}' is deprecated and will be removed in a future release. Please consult the latest version of the sample configuration file.")
+ log.warning(
+ f"Config option '{key}' is deprecated and will be removed in a future release. Please consult the latest version of the sample configuration file."
+ )
@staticmethod
def _parse_allowed_origin_hostnames(allowed_origin_hostnames):
@@ -1127,7 +1187,7 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
def parse(string):
# a string enclosed in fwd slashes will be parsed as a regexp: e.g. //
- if string[0] == '/' and string[-1] == '/':
+ if string[0] == "/" and string[-1] == "/":
string = string[1:-1]
return re.compile(string, flags=(re.UNICODE))
return string
@@ -1148,24 +1208,24 @@ def reload_config_options(current_config):
if current_config._raw_config[option] != modified_config[option]:
current_config._raw_config[option] = modified_config[option]
setattr(current_config, option, modified_config[option])
- log.info(f'Reloaded {option}')
+ log.info(f"Reloaded {option}")
-def get_database_engine_options(kwargs, model_prefix=''):
+def get_database_engine_options(kwargs, model_prefix=""):
"""
Allow options for the SQLAlchemy database engine to be passed by using
the prefix "database_engine_option".
"""
conversions: Dict[str, Callable[[Any], Union[bool, int]]] = {
- 'convert_unicode': string_as_bool,
- 'pool_timeout': int,
- 'echo': string_as_bool,
- 'echo_pool': string_as_bool,
- 'pool_recycle': int,
- 'pool_size': int,
- 'max_overflow': int,
- 'pool_threadlocal': string_as_bool,
- 'server_side_cursors': string_as_bool
+ "convert_unicode": string_as_bool,
+ "pool_timeout": int,
+ "echo": string_as_bool,
+ "echo_pool": string_as_bool,
+ "pool_recycle": int,
+ "pool_size": int,
+ "max_overflow": int,
+ "pool_threadlocal": string_as_bool,
+ "server_side_cursors": string_as_bool,
}
prefix = f"{model_prefix}database_engine_option_"
prefix_len = len(prefix)
@@ -1194,7 +1254,7 @@ def init_models_from_config(config, map_install_models=False, object_store=None,
database_query_profiling_proxy=config.database_query_profiling_proxy,
object_store=object_store,
trace_logger=trace_logger,
- use_pbkdf2=config.get_bool('use_pbkdf2', True),
+ use_pbkdf2=config.get_bool("use_pbkdf2", True),
slow_query_log_threshold=config.slow_query_log_threshold,
thread_local_log=config.thread_local_log,
log_query_counts=config.database_log_query_counts,
@@ -1218,19 +1278,25 @@ def configure_logging(config):
paste_configures_logging = config.global_conf_parser.has_section("loggers")
else:
paste_configures_logging = False
- auto_configure_logging = not paste_configures_logging and string_as_bool(config.get("auto_configure_logging", "True"))
+ auto_configure_logging = not paste_configures_logging and string_as_bool(
+ config.get("auto_configure_logging", "True")
+ )
if auto_configure_logging:
- logging_conf = config.get('logging', None)
+ logging_conf = config.get("logging", None)
if logging_conf is None:
# if using the default logging config, honor the log_level setting
logging_conf = LOGGING_CONFIG_DEFAULT
- if config.get('log_level', 'DEBUG') != 'DEBUG':
- logging_conf['handlers']['console']['level'] = config.get('log_level', 'DEBUG')
+ if config.get("log_level", "DEBUG") != "DEBUG":
+ logging_conf["handlers"]["console"]["level"] = config.get("log_level", "DEBUG")
# configure logging with logging dict in config, template *FileHandler handler filenames with the `filename_template` option
- for name, conf in logging_conf.get('handlers', {}).items():
- if conf['class'].startswith('logging.') and conf['class'].endswith('FileHandler') and 'filename_template' in conf:
- conf['filename'] = conf.pop('filename_template').format(**get_stack_facts(config=config))
- logging_conf['handlers'][name] = conf
+ for name, conf in logging_conf.get("handlers", {}).items():
+ if (
+ conf["class"].startswith("logging.")
+ and conf["class"].endswith("FileHandler")
+ and "filename_template" in conf
+ ):
+ conf["filename"] = conf.pop("filename_template").format(**get_stack_facts(config=config))
+ logging_conf["handlers"][name] = conf
logging.config.dictConfig(logging_conf)
@@ -1249,15 +1315,15 @@ class ConfiguresGalaxyMixin:
def wait_for_toolbox_reload(self, old_toolbox):
timer = ExecutionTimer()
- log.debug('Waiting for toolbox reload')
+ log.debug("Waiting for toolbox reload")
# Wait till toolbox reload has been triggered (or more than 60 seconds have passed)
while timer.elapsed < 60:
if self.toolbox.has_reloaded(old_toolbox):
- log.debug('Finished waiting for toolbox reload %s', timer)
+ log.debug("Finished waiting for toolbox reload %s", timer)
break
time.sleep(0.1)
else:
- log.warning('Waiting for toolbox reload timed out after 60 seconds')
+ log.warning("Waiting for toolbox reload timed out after 60 seconds")
def _configure_tool_config_files(self):
if self.config.shed_tool_config_file not in self.config.tool_configs:
@@ -1265,17 +1331,19 @@ class ConfiguresGalaxyMixin:
# The value of migrated_tools_config is the file reserved for containing only those tools that have been
# eliminated from the distribution and moved to the tool shed. If migration checking is disabled, only add it if
# it exists (since this may be an existing deployment where migrations were previously run).
- if (os.path.exists(self.config.migrated_tools_config)
- and self.config.migrated_tools_config not in self.config.tool_configs):
+ if (
+ os.path.exists(self.config.migrated_tools_config)
+ and self.config.migrated_tools_config not in self.config.tool_configs
+ ):
self.config.tool_configs.append(self.config.migrated_tools_config)
def _configure_toolbox(self):
+ import galaxy.tools.search
from galaxy import tools
- from galaxy.tools.biotools import get_galaxy_biotools_metadata_source
from galaxy.managers.citations import CitationsManager
from galaxy.tool_util.deps import containers
from galaxy.tool_util.deps.dependencies import AppInfo
- import galaxy.tools.search
+ from galaxy.tools.biotools import get_galaxy_biotools_metadata_source
if not isinstance(self, BasicSharedApp):
raise Exception("Must inherit from BasicSharedApp")
@@ -1308,15 +1376,19 @@ class ConfiguresGalaxyMixin:
mulled_resolution_cache = None
if self.config.mulled_resolution_cache_type:
cache_opts = {
- 'cache.type': self.config.mulled_resolution_cache_type,
- 'cache.data_dir': self.config.mulled_resolution_cache_data_dir,
- 'cache.lock_dir': self.config.mulled_resolution_cache_lock_dir,
+ "cache.type": self.config.mulled_resolution_cache_type,
+ "cache.data_dir": self.config.mulled_resolution_cache_data_dir,
+ "cache.lock_dir": self.config.mulled_resolution_cache_lock_dir,
}
- mulled_resolution_cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache('mulled_resolution')
+ mulled_resolution_cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache(
+ "mulled_resolution"
+ )
self.container_finder = containers.ContainerFinder(app_info, mulled_resolution_cache=mulled_resolution_cache)
self._set_enabled_container_types()
index_help = getattr(self.config, "index_tool_help", True)
- self.toolbox_search = galaxy.tools.search.ToolBoxSearch(self.toolbox, index_dir=self.config.tool_search_index_dir, index_help=index_help)
+ self.toolbox_search = galaxy.tools.search.ToolBoxSearch(
+ self.toolbox, index_dir=self.config.tool_search_index_dir, index_help=index_help
+ )
def reindex_tool_search(self):
# Call this when tools are added or removed.
@@ -1330,28 +1402,37 @@ class ConfiguresGalaxyMixin:
for enabled_container_type in self.container_finder._enabled_container_types(destination.params):
container_types_to_destinations[enabled_container_type].append(destination)
self.toolbox.dependency_manager.set_enabled_container_types(container_types_to_destinations)
- self.toolbox.dependency_manager.resolver_classes.update(self.container_finder.default_container_registry.resolver_classes)
- self.toolbox.dependency_manager.dependency_resolvers.extend(self.container_finder.default_container_registry.container_resolvers)
+ self.toolbox.dependency_manager.resolver_classes.update(
+ self.container_finder.default_container_registry.resolver_classes
+ )
+ self.toolbox.dependency_manager.dependency_resolvers.extend(
+ self.container_finder.default_container_registry.container_resolvers
+ )
def _configure_tool_data_tables(self, from_shed_config):
from galaxy.tools.data import ToolDataTableManager
# Initialize tool data tables using the config defined by self.config.tool_data_table_config_path.
- self.tool_data_tables = ToolDataTableManager(tool_data_path=self.config.tool_data_path,
- config_filename=self.config.tool_data_table_config_path,
- other_config_dict=self.config)
+ self.tool_data_tables = ToolDataTableManager(
+ tool_data_path=self.config.tool_data_path,
+ config_filename=self.config.tool_data_table_config_path,
+ other_config_dict=self.config,
+ )
# Load additional entries defined by self.config.shed_tool_data_table_config into tool data tables.
try:
- self.tool_data_tables.load_from_config_file(config_filename=self.config.shed_tool_data_table_config,
- tool_data_path=self.tool_data_tables.tool_data_path,
- from_shed_config=from_shed_config)
+ self.tool_data_tables.load_from_config_file(
+ config_filename=self.config.shed_tool_data_table_config,
+ tool_data_path=self.tool_data_tables.tool_data_path,
+ from_shed_config=from_shed_config,
+ )
except OSError as exc:
# Missing shed_tool_data_table_config is okay if it's the default
- if exc.errno != errno.ENOENT or self.config.is_set('shed_tool_data_table_config'):
+ if exc.errno != errno.ENOENT or self.config.is_set("shed_tool_data_table_config"):
raise
def _configure_datatypes_registry(self, installed_repository_manager=None):
from galaxy.datatypes import registry
+
# Create an empty datatypes registry.
self.datatypes_registry = registry.Registry(self.config)
if installed_repository_manager and self.config.load_tool_shed_datatypes:
@@ -1372,10 +1453,12 @@ class ConfiguresGalaxyMixin:
def _configure_object_store(self, **kwds):
from galaxy.objectstore import build_object_store_from_config
+
self.object_store = build_object_store_from_config(self.config, **kwds)
def _configure_security(self):
from galaxy.security import idencoding
+
self.security = idencoding.IdEncodingHelper(id_secret=self.config.id_secret)
BaseDatabaseIdField.security = self.security
@@ -1394,22 +1477,34 @@ class ConfiguresGalaxyMixin:
install_db_url = self.config.install_database_connection
# TODO: Consider more aggressive check here that this is not the same
# database file under the hood.
- combined_install_database = not(install_db_url and install_db_url != db_url)
+ combined_install_database = not (install_db_url and install_db_url != db_url)
install_db_url = install_db_url or db_url
- install_database_options = self.config.database_engine_options if combined_install_database else self.config.install_database_engine_options
+ install_database_options = (
+ self.config.database_engine_options
+ if combined_install_database
+ else self.config.install_database_engine_options
+ )
if self.config.database_wait:
self._wait_for_database(db_url)
if getattr(self.config, "max_metadata_value_size", None):
from galaxy.model import custom_types
+
custom_types.MAX_METADATA_VALUE_SIZE = self.config.max_metadata_value_size
if check_migrate_databases:
# Initialize database / check for appropriate schema version. # If this
# is a new installation, we'll restrict the tool migration messaging.
from galaxy.model.migrate.check import create_or_verify_database
- create_or_verify_database(db_url, config_file, self.config.database_engine_options, app=self, map_install_models=combined_install_database)
+
+ create_or_verify_database(
+ db_url,
+ config_file,
+ self.config.database_engine_options,
+ app=self,
+ map_install_models=combined_install_database,
+ )
if not combined_install_database:
tsi_create_or_verify_database(install_db_url, install_database_options, app=self)
@@ -1417,17 +1512,17 @@ class ConfiguresGalaxyMixin:
self.config,
map_install_models=combined_install_database,
object_store=self.object_store,
- trace_logger=getattr(self, "trace_logger", None)
+ trace_logger=getattr(self, "trace_logger", None),
)
if combined_install_database:
log.info("Install database targetting Galaxy's database configuration.")
self.install_model = self.model
else:
from galaxy.model.tool_shed_install import mapping as install_mapping
+
install_db_url = self.config.install_database_connection
log.info(f"Install database using its own connection {install_db_url}")
- self.install_model = install_mapping.init(install_db_url,
- install_database_options)
+ self.install_model = install_mapping.init(install_db_url, install_database_options)
def _configure_signal_handlers(self, handlers):
for sig, handler in handlers.items():
diff --git a/lib/galaxy/config/config_manage.py b/lib/galaxy/config/config_manage.py
index 26638a11ab4..8f508a1688d 100644
--- a/lib/galaxy/config/config_manage.py
+++ b/lib/galaxy/config/config_manage.py
@@ -7,7 +7,11 @@ import sys
import tempfile
from io import StringIO
from textwrap import TextWrapper
-from typing import Any, List, NamedTuple
+from typing import (
+ Any,
+ List,
+ NamedTuple,
+)
import requests
import yaml
@@ -18,7 +22,7 @@ try:
except ImportError:
Core = None
-if __name__ == '__main__':
+if __name__ == "__main__":
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)))
@@ -38,7 +42,6 @@ from galaxy.util.yaml_util import (
ordered_load,
)
-
DESCRIPTION = "Convert configuration files."
APP_DESCRIPTION = """Application to target for operation (i.e. galaxy, tool_shed, or reports))"""
@@ -49,151 +52,224 @@ EXTRA_SERVER_MESSAGE = "Additional server section after [%s] encountered [%s], w
MISSING_FILTER_TYPE_MESSAGE = "Missing filter type for section [%s], it will be ignored."
UNHANDLED_FILTER_TYPE_MESSAGE = "Unhandled filter type encountered [%s] for section [%s]."
NO_APP_MAIN_MESSAGE = "No app:main section found, using application defaults throughout."
-YAML_COMMENT_WRAPPER = TextWrapper(initial_indent="# ", subsequent_indent="# ", break_long_words=False, break_on_hyphens=False)
-RST_DESCRIPTION_WRAPPER = TextWrapper(initial_indent=" ", subsequent_indent=" ", break_long_words=False, break_on_hyphens=False)
+YAML_COMMENT_WRAPPER = TextWrapper(
+ initial_indent="# ", subsequent_indent="# ", break_long_words=False, break_on_hyphens=False
+)
+RST_DESCRIPTION_WRAPPER = TextWrapper(
+ initial_indent=" ", subsequent_indent=" ", break_long_words=False, break_on_hyphens=False
+)
-UWSGI_OPTIONS = dict([
- ('http', {
- 'desc': """The address and port on which to listen. By default, only listen to localhost ($app_name will not be accessible over the network). Use ':$default_port' to listen on all available network interfaces.""",
- 'default': '127.0.0.1:$default_port',
- 'type': 'str',
- }),
- ('buffer-size', {
- 'desc': """By default uWSGI allocates a very small buffer (4096 bytes) for the headers of each request. If you start receiving "invalid request block size" in your logs, it could mean you need a bigger buffer. We recommend at least 16384.""",
- 'default': 16384,
- 'type': 'int',
- }),
- ('processes', {
- 'desc': """Number of web server (worker) processes to fork after the application has loaded. If this is set to greater than 1, thunder-lock likely should be enabled below.""",
- 'default': 1,
- 'type': 'int',
- }),
- ('threads', {
- 'desc': """Number of threads for each web server process.""",
- 'default': 4,
- 'type': 'int',
- }),
- ('offload-threads', {
- 'desc': """Number of threads for serving static content and handling internal routing requests.""",
- 'default': 2,
- 'type': 'int',
- }),
- ('static-map.1', {
- 'key': 'static-map',
- 'desc': """Mapping to serve static content.""",
- 'default': '/static=static',
- 'type': 'str',
- }),
- ('static-map.2', {
- 'key': 'static-map',
- 'desc': """Mapping to serve the favicon.""",
- 'default': '/favicon.ico=static/favicon.ico',
- 'type': 'str',
- }),
- ('static-safe', {
- 'key': 'static-safe',
- 'desc': """Allow serving certain assets out of `client`. Most modern Galaxy interfaces bundle all of this, but some older pages still serve these via symlink, requiring this rule.""",
- 'default': 'client/src/assets',
- 'type': 'str',
- }),
- ('master', {
- 'desc': """Enable the master process manager. Disabled by default for maximum compatibility with CTRL+C, but should be enabled for use with --daemon and/or production deployments.""",
- 'default': False,
- 'type': 'bool',
- }),
- ('virtualenv', {
- 'desc': """Path to the application's Python virtual environment. If using Conda for Galaxy's framework dependencies (not tools!), do not set this.""",
- 'default': '.venv',
- 'type': 'str',
- }),
- ('pythonpath', {
- 'desc': """Path to the application's Python library.""",
- 'default': 'lib',
- 'type': 'str',
- }),
- ('module', {
- 'desc': """The entry point which returns the web application (e.g. Galaxy, Reports, etc.) that you are loading.""",
- 'default': '$uwsgi_module',
- 'type': 'str',
- }),
- ('#mount', {
- 'desc': """Mount the web application (e.g. Galaxy, Reports, etc.) at the given URL prefix. Cannot be used together with 'module:' above.""",
- 'default': '/galaxy=$uwsgi_module',
- 'type': 'str',
- }),
- ('manage-script-name', {
- 'desc': """Make uWSGI rewrite PATH_INFO and SCRIPT_NAME according to mount-points. Set this to true if a URL prefix is used.""",
- 'default': False,
- 'type': 'bool',
- }),
- ('thunder-lock', {
- 'desc': """It is usually a good idea to set this to ``true`` if processes is greater than 1.""",
- 'default': False,
- 'type': 'bool',
- }),
- ('die-on-term', {
- 'desc': """Cause uWSGI to respect the traditional behavior of dying on SIGTERM (its default is to brutally reload workers)""",
- 'default': True,
- 'type': 'bool',
- }),
- ('hook-master-start.1', {
- 'key': 'hook-master-start',
- 'desc': """Cause uWSGI to gracefully reload workers and mules upon receipt of SIGINT (its default is to brutally kill workers)""",
- 'default': 'unix_signal:2 gracefully_kill_them_all',
- 'type': 'str',
- }),
- ('hook-master-start.2', {
- 'key': 'hook-master-start',
- 'desc': """Cause uWSGI to gracefully reload workers and mules upon receipt of SIGTERM (its default is to brutally kill workers)""",
- 'default': 'unix_signal:15 gracefully_kill_them_all',
- 'type': 'str',
- }),
- ('py-call-osafterfork', {
- 'desc': """Feature necessary for proper mule signal handling on Python versions below 3.7.2. The default is set to false to prevent a runtime error under Python 3.7.2 and newer (see https://github.com/unbit/uwsgi/issues/1978).""",
- 'default': False,
- 'type': 'bool',
- }),
- ('enable-threads', {
- 'desc': """Ensure application threads will run if `threads` is unset.""",
- 'default': True,
- 'type': 'bool',
- }),
- ('umask', {
- 'desc': """uWSGI default umask. On some systems uWSGI has a default umask of 000, for Galaxy a somewhat safer default is chosen. If Galaxy submits jobs as real user then all users needs to be able to read the files, i.e. the umask needs to be '022' or the Galaxy users need to be in the same group as the Galaxy system user""",
- 'default': '027',
- 'type': 'str',
- }),
- # ('route-uri', {
- # 'default': '^/proxy/ goto:proxy'
- # }),
- # ('route', {
- # 'default': '.* last:'
- # }),
- # ('route-label', {
- # 'default': 'proxy'
- # }),
- # ('route-run', {
- # 'default': 'rpcvar:TARGET_HOST galaxy_dynamic_proxy_mapper ${HTTP_HOST} ${cookie[galaxysession]}'
- # }),
- # ('route-run', {
- # 'default': "['log:Proxy ${HTTP_HOST} to ${TARGET_HOST}', 'httpdumb:${TARGET_HOST}']",
- # }),
- # ('http-raw-body', {
- # 'default': True
- # }),
-])
+UWSGI_OPTIONS = dict(
+ [
+ (
+ "http",
+ {
+ "desc": """The address and port on which to listen. By default, only listen to localhost ($app_name will not be accessible over the network). Use ':$default_port' to listen on all available network interfaces.""",
+ "default": "127.0.0.1:$default_port",
+ "type": "str",
+ },
+ ),
+ (
+ "buffer-size",
+ {
+ "desc": """By default uWSGI allocates a very small buffer (4096 bytes) for the headers of each request. If you start receiving "invalid request block size" in your logs, it could mean you need a bigger buffer. We recommend at least 16384.""",
+ "default": 16384,
+ "type": "int",
+ },
+ ),
+ (
+ "processes",
+ {
+ "desc": """Number of web server (worker) processes to fork after the application has loaded. If this is set to greater than 1, thunder-lock likely should be enabled below.""",
+ "default": 1,
+ "type": "int",
+ },
+ ),
+ (
+ "threads",
+ {
+ "desc": """Number of threads for each web server process.""",
+ "default": 4,
+ "type": "int",
+ },
+ ),
+ (
+ "offload-threads",
+ {
+ "desc": """Number of threads for serving static content and handling internal routing requests.""",
+ "default": 2,
+ "type": "int",
+ },
+ ),
+ (
+ "static-map.1",
+ {
+ "key": "static-map",
+ "desc": """Mapping to serve static content.""",
+ "default": "/static=static",
+ "type": "str",
+ },
+ ),
+ (
+ "static-map.2",
+ {
+ "key": "static-map",
+ "desc": """Mapping to serve the favicon.""",
+ "default": "/favicon.ico=static/favicon.ico",
+ "type": "str",
+ },
+ ),
+ (
+ "static-safe",
+ {
+ "key": "static-safe",
+ "desc": """Allow serving certain assets out of `client`. Most modern Galaxy interfaces bundle all of this, but some older pages still serve these via symlink, requiring this rule.""",
+ "default": "client/src/assets",
+ "type": "str",
+ },
+ ),
+ (
+ "master",
+ {
+ "desc": """Enable the master process manager. Disabled by default for maximum compatibility with CTRL+C, but should be enabled for use with --daemon and/or production deployments.""",
+ "default": False,
+ "type": "bool",
+ },
+ ),
+ (
+ "virtualenv",
+ {
+ "desc": """Path to the application's Python virtual environment. If using Conda for Galaxy's framework dependencies (not tools!), do not set this.""",
+ "default": ".venv",
+ "type": "str",
+ },
+ ),
+ (
+ "pythonpath",
+ {
+ "desc": """Path to the application's Python library.""",
+ "default": "lib",
+ "type": "str",
+ },
+ ),
+ (
+ "module",
+ {
+ "desc": """The entry point which returns the web application (e.g. Galaxy, Reports, etc.) that you are loading.""",
+ "default": "$uwsgi_module",
+ "type": "str",
+ },
+ ),
+ (
+ "#mount",
+ {
+ "desc": """Mount the web application (e.g. Galaxy, Reports, etc.) at the given URL prefix. Cannot be used together with 'module:' above.""",
+ "default": "/galaxy=$uwsgi_module",
+ "type": "str",
+ },
+ ),
+ (
+ "manage-script-name",
+ {
+ "desc": """Make uWSGI rewrite PATH_INFO and SCRIPT_NAME according to mount-points. Set this to true if a URL prefix is used.""",
+ "default": False,
+ "type": "bool",
+ },
+ ),
+ (
+ "thunder-lock",
+ {
+ "desc": """It is usually a good idea to set this to ``true`` if processes is greater than 1.""",
+ "default": False,
+ "type": "bool",
+ },
+ ),
+ (
+ "die-on-term",
+ {
+ "desc": """Cause uWSGI to respect the traditional behavior of dying on SIGTERM (its default is to brutally reload workers)""",
+ "default": True,
+ "type": "bool",
+ },
+ ),
+ (
+ "hook-master-start.1",
+ {
+ "key": "hook-master-start",
+ "desc": """Cause uWSGI to gracefully reload workers and mules upon receipt of SIGINT (its default is to brutally kill workers)""",
+ "default": "unix_signal:2 gracefully_kill_them_all",
+ "type": "str",
+ },
+ ),
+ (
+ "hook-master-start.2",
+ {
+ "key": "hook-master-start",
+ "desc": """Cause uWSGI to gracefully reload workers and mules upon receipt of SIGTERM (its default is to brutally kill workers)""",
+ "default": "unix_signal:15 gracefully_kill_them_all",
+ "type": "str",
+ },
+ ),
+ (
+ "py-call-osafterfork",
+ {
+ "desc": """Feature necessary for proper mule signal handling on Python versions below 3.7.2. The default is set to false to prevent a runtime error under Python 3.7.2 and newer (see https://github.com/unbit/uwsgi/issues/1978).""",
+ "default": False,
+ "type": "bool",
+ },
+ ),
+ (
+ "enable-threads",
+ {
+ "desc": """Ensure application threads will run if `threads` is unset.""",
+ "default": True,
+ "type": "bool",
+ },
+ ),
+ (
+ "umask",
+ {
+ "desc": """uWSGI default umask. On some systems uWSGI has a default umask of 000, for Galaxy a somewhat safer default is chosen. If Galaxy submits jobs as real user then all users needs to be able to read the files, i.e. the umask needs to be '022' or the Galaxy users need to be in the same group as the Galaxy system user""",
+ "default": "027",
+ "type": "str",
+ },
+ ),
+ # ('route-uri', {
+ # 'default': '^/proxy/ goto:proxy'
+ # }),
+ # ('route', {
+ # 'default': '.* last:'
+ # }),
+ # ('route-label', {
+ # 'default': 'proxy'
+ # }),
+ # ('route-run', {
+ # 'default': 'rpcvar:TARGET_HOST galaxy_dynamic_proxy_mapper ${HTTP_HOST} ${cookie[galaxysession]}'
+ # }),
+ # ('route-run', {
+ # 'default': "['log:Proxy ${HTTP_HOST} to ${TARGET_HOST}', 'httpdumb:${TARGET_HOST}']",
+ # }),
+ # ('http-raw-body', {
+ # 'default': True
+ # }),
+ ]
+)
-SHED_ONLY_UWSGI_OPTIONS = [('cron', {
- 'desc': """Task for rebuilding Toolshed search indexes using the uWSGI cron-like interface.""",
- 'default': "0 -1 -1 -1 -1 python scripts/tool_shed/build_ts_whoosh_index.py -c config/tool_shed.yml --config-section tool_shed",
- 'type': 'str',
-})]
+SHED_ONLY_UWSGI_OPTIONS = [
+ (
+ "cron",
+ {
+ "desc": """Task for rebuilding Toolshed search indexes using the uWSGI cron-like interface.""",
+ "default": "0 -1 -1 -1 -1 python scripts/tool_shed/build_ts_whoosh_index.py -c config/tool_shed.yml --config-section tool_shed",
+ "type": "str",
+ },
+ )
+]
DROP_OPTION_VALUE = object()
class _OptionAction:
-
def converted(self, args, app_desc, key, value):
pass
@@ -202,13 +278,11 @@ class _OptionAction:
class _DeprecatedAction(_OptionAction):
-
def lint(self, args, app_desc, key, value):
print(f"Option [{key}] has been deprecated, this will likely be dropped in future releases of Galaxy.")
class _DeprecatedAndDroppedAction(_OptionAction):
-
def converted(self, args, app_desc, key, value):
print(f"Option [{key}] has been deprecated and dropped. It is not included in converted configuration.")
return DROP_OPTION_VALUE
@@ -218,7 +292,6 @@ class _DeprecatedAndDroppedAction(_OptionAction):
class _PasteAppFactoryAction(_OptionAction):
-
def converted(self, args, app_desc, key, value):
if value not in app_desc.expected_app_factories:
raise Exception(f"Ending convert process - unknown paste factory encountered [{value}]")
@@ -230,7 +303,6 @@ class _PasteAppFactoryAction(_OptionAction):
class _ProductionUnsafe(_OptionAction):
-
def __init__(self, unsafe_value):
self.unsafe_value = unsafe_value
@@ -242,7 +314,6 @@ class _ProductionUnsafe(_OptionAction):
class _ProductionPerformance(_OptionAction):
-
def lint(self, args, app_desc, key, value):
template = "Problem - option [%s] should not be set to [%s] in production environments - it may cause performance issues or instability."
message = template % (key, value)
@@ -250,14 +321,12 @@ class _ProductionPerformance(_OptionAction):
class _HandleFilterWithAction(_OptionAction):
-
def converted(self, args, app_desc, key, value):
print("filter-with converted to prefixed module load of uwsgi module, dropping from converted configuration")
return DROP_OPTION_VALUE
class _RenameAction(_OptionAction):
-
def __init__(self, new_name):
self.new_name = new_name
@@ -271,50 +340,50 @@ class _RenameAction(_OptionAction):
OPTION_ACTIONS = {
- 'use_beaker_session': _DeprecatedAndDroppedAction(),
- 'use_interactive': _DeprecatedAndDroppedAction(),
- 'session_type': _DeprecatedAndDroppedAction(),
- 'session_data_dir': _DeprecatedAndDroppedAction(),
- 'session_key': _DeprecatedAndDroppedAction(),
- 'session_secret': _DeprecatedAndDroppedAction(),
- 'paste.app_factory': _PasteAppFactoryAction(),
- 'filter-with': _HandleFilterWithAction(),
- 'debug': _ProductionUnsafe(True),
- 'serve_xss_vulnerable_mimetypes': _ProductionUnsafe(True),
- 'use_printdebug': _ProductionUnsafe(True),
- 'id_secret': _ProductionUnsafe('USING THE DEFAULT IS NOT SECURE!'),
- 'master_api_key': _ProductionUnsafe('changethis'),
- 'external_service_type_config_file': _DeprecatedAndDroppedAction(),
- 'external_service_type_path': _DeprecatedAndDroppedAction(),
- 'enable_sequencer_communication': _DeprecatedAndDroppedAction(),
- 'run_workflow_toolform_upgrade': _DeprecatedAndDroppedAction(),
+ "use_beaker_session": _DeprecatedAndDroppedAction(),
+ "use_interactive": _DeprecatedAndDroppedAction(),
+ "session_type": _DeprecatedAndDroppedAction(),
+ "session_data_dir": _DeprecatedAndDroppedAction(),
+ "session_key": _DeprecatedAndDroppedAction(),
+ "session_secret": _DeprecatedAndDroppedAction(),
+ "paste.app_factory": _PasteAppFactoryAction(),
+ "filter-with": _HandleFilterWithAction(),
+ "debug": _ProductionUnsafe(True),
+ "serve_xss_vulnerable_mimetypes": _ProductionUnsafe(True),
+ "use_printdebug": _ProductionUnsafe(True),
+ "id_secret": _ProductionUnsafe("USING THE DEFAULT IS NOT SECURE!"),
+ "master_api_key": _ProductionUnsafe("changethis"),
+ "external_service_type_config_file": _DeprecatedAndDroppedAction(),
+ "external_service_type_path": _DeprecatedAndDroppedAction(),
+ "enable_sequencer_communication": _DeprecatedAndDroppedAction(),
+ "run_workflow_toolform_upgrade": _DeprecatedAndDroppedAction(),
# Next 4 were from library search which is no longer available.
- 'enable_lucene_library_search': _DeprecatedAndDroppedAction(),
- 'fulltext_max_size': _DeprecatedAndDroppedAction(),
- 'fulltext_noindex_filetypes': _DeprecatedAndDroppedAction(),
- 'fulltext_url': _DeprecatedAndDroppedAction(),
- 'enable_beta_job_managers': _DeprecatedAndDroppedAction(),
- 'enable_legacy_sample_tracking_api': _DeprecatedAction(),
- 'enable_new_user_preferences': _DeprecatedAndDroppedAction(),
- 'force_beta_workflow_scheduled_for_collections': _DeprecatedAndDroppedAction(),
- 'force_beta_workflow_scheduled_min_steps': _DeprecatedAndDroppedAction(),
- 'history_local_serial_workflow_scheduling': _ProductionPerformance(),
- 'allow_library_path_paste': _RenameAction("allow_path_paste"),
- 'trust_ipython_notebook_conversion': _RenameAction("trust_jupyter_notebook_conversion"),
- 'enable_beta_tool_command_isolation': _DeprecatedAndDroppedAction(),
- 'enable_beta_ts_api_install': _DeprecatedAndDroppedAction(),
- 'single_user': _ProductionUnsafe(True),
- 'tool_submission_burst_threads': _DeprecatedAndDroppedAction(),
- 'tool_submission_burst_at': _DeprecatedAndDroppedAction(),
- 'toolform_upgrade': _DeprecatedAndDroppedAction(),
- 'enable_beta_mulled_containers': _DeprecatedAndDroppedAction(),
- 'enable_communication_server': _DeprecatedAndDroppedAction(),
- 'communication_server_host': _DeprecatedAndDroppedAction(),
- 'communication_server_port': _DeprecatedAndDroppedAction(),
- 'persistent_communication_rooms': _DeprecatedAndDroppedAction(),
- 'legacy_eager_objectstore_initialization': _DeprecatedAndDroppedAction(),
- 'enable_openid': _DeprecatedAndDroppedAction(),
- 'openid_consumer_cache_path': _DeprecatedAndDroppedAction(),
+ "enable_lucene_library_search": _DeprecatedAndDroppedAction(),
+ "fulltext_max_size": _DeprecatedAndDroppedAction(),
+ "fulltext_noindex_filetypes": _DeprecatedAndDroppedAction(),
+ "fulltext_url": _DeprecatedAndDroppedAction(),
+ "enable_beta_job_managers": _DeprecatedAndDroppedAction(),
+ "enable_legacy_sample_tracking_api": _DeprecatedAction(),
+ "enable_new_user_preferences": _DeprecatedAndDroppedAction(),
+ "force_beta_workflow_scheduled_for_collections": _DeprecatedAndDroppedAction(),
+ "force_beta_workflow_scheduled_min_steps": _DeprecatedAndDroppedAction(),
+ "history_local_serial_workflow_scheduling": _ProductionPerformance(),
+ "allow_library_path_paste": _RenameAction("allow_path_paste"),
+ "trust_ipython_notebook_conversion": _RenameAction("trust_jupyter_notebook_conversion"),
+ "enable_beta_tool_command_isolation": _DeprecatedAndDroppedAction(),
+ "enable_beta_ts_api_install": _DeprecatedAndDroppedAction(),
+ "single_user": _ProductionUnsafe(True),
+ "tool_submission_burst_threads": _DeprecatedAndDroppedAction(),
+ "tool_submission_burst_at": _DeprecatedAndDroppedAction(),
+ "toolform_upgrade": _DeprecatedAndDroppedAction(),
+ "enable_beta_mulled_containers": _DeprecatedAndDroppedAction(),
+ "enable_communication_server": _DeprecatedAndDroppedAction(),
+ "communication_server_host": _DeprecatedAndDroppedAction(),
+ "communication_server_port": _DeprecatedAndDroppedAction(),
+ "persistent_communication_rooms": _DeprecatedAndDroppedAction(),
+ "legacy_eager_objectstore_initialization": _DeprecatedAndDroppedAction(),
+ "enable_openid": _DeprecatedAndDroppedAction(),
+ "openid_consumer_cache_path": _DeprecatedAndDroppedAction(),
}
@@ -348,10 +417,12 @@ class OptionValue(NamedTuple):
GALAXY_APP = App(
["universe_wsgi.ini", "config/galaxy.ini"],
"8080",
- ["galaxy.web.buildapp:app_factory"], # TODO: Galaxy could call factory a few different things and they'd all be fine.
+ [
+ "galaxy.web.buildapp:app_factory"
+ ], # TODO: Galaxy could call factory a few different things and they'd all be fine.
"config/galaxy.yml",
str(GALAXY_CONFIG_SCHEMA_PATH),
- 'galaxy.webapps.galaxy.buildapp:uwsgi_app()',
+ "galaxy.webapps.galaxy.buildapp:uwsgi_app()",
)
SHED_APP = App(
["tool_shed_wsgi.ini", "config/tool_shed.ini"],
@@ -359,7 +430,7 @@ SHED_APP = App(
["tool_shed.webapp.buildapp:app_factory"],
"config/tool_shed.yml",
"lib/tool_shed/webapp/config_schema.yml",
- 'tool_shed.webapp.buildapp:uwsgi_app()',
+ "tool_shed.webapp.buildapp:uwsgi_app()",
)
REPORTS_APP = App(
["reports_wsgi.ini", "config/reports.ini"],
@@ -367,7 +438,7 @@ REPORTS_APP = App(
["galaxy.webapps.reports.buildapp:app_factory"],
"config/reports.yml",
"lib/galaxy/webapps/reports/config_schema.yml",
- 'galaxy.webapps.reports.buildapp:uwsgi_app()',
+ "galaxy.webapps.reports.buildapp:uwsgi_app()",
)
APPS = {"galaxy": GALAXY_APP, "tool_shed": SHED_APP, "reports": REPORTS_APP}
@@ -386,15 +457,11 @@ def main(argv=None):
def _arg_parser():
parser = argparse.ArgumentParser(description=DESCRIPTION)
- parser.add_argument('action', metavar='ACTION', type=str,
- choices=list(ACTIONS.keys()),
- help='action to perform')
- parser.add_argument('app', metavar='APP', type=str, nargs="?",
- help=APP_DESCRIPTION)
- parser.add_argument('--add-comments', default=False, action="store_true")
- parser.add_argument('--dry-run', default=False, action="store_true",
- help=DRY_RUN_DESCRIPTION)
- parser.add_argument('--galaxy_root', default=".", type=str)
+ parser.add_argument("action", metavar="ACTION", type=str, choices=list(ACTIONS.keys()), help="action to perform")
+ parser.add_argument("app", metavar="APP", type=str, nargs="?", help=APP_DESCRIPTION)
+ parser.add_argument("--add-comments", default=False, action="store_true")
+ parser.add_argument("--dry-run", default=False, action="store_true", help=DRY_RUN_DESCRIPTION)
+ parser.add_argument("--galaxy_root", default=".", type=str)
return parser
@@ -438,7 +505,7 @@ def _write_option_rst(args, rst, key, heading_level, option_value):
def _build_uwsgi_schema(args, app_desc):
- req = requests.get('https://raw.githubusercontent.com/unbit/uwsgi-docs/master/Options.rst')
+ req = requests.get("https://raw.githubusercontent.com/unbit/uwsgi-docs/master/Options.rst")
rst_options = req.text
last_line = None
current_opt = None
@@ -451,7 +518,7 @@ def _build_uwsgi_schema(args, app_desc):
if line and (line == dots):
current_opt = last_line
option = {
- 'type': 'any',
+ "type": "any",
}
options[current_opt] = option
@@ -467,7 +534,7 @@ def _build_uwsgi_schema(args, app_desc):
schema = {
"type": "map",
"desc": "uwsgi definition, see https://uwsgi-docs.readthedocs.io/en/latest/Options.html",
- "mapping": options
+ "mapping": options,
}
contents = ordered_dump(schema)
_write_to_file(args, contents, UWSGI_SCHEMA_PATH)
@@ -534,14 +601,14 @@ def _validate(args, app_desc):
if raw_config.get(app_desc.app_name) is None:
raw_config[app_desc.app_name] = {}
# Rewrite the file any way to merge any duplicate keys
- with tempfile.NamedTemporaryFile('w', delete=False, suffix=".yml") as config_p:
+ with tempfile.NamedTemporaryFile("w", delete=False, suffix=".yml") as config_p:
ordered_dump(raw_config, config_p)
def _clean(p, k, v):
- return k not in ['reloadable', 'path_resolves_to', 'per_host']
+ return k not in ["reloadable", "path_resolves_to", "per_host"]
clean_schema = remap(app_desc.schema.raw_schema, _clean)
- with tempfile.NamedTemporaryFile('w', suffix=".yml") as fp:
+ with tempfile.NamedTemporaryFile("w", suffix=".yml") as fp:
ordered_dump(clean_schema, fp)
fp.flush()
c = Core(
@@ -553,14 +620,12 @@ def _validate(args, app_desc):
class PrefixFilter:
-
def __init__(self, name, prefix):
self.name = name
self.prefix = prefix
class GzipFilter:
-
def __init__(self, name):
self.name = name
@@ -585,7 +650,7 @@ def _run_conversion(args, app_desc):
server_section = section
if section.startswith("filter:"):
- filter_name = section[len("filter:"):]
+ filter_name = section[len("filter:") :]
filter_type = p.get(section, "use")
if filter_type is None:
MISSING_FILTER_TYPE_MESSAGE
@@ -681,18 +746,20 @@ def _build_sample_yaml(args, app_desc):
if not isinstance(field_value, str):
continue
- new_field_value = string.Template(field_value).safe_substitute(**{
- 'default_port': str(app_desc.default_port),
- 'app_name': app_desc.app_name,
- 'uwsgi_module': app_desc.uwsgi_module,
- })
+ new_field_value = string.Template(field_value).safe_substitute(
+ **{
+ "default_port": str(app_desc.default_port),
+ "app_name": app_desc.app_name,
+ "uwsgi_module": app_desc.uwsgi_module,
+ }
+ )
value[field] = new_field_value
description = getattr(schema, "description", None)
if description:
description = description.lstrip()
as_comment = "\n".join(f"# {line}" for line in description.split("\n")) + "\n"
f.write(as_comment)
- _write_sample_section(args, f, 'uwsgi', Schema(UWSGI_OPTIONS), as_comment=False, uwsgi_hack=True)
+ _write_sample_section(args, f, "uwsgi", Schema(UWSGI_OPTIONS), as_comment=False, uwsgi_hack=True)
_write_sample_section(args, f, app_desc.app_name, schema)
destination = os.path.join(args.galaxy_root, app_desc.sample_destination)
_write_to_file(args, f, destination)
@@ -729,7 +796,7 @@ def _write_sample_section(args, f, section_header, schema, as_comment=True, uwsg
option = schema.get_app_option(key)
option_value = OptionValue(key, default, option)
# support uWSGI "dumb YAML parser" (unbit/uwsgi#863)
- key = option.get('key', key)
+ key = option.get("key", key)
_write_option(args, f, key, option_value, as_comment=as_comment, uwsgi_hack=uwsgi_hack)
@@ -835,5 +902,5 @@ ACTIONS = {
}
-if __name__ == '__main__':
+if __name__ == "__main__":
main()
diff --git a/lib/galaxy/config/schema.py b/lib/galaxy/config/schema.py
index ebd88e05c74..b8f87c7bc85 100644
--- a/lib/galaxy/config/schema.py
+++ b/lib/galaxy/config/schema.py
@@ -16,12 +16,11 @@ UNKNOWN_OPTION = {
"type": "str",
"required": False,
"unknown_option": True,
- "desc": "Unknown option, may want to remove or report to Galaxy team."
+ "desc": "Unknown option, may want to remove or report to Galaxy team.",
}
class Schema:
-
def __init__(self, mapping):
self.app_schema = mapping
@@ -36,11 +35,10 @@ class Schema:
class AppSchema(Schema):
-
def __init__(self, schema_path, app_name):
self.raw_schema = self._read_schema(schema_path)
self.description = self.raw_schema.get("desc", None)
- app_schema = self.raw_schema['mapping'][app_name]['mapping']
+ app_schema = self.raw_schema["mapping"][app_name]["mapping"]
self._preprocess(app_schema)
super().__init__(app_schema)
@@ -55,13 +53,13 @@ class AppSchema(Schema):
self._paths_to_resolve = {} # {config option: referenced config option}
self._per_host_options = set() # config options that can be set using a per_host config parameter
for key, data in app_schema.items():
- self._defaults[key] = data.get('default')
- if data.get('reloadable'):
+ self._defaults[key] = data.get("default")
+ if data.get("reloadable"):
self._reloadable_options.add(key)
- if data.get('per_host'):
+ if data.get("per_host"):
self._per_host_options.add(key)
- if data.get('path_resolves_to'):
- self._paths_to_resolve[key] = data.get('path_resolves_to')
+ if data.get("path_resolves_to"):
+ self._paths_to_resolve[key] = data.get("path_resolves_to")
@property
def defaults(self):
@@ -81,16 +79,19 @@ class AppSchema(Schema):
def validate_path_resolution_graph(self):
"""This method is for tests only: we SHOULD validate the schema's path resolution graph
- as part of automated testing; but we should NOT validate it at runtime.
+ as part of automated testing; but we should NOT validate it at runtime.
"""
+
def check_exists(option, key):
if not option:
- message = "Invalid schema: property '{}' listed as path resolution target " \
+ message = (
+ "Invalid schema: property '{}' listed as path resolution target "
"for '{}' does not exist".format(resolves_to, key)
+ )
raise_error(message)
def check_type_is_str_or_any(option, key):
- if option.get('type') not in ('str', 'any'):
+ if option.get("type") not in ("str", "any"):
message = f"Invalid schema: property '{key}' should have type 'str'"
raise_error(message)
@@ -100,9 +101,9 @@ class AppSchema(Schema):
visited.clear()
while key:
visited.add(key)
- key = self.app_schema[key].get('path_resolves_to')
+ key = self.app_schema[key].get("path_resolves_to")
if key and key in visited:
- raise_error('Invalid schema: cycle detected')
+ raise_error("Invalid schema: cycle detected")
def raise_error(message):
log.error(message)
diff --git a/lib/galaxy/config/script.py b/lib/galaxy/config/script.py
index 82cdad9018a..b6f724fad91 100644
--- a/lib/galaxy/config/script.py
+++ b/lib/galaxy/config/script.py
@@ -17,26 +17,26 @@ DESCRIPTION = "Initialize a directory with a minimal Galaxy config."
HELP_CONFIG_DIR = "Directory containing the configuration files for Galaxy."
HELP_DATA_DIR = "Directory containing Galaxy-created data."
HELP_FORCE = "Overwrite existing files if they already exist."
-HELP_WSGI_SERVER = ("Web server stack used to host Galaxy web application, and if uWSGI, which protocol to use.")
+HELP_WSGI_SERVER = "Web server stack used to host Galaxy web application, and if uWSGI, which protocol to use."
HELP_LIBDRMAA = (
"Configure Galaxy to submit jobs to a cluster via DRMAA by supplying the path to a libdrmaa.so file using this "
"argument."
)
-HELP_INSTALL = ("Install optional dependencies required by specified configuration (e.g. drmaa, etc...).")
+HELP_INSTALL = "Install optional dependencies required by specified configuration (e.g. drmaa, etc...)."
HELP_HOST = (
'Host to bind Galaxy to - defaults to localhost. Specify an IP address or "all" to listen on all interfaces.'
)
-HELP_PORT = ("Port to bind Galaxy to.")
-HELP_DB_CONN = ("Galaxy database connection URI.")
+HELP_PORT = "Port to bind Galaxy to."
+HELP_DB_CONN = "Galaxy database connection URI."
DEFAULT_HOST = "localhost"
DEFAULT_YML = "galaxy.yml"
-DEFAULT_DB_CONN = 'sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE'
+DEFAULT_DB_CONN = "sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE"
-SAMPLES_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), 'sample'))
-GALAXY_CONFIG_TEMPLATE_FILE = os.path.join(SAMPLES_PATH, 'galaxy.yml.sample')
-STATIC_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'web', 'framework', 'static'))
-CLIENT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, 'client'))
+SAMPLES_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "sample"))
+GALAXY_CONFIG_TEMPLATE_FILE = os.path.join(SAMPLES_PATH, "galaxy.yml.sample")
+STATIC_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "web", "framework", "static"))
+CLIENT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, "client"))
MSG_CONFIG_SUMMARY = """
For help on configuring Galaxy, consult the documentation at: \n {}
@@ -51,15 +51,15 @@ Start Galaxy by running the command from directory [{}]:
# whole thing into galaxy.config for templating, so for now just substitute some lines. In the future we will build
# configs differently.
GALAXY_CONFIG_SUBSTITUTIONS = {
- ' http: 127.0.0.1:8080': ' ${uwsgi_transport}: ${host}:${port}',
- ' static-map: /static=static': ' static-map: /static=${static_path}',
- ' static-map: /favicon.ico=static/favicon.ico': ' static-map: /static=${static_path}/favicon.ico',
- ' static-safe: client/src/assets': ' ${client_path}/src/assets',
- ' virtualenv: .venv': ' #venv: .venv # not used when running installed',
- ' pythonpath: lib': ' #pythonpath: lib # not used when running installed',
- ' #config_dir: false': ' config_dir: ${config_dir}',
- ' #data_dir: false': ' data_dir: ${data_dir}',
- ' #database_connection: sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE': ' database_connection: ${database_connection}',
+ " http: 127.0.0.1:8080": " ${uwsgi_transport}: ${host}:${port}",
+ " static-map: /static=static": " static-map: /static=${static_path}",
+ " static-map: /favicon.ico=static/favicon.ico": " static-map: /static=${static_path}/favicon.ico",
+ " static-safe: client/src/assets": " ${client_path}/src/assets",
+ " virtualenv: .venv": " #venv: .venv # not used when running installed",
+ " pythonpath: lib": " #pythonpath: lib # not used when running installed",
+ " #config_dir: false": " config_dir: ${config_dir}",
+ " #data_dir: false": " data_dir: ${data_dir}",
+ " #database_connection: sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE": " database_connection: ${database_connection}",
}
@@ -68,8 +68,9 @@ def main(argv=None):
arg_parser = ArgumentParser(description=DESCRIPTION)
arg_parser.add_argument("--config-dir", default=".", help=HELP_CONFIG_DIR)
arg_parser.add_argument("--data-dir", default="./data", help=HELP_DATA_DIR)
- arg_parser.add_argument("--wsgi-server", choices=["uwsgi-http", "uwsgi-native"], default="uwsgi-http",
- help=HELP_WSGI_SERVER)
+ arg_parser.add_argument(
+ "--wsgi-server", choices=["uwsgi-http", "uwsgi-native"], default="uwsgi-http", help=HELP_WSGI_SERVER
+ )
arg_parser.add_argument("--host", default=DEFAULT_HOST, help=HELP_HOST)
arg_parser.add_argument("--port", default="8080", help=HELP_PORT)
arg_parser.add_argument("--db-conn", default=DEFAULT_DB_CONN, help=HELP_DB_CONN)
@@ -129,7 +130,7 @@ def _determine_mode(args):
def _determine_host(args):
- return '0.0.0.0' if args.host == 'all' else args.host
+ return "0.0.0.0" if args.host == "all" else args.host
def _determine_yml_file(config_dir):
@@ -140,7 +141,7 @@ def _handle_galaxy_yml(args, config_dir, data_dir):
force = args.force
yml_file = _determine_yml_file(config_dir)
_check_file(yml_file, force)
- uwsgi_transport = 'socket' if args.wsgi_server == 'uwsgi-native' else 'http'
+ uwsgi_transport = "socket" if args.wsgi_server == "uwsgi-native" else "http"
config_dict = dict(
port=args.port,
host=_determine_host(args),
@@ -155,16 +156,14 @@ def _handle_galaxy_yml(args, config_dir, data_dir):
galaxy_config_template = []
with open(GALAXY_CONFIG_TEMPLATE_FILE) as fh:
for line in fh:
- line = line.rstrip('\n')
+ line = line.rstrip("\n")
for k, v in GALAXY_CONFIG_SUBSTITUTIONS.items():
if line == k:
line = v
galaxy_config_template.append(line)
- galaxy_config_template = string.Template('\n'.join(galaxy_config_template))
+ galaxy_config_template = string.Template("\n".join(galaxy_config_template))
- galaxy_config = galaxy_config_template.safe_substitute(
- **config_dict
- )
+ galaxy_config = galaxy_config_template.safe_substitute(**config_dict)
open(yml_file, "w").write(galaxy_config)
@@ -182,5 +181,5 @@ def _check_file(path, force):
sys.exit(1)
-if __name__ == '__main__':
+if __name__ == "__main__":
main()
diff --git a/lib/galaxy/config_watchers.py b/lib/galaxy/config_watchers.py
index 4601319deba..0482aa9ac40 100644
--- a/lib/galaxy/config_watchers.py
+++ b/lib/galaxy/config_watchers.py
@@ -31,42 +31,42 @@ class ConfigWatchers:
try:
# Run and wait for toolbox reload on the process that watches the config files.
# The toolbox reload will update the integrated_tool_panel_file
- self.app.queue_worker.send_local_control_task('reload_toolbox', get_response=True),
+ self.app.queue_worker.send_local_control_task("reload_toolbox", get_response=True),
except Exception:
save_integrated_tool_panel = True
log.exception("Exception occured while reloading toolbox")
- self.app.queue_worker.send_control_task('reload_toolbox', noop_self=True, kwargs={'save_integrated_tool_panel': save_integrated_tool_panel}),
+ self.app.queue_worker.send_control_task(
+ "reload_toolbox", noop_self=True, kwargs={"save_integrated_tool_panel": save_integrated_tool_panel}
+ ),
self.tool_config_watcher = get_tool_conf_watcher(
reload_callback=reload_toolbox,
tool_cache=self.app.tool_cache,
)
self.data_manager_config_watcher = get_tool_conf_watcher(
- reload_callback=lambda: self.app.queue_worker.send_control_task('reload_data_managers'),
+ reload_callback=lambda: self.app.queue_worker.send_control_task("reload_data_managers"),
)
- self.tool_data_watcher = get_watcher(self.app.config, 'watch_tool_data_dir', monitor_what_str='data tables')
+ self.tool_data_watcher = get_watcher(self.app.config, "watch_tool_data_dir", monitor_what_str="data tables")
self.tool_watcher = get_tool_watcher(self, app.config)
- if getattr(self.app, 'is_job_handler', False):
- self.job_rule_watcher = get_watcher(app.config, 'watch_job_rules', monitor_what_str='job rules')
+ if getattr(self.app, "is_job_handler", False):
+ self.job_rule_watcher = get_watcher(app.config, "watch_job_rules", monitor_what_str="job rules")
else:
- self.job_rule_watcher = get_watcher(app.config, '__invalid__')
- self.core_config_watcher = get_watcher(
- app.config,
- 'watch_core_config',
- monitor_what_str='core config file'
- )
- self.tour_watcher = get_watcher(app.config, 'watch_tours', monitor_what_str='tours')
+ self.job_rule_watcher = get_watcher(app.config, "__invalid__")
+ self.core_config_watcher = get_watcher(app.config, "watch_core_config", monitor_what_str="core config file")
+ self.tour_watcher = get_watcher(app.config, "watch_tours", monitor_what_str="tours")
@property
def watchers(self):
- return (self.tool_watcher,
- self.tool_config_watcher,
- self.data_manager_config_watcher,
- self.tool_data_watcher,
- self.tool_watcher,
- self.job_rule_watcher,
- self.core_config_watcher,
- self.tour_watcher)
+ return (
+ self.tool_watcher,
+ self.tool_config_watcher,
+ self.data_manager_config_watcher,
+ self.tool_data_watcher,
+ self.tool_watcher,
+ self.job_rule_watcher,
+ self.core_config_watcher,
+ self.tour_watcher,
+ )
def change_state(self, active):
if active:
@@ -82,24 +82,27 @@ class ConfigWatchers:
for tool_data_path in self.tool_data_paths:
self.tool_data_watcher.watch_directory(
tool_data_path,
- callback=lambda path: self.app.queue_worker.send_control_task('reload_tool_data_tables', kwargs={'path': path}),
- require_extensions=('.loc',),
+ callback=lambda path: self.app.queue_worker.send_control_task(
+ "reload_tool_data_tables", kwargs={"path": path}
+ ),
+ require_extensions=(".loc",),
recursive=True,
)
for job_rules_directory in self.job_rules_paths:
self.job_rule_watcher.watch_directory(
job_rules_directory,
- callback=lambda: self.app.queue_worker.send_control_task('reload_job_rules'),
+ callback=lambda: self.app.queue_worker.send_control_task("reload_job_rules"),
recursive=True,
- ignore_extensions=('.pyc', '.pyo', '.pyd'))
+ ignore_extensions=(".pyc", ".pyo", ".pyd"),
+ )
if self.app.config.config_file:
self.core_config_watcher.watch_file(
self.app.config.config_file,
- callback=lambda path: self.app.queue_worker.send_control_task('reload_core_config')
+ callback=lambda path: self.app.queue_worker.send_control_task("reload_core_config"),
)
self.tour_watcher.watch_directory(
self.app.config.tour_config_dir,
- callback=lambda path: self.app.queue_worker.send_control_task('reload_tour', kwargs={'path': path})
+ callback=lambda path: self.app.queue_worker.send_control_task("reload_tour", kwargs={"path": path}),
)
self.active = True
@@ -109,7 +112,7 @@ class ConfigWatchers:
self.active = False
def update_watch_data_table_paths(self):
- if hasattr(self.tool_data_watcher, 'monitored_dirs'):
+ if hasattr(self.tool_data_watcher, "monitored_dirs"):
for tool_data_table_path in self.tool_data_paths:
if tool_data_table_path not in self.tool_data_watcher.monitored_dirs:
self.tool_data_watcher.watch_directory(tool_data_table_path)
@@ -117,25 +120,25 @@ class ConfigWatchers:
@property
def data_manager_configs(self):
data_manager_configs = []
- if hasattr(self.app.config, 'data_manager_config_file'):
+ if hasattr(self.app.config, "data_manager_config_file"):
data_manager_configs.append(self.app.config.data_manager_config_file)
- if hasattr(self.app.config, 'shed_data_manager_config_file'):
+ if hasattr(self.app.config, "shed_data_manager_config_file"):
data_manager_configs.append(self.app.config.shed_data_manager_config_file)
return data_manager_configs
@property
def tool_data_paths(self):
tool_data_paths = []
- if hasattr(self.app.config, 'tool_data_path'):
+ if hasattr(self.app.config, "tool_data_path"):
tool_data_paths.append(self.app.config.tool_data_path)
- if hasattr(self.app.config, 'shed_tool_data_path'):
+ if hasattr(self.app.config, "shed_tool_data_path"):
tool_data_paths.append(self.app.config.shed_tool_data_path)
return tool_data_paths
@property
def tool_config_paths(self):
tool_config_paths = []
- if hasattr(self.app.config, 'tool_configs'):
+ if hasattr(self.app.config, "tool_configs"):
tool_config_paths = self.app.config.tool_configs
return tool_config_paths
diff --git a/lib/galaxy/containers/__init__.py b/lib/galaxy/containers/__init__.py
index ffb9c9e25c8..35d9cbf9012 100644
--- a/lib/galaxy/containers/__init__.py
+++ b/lib/galaxy/containers/__init__.py
@@ -12,24 +12,30 @@ import uuid
from abc import (
ABCMeta,
abstractmethod,
- abstractproperty
+ abstractproperty,
+)
+from typing import (
+ Any,
+ Dict,
+ NamedTuple,
+ Optional,
+ Type,
)
-from typing import Any, Dict, NamedTuple, Optional, Type
import yaml
from galaxy.exceptions import ContainerCLIError
from galaxy.util.submodules import import_submodules
-
-DEFAULT_CONTAINER_TYPE = 'docker'
-DEFAULT_CONF = {'_default_': {'type': DEFAULT_CONTAINER_TYPE}}
+DEFAULT_CONTAINER_TYPE = "docker"
+DEFAULT_CONF = {"_default_": {"type": DEFAULT_CONTAINER_TYPE}}
log = logging.getLogger(__name__)
class ContainerPort(NamedTuple):
"""Named tuple representing ports published by a container, with attributes"""
+
port: int # Port number (inside the container)
protocol: str # Port protocol, either ``tcp`` or ``udp``
hostaddr: str # Address or hostname where the published port can be accessed
@@ -57,13 +63,11 @@ class ContainerVolume(metaclass=ABCMeta):
@abstractmethod
def __str__(self):
- """Return this container type's string representation of the volume.
- """
+ """Return this container type's string representation of the volume."""
@abstractmethod
def to_native(self):
- """Return this container type's native representation of the volume.
- """
+ """Return this container type's native representation of the volume."""
@property
def mode_is_valid(self):
@@ -71,7 +75,6 @@ class ContainerVolume(metaclass=ABCMeta):
class Container(metaclass=ABCMeta):
-
def __init__(self, interface, id, name=None, **kwargs):
"""
@@ -149,9 +152,13 @@ class Container(metaclass=ABCMeta):
if port == mapping.port:
return mapping
if port is None:
- log.warning("Container %s (%s): Don't know how to map ports to containers with multiple exposed ports "
- "when a specific port is not requested. Arbitrarily choosing first: %s",
- self.name, self.id, mapping)
+ log.warning(
+ "Container %s (%s): Don't know how to map ports to containers with multiple exposed ports "
+ "when a specific port is not requested. Arbitrarily choosing first: %s",
+ self.name,
+ self.id,
+ mapping,
+ )
return mapping
else:
if port is None:
@@ -167,7 +174,7 @@ class ContainerInterface(metaclass=ABCMeta):
container_class: Optional[Type[Container]] = None
volume_class = Optional[Type[ContainerVolume]]
conf_defaults: Dict[str, Optional[Any]] = {
- 'name_prefix': 'galaxy_',
+ "name_prefix": "galaxy_",
}
option_map: Dict[str, Dict] = {}
publish_port_list_required = False
@@ -190,18 +197,18 @@ class ContainerInterface(metaclass=ABCMeta):
return command
def _guess_kwopt_type(self, val):
- opttype = 'string'
+ opttype = "string"
if isinstance(val, bool):
- opttype = 'boolean'
+ opttype = "boolean"
elif isinstance(val, list):
- opttype = 'list'
+ opttype = "list"
try:
if isinstance(val[0], tuple) and len(val[0]) == 3:
- opttype = 'list_of_kovtrips'
+ opttype = "list_of_kovtrips"
except IndexError:
pass
elif isinstance(val, dict):
- opttype = 'list_of_kvpairs'
+ opttype = "list_of_kvpairs"
return opttype
def _guess_kwopt_flag(self, opt):
@@ -214,34 +221,35 @@ class ContainerInterface(metaclass=ABCMeta):
optdef = self.option_map[opt]
except KeyError:
optdef = {
- 'flag': self._guess_kwopt_flag(opt),
- 'type': self._guess_kwopt_type(val),
+ "flag": self._guess_kwopt_flag(opt),
+ "type": self._guess_kwopt_type(val),
}
- log.warning("option '%s' not in %s.option_map, guessing flag '%s' type '%s'",
- opt, self.__class__.__name__, optdef['flag'], optdef['type'])
- opts.append(getattr(self, f"_stringify_kwopt_{optdef['type']}")(optdef['flag'], val))
- return ' '.join(opts)
+ log.warning(
+ "option '%s' not in %s.option_map, guessing flag '%s' type '%s'",
+ opt,
+ self.__class__.__name__,
+ optdef["flag"],
+ optdef["type"],
+ )
+ opts.append(getattr(self, f"_stringify_kwopt_{optdef['type']}")(optdef["flag"], val))
+ return " ".join(opts)
def _stringify_kwopt_boolean(self, flag, val):
- """
- """
- return f'{flag}={str(val).lower()}'
+ """ """
+ return f"{flag}={str(val).lower()}"
def _stringify_kwopt_string(self, flag, val):
- """
- """
- return f'{flag} {shlex.quote(str(val))}'
+ """ """
+ return f"{flag} {shlex.quote(str(val))}"
def _stringify_kwopt_list(self, flag, val):
- """
- """
+ """ """
if isinstance(val, str):
return self._stringify_kwopt_string(flag, val)
- return ' '.join(f'{flag} {shlex.quote(str(v))}' for v in val)
+ return " ".join(f"{flag} {shlex.quote(str(v))}" for v in val)
def _stringify_kwopt_list_of_kvpairs(self, flag, val):
- """
- """
+ """ """
kwopt_list = []
if isinstance(val, list):
# ['foo=bar', 'baz=quux']
@@ -249,22 +257,21 @@ class ContainerInterface(metaclass=ABCMeta):
else:
# {'foo': 'bar', 'baz': 'quux'}
for k, v in dict(val).items():
- kwopt_list.append(f'{k}={v}')
+ kwopt_list.append(f"{k}={v}")
return self._stringify_kwopt_list(flag, kwopt_list)
def _stringify_kwopt_list_of_kovtrips(self, flag, val):
- """
- """
+ """ """
if isinstance(val, str):
return self._stringify_kwopt_string(flag, val)
kwopt_list = []
for k, o, v in val:
- kwopt_list.append(f'{k}{o}{v}')
+ kwopt_list.append(f"{k}{o}{v}")
return self._stringify_kwopt_list(flag, kwopt_list)
def _run_command(self, command, verbose=False):
if verbose:
- log.debug('running command: [%s]', command)
+ log.debug("running command: [%s]", command)
command_list = self._normalize_command(command)
p = subprocess.Popen(command_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
stdout, stderr = p.communicate()
@@ -279,7 +286,8 @@ class ContainerInterface(metaclass=ABCMeta):
stderr=stderr.strip(),
returncode=p.returncode,
command=command,
- subprocess_command=command_list)
+ subprocess_command=command_list,
+ )
@property
def key(self):
@@ -294,25 +302,19 @@ class ContainerInterface(metaclass=ABCMeta):
def set_kwopts_name(self, kwopts):
if self._name_prefix is not None:
- name = '{prefix}{name}'.format(
- prefix=self._name_prefix,
- name=kwopts.get('name', uuid.uuid4().hex)
- )
- kwopts['name'] = name
+ name = "{prefix}{name}".format(prefix=self._name_prefix, name=kwopts.get("name", uuid.uuid4().hex))
+ kwopts["name"] = name
def validate_config(self):
- """
- """
+ """ """
self._name_prefix = self._conf.name_prefix
@abstractmethod
def run_in_container(self, command, image=None, **kwopts):
- """
- """
+ """ """
class ContainerInterfaceConfig(dict):
-
def __setattr__(self, name, value):
self[name] = value
@@ -345,7 +347,7 @@ def build_container_interfaces(containers_config_file, containers_conf=None):
interface_classes = _get_interface_modules()
interfaces = {}
for k, conf in containers_conf.items():
- container_type = conf.get('type', DEFAULT_CONTAINER_TYPE)
+ container_type = conf.get("type", DEFAULT_CONTAINER_TYPE)
assert container_type in interface_classes, f"unknown container interface type: {container_type}"
interfaces[k] = interface_classes[container_type](conf, k, containers_config_file)
return interfaces
@@ -363,7 +365,7 @@ def parse_containers_config(containers_config_file):
try:
with open(containers_config_file) as fh:
c = yaml.safe_load(fh)
- conf.update(c.get('containers', {}))
+ conf.update(c.get("containers", {}))
except OSError as exc:
if exc.errno == errno.ENOENT:
log.debug("config file '%s' does not exist, running with default config", containers_config_file)
@@ -377,7 +379,10 @@ def _get_interface_modules():
modules = import_submodules(sys.modules[__name__])
for module in modules:
module_names = [getattr(module, _) for _ in dir(module)]
- classes = [_ for _ in module_names if inspect.isclass(_)
- and not _ == ContainerInterface and issubclass(_, ContainerInterface)]
+ classes = [
+ _
+ for _ in module_names
+ if inspect.isclass(_) and not _ == ContainerInterface and issubclass(_, ContainerInterface)
+ ]
interfaces.extend(classes)
return {x.container_type: x for x in interfaces}
diff --git a/lib/galaxy/containers/docker.py b/lib/galaxy/containers/docker.py
index 438483d70f7..3f08f84f053 100644
--- a/lib/galaxy/containers/docker.py
+++ b/lib/galaxy/containers/docker.py
@@ -6,9 +6,17 @@ import logging
import os
import shlex
from functools import partial
-from itertools import cycle, repeat
+from itertools import (
+ cycle,
+ repeat,
+)
from time import sleep
-from typing import Any, Dict, Optional, Type
+from typing import (
+ Any,
+ Dict,
+ Optional,
+ Type,
+)
try:
import docker
@@ -16,24 +24,30 @@ except ImportError:
docker = None # type: ignore[assignment]
try:
- from requests.exceptions import ConnectionError, ReadTimeout
+ from requests.exceptions import (
+ ConnectionError,
+ ReadTimeout,
+ )
except ImportError:
ConnectionError = None # type: ignore[assignment,misc]
ReadTimeout = None # type: ignore[assignment,misc]
-from galaxy.containers import Container, ContainerInterface
+from galaxy.containers import (
+ Container,
+ ContainerInterface,
+)
from galaxy.containers.docker_decorators import (
docker_columns,
- docker_json
+ docker_json,
)
from galaxy.containers.docker_model import (
DockerContainer,
- DockerVolume
+ DockerVolume,
)
from galaxy.exceptions import (
ContainerCLIError,
ContainerImageNotFound,
- ContainerNotFound
+ ContainerNotFound,
)
from galaxy.util.json import safe_dumps_formatted
@@ -45,18 +59,18 @@ class DockerInterface(ContainerInterface):
container_class: Type[Container] = DockerContainer
volume_class = DockerVolume
conf_defaults: Dict[str, Optional[Any]] = {
- 'host': None,
- 'tls': False,
- 'force_tlsverify': False,
- 'auto_remove': True,
- 'image': None,
- 'cpus': None,
- 'memory': None,
+ "host": None,
+ "tls": False,
+ "force_tlsverify": False,
+ "auto_remove": True,
+ "image": None,
+ "cpus": None,
+ "memory": None,
}
# These values are inserted into kwopts for run commands
conf_run_kwopts = (
- 'cpus',
- 'memory',
+ "cpus",
+ "memory",
)
def validate_config(self):
@@ -92,7 +106,7 @@ class DockerInterface(ContainerInterface):
"""
try:
inspect = self.image_inspect(image)
- return inspect['RepoDigests'][0]
+ return inspect["RepoDigests"][0]
except ContainerImageNotFound:
return image
@@ -107,45 +121,45 @@ class DockerInterface(ContainerInterface):
class DockerCLIInterface(DockerInterface):
- container_type = 'docker_cli'
+ container_type = "docker_cli"
conf_defaults: Dict[str, Optional[Any]] = {
- 'command_template': '{executable} {global_kwopts} {subcommand} {args}',
- 'executable': 'docker',
+ "command_template": "{executable} {global_kwopts} {subcommand} {args}",
+ "executable": "docker",
}
option_map = {
# `run` options
- 'environment': {'flag': '--env', 'type': 'list_of_kvpairs'},
- 'volumes': {'flag': '--volume', 'type': 'docker_volumes'},
- 'name': {'flag': '--name', 'type': 'string'},
- 'detach': {'flag': '--detach', 'type': 'boolean'},
- 'publish_all_ports': {'flag': '--publish-all', 'type': 'boolean'},
- 'publish_port_random': {'flag': '--publish', 'type': 'string'},
- 'auto_remove': {'flag': '--rm', 'type': 'boolean'},
- 'cpus': {'flag': '--cpus', 'type': 'string'},
- 'memory': {'flag': '--memory', 'type': 'string'},
+ "environment": {"flag": "--env", "type": "list_of_kvpairs"},
+ "volumes": {"flag": "--volume", "type": "docker_volumes"},
+ "name": {"flag": "--name", "type": "string"},
+ "detach": {"flag": "--detach", "type": "boolean"},
+ "publish_all_ports": {"flag": "--publish-all", "type": "boolean"},
+ "publish_port_random": {"flag": "--publish", "type": "string"},
+ "auto_remove": {"flag": "--rm", "type": "boolean"},
+ "cpus": {"flag": "--cpus", "type": "string"},
+ "memory": {"flag": "--memory", "type": "string"},
}
def validate_config(self):
- log.warning('The `docker_cli` interface is deprecated and will be removed in Galaxy 18.09, please use `docker`')
+ log.warning("The `docker_cli` interface is deprecated and will be removed in Galaxy 18.09, please use `docker`")
super().validate_config()
global_kwopts = []
if self._conf.host:
- global_kwopts.append('--host')
+ global_kwopts.append("--host")
global_kwopts.append(shlex.quote(self._conf.host))
if self._conf.force_tlsverify:
- global_kwopts.append('--tlsverify')
- self._docker_command = self._conf['command_template'].format(
- executable=self._conf['executable'],
- global_kwopts=' '.join(global_kwopts),
- subcommand='{subcommand}',
- args='{args}'
+ global_kwopts.append("--tlsverify")
+ self._docker_command = self._conf["command_template"].format(
+ executable=self._conf["executable"],
+ global_kwopts=" ".join(global_kwopts),
+ subcommand="{subcommand}",
+ args="{args}",
)
def _filter_by_id_or_name(self, id, name):
if id:
- return f'--filter id={id}'
+ return f"--filter id={id}"
elif name:
- return f'--filter name={name}'
+ return f"--filter name={name}"
return None
def _stringify_kwopt_docker_volumes(self, flag, val):
@@ -160,20 +174,20 @@ class DockerCLIInterface(DockerInterface):
for hostvol, guestopts in val.items():
if isinstance(guestopts, str):
# {'/host/vol': '/container/vol'}
- kwopt_list.append(f'{hostvol}:{guestopts}')
+ kwopt_list.append(f"{hostvol}:{guestopts}")
else:
# {'/host/vol': {'bind': '/container/vol'}}
# {'/host/vol': {'bind': '/container/vol', 'mode': 'rw'}}
- mode = guestopts.get('mode', '')
- kwopt_list.append('{vol}:{bind}{mode}'.format(
- vol=hostvol,
- bind=guestopts['bind'],
- mode=f":{mode}" if mode else ''
- ))
+ mode = guestopts.get("mode", "")
+ kwopt_list.append(
+ "{vol}:{bind}{mode}".format(
+ vol=hostvol, bind=guestopts["bind"], mode=f":{mode}" if mode else ""
+ )
+ )
return self._stringify_kwopt_list(flag, kwopt_list)
def _run_docker(self, subcommand, args=None, verbose=False):
- command = self._docker_command.format(subcommand=subcommand, args=args or '')
+ command = self._docker_command.format(subcommand=subcommand, args=args or "")
return self._run_command(command, verbose=verbose)
#
@@ -182,24 +196,24 @@ class DockerCLIInterface(DockerInterface):
@docker_columns
def ps(self, id=None, name=None):
- return self._run_docker(subcommand='ps', args=self._filter_by_id_or_name(id, name))
+ return self._run_docker(subcommand="ps", args=self._filter_by_id_or_name(id, name))
def run(self, command, image=None, **kwopts):
- args = '{kwopts} {image} {command}'.format(
+ args = "{kwopts} {image} {command}".format(
kwopts=self._stringify_kwopts(kwopts),
image=image or self._default_image,
- command=command if command else ''
+ command=command if command else "",
).strip()
- container_id = self._run_docker(subcommand='run', args=args, verbose=True)
+ container_id = self._run_docker(subcommand="run", args=args, verbose=True)
return DockerContainer.from_id(self, container_id)
@docker_json
def inspect(self, container_id):
try:
- return self._run_docker(subcommand='inspect', args=container_id)[0]
+ return self._run_docker(subcommand="inspect", args=container_id)[0]
except (IndexError, ContainerCLIError) as exc:
msg = f"Invalid container id: {container_id}"
- if exc.stdout == '[]' and exc.stderr == f'Error: no such object: {container_id}':
+ if exc.stdout == "[]" and exc.stderr == f"Error: no such object: {container_id}":
log.warning(msg)
return []
else:
@@ -208,10 +222,10 @@ class DockerCLIInterface(DockerInterface):
@docker_json
def image_inspect(self, image):
try:
- return self._run_docker(subcommand='image inspect', args=image)[0]
+ return self._run_docker(subcommand="image inspect", args=image)[0]
except (IndexError, ContainerCLIError) as exc:
msg = f"{image} not pulled, cannot get digest"
- if exc.stdout == '[]' and exc.stderr == f'Error: no such image: {image}':
+ if exc.stdout == "[]" and exc.stderr == f"Error: no such image: {image}":
log.warning(msg, image)
return []
else:
@@ -219,8 +233,7 @@ class DockerCLIInterface(DockerInterface):
class DockerAPIClient:
- """Wraps a ``docker.APIClient`` to catch exceptions.
- """
+ """Wraps a ``docker.APIClient`` to catch exceptions."""
_exception_retry_time = 5
_default_max_tries = 10
@@ -234,7 +247,7 @@ class DockerAPIClient:
if isinstance(f, partial):
f = f.func
try:
- return getattr(f, '__qualname__', f"{f.im_class.__name__}.{f.__name__}")
+ return getattr(f, "__qualname__", f"{f.im_class.__name__}.{f.__name__}")
except AttributeError:
return f.__name__
@@ -253,15 +266,15 @@ class DockerAPIClient:
@staticmethod
def _init_client():
kwargs = DockerAPIClient._client_kwargs.copy()
- if DockerAPIClient._host_iter is not None and 'base_url' not in kwargs:
- kwargs['base_url'] = next(DockerAPIClient._host_iter)
+ if DockerAPIClient._host_iter is not None and "base_url" not in kwargs:
+ kwargs["base_url"] = next(DockerAPIClient._host_iter)
DockerAPIClient._client = docker.APIClient(*DockerAPIClient._client_args, **kwargs)
- log.info('Initialized Docker API client for server: %s', kwargs.get('base_url', 'localhost'))
+ log.info("Initialized Docker API client for server: %s", kwargs.get("base_url", "localhost"))
@staticmethod
def _default_client_handler(fname, *args, **kwargs):
- success_test = kwargs.pop('success_test', None)
- max_tries = kwargs.pop('max_tries', DockerAPIClient._default_max_tries)
+ success_test = kwargs.pop("success_test", None)
+ max_tries = kwargs.pop("max_tries", DockerAPIClient._default_max_tries)
for tries in range(1, max_tries + 1):
retry_time = DockerAPIClient._exception_retry_time
reinit = False
@@ -272,7 +285,7 @@ class DockerAPIClient:
try:
r = f(*args, **kwargs)
if tries > 1:
- log.info('%s() succeeded on attempt %s', qualname, tries)
+ log.info("%s() succeeded on attempt %s", qualname, tries)
return r
except (ConnectionError, docker.errors.APIError, ReadTimeout) as exc:
if isinstance(exc, ConnectionError):
@@ -283,8 +296,9 @@ class DockerAPIClient:
else: # ReadTimeout
reinit = True
retry_time = 0
- log.warning("Caught exception on %s(): %s: %s",
- DockerAPIClient._qualname(f), exc.__class__.__name__, exc)
+ log.warning(
+ "Caught exception on %s(): %s: %s", DockerAPIClient._qualname(f), exc.__class__.__name__, exc
+ )
if reinit:
log.warning("Reinitializing Docker API client due to connection-oriented failure")
DockerAPIClient._init_client()
@@ -299,7 +313,7 @@ class DockerAPIClient:
return r
elif tries >= max_tries:
log.error("Maximum number of attempts (%s) exceeded", max_tries)
- if 'response' in exc and DockerAPIClient._nonfatal_error(exc.response.status_code):
+ if "response" in exc and DockerAPIClient._nonfatal_error(exc.response.status_code):
return None
else:
raise
@@ -309,15 +323,14 @@ class DockerAPIClient:
def __init__(self, *args, **kwargs):
# Only initialize the host iterator once
- host_iter = kwargs.pop('host_iter', None)
+ host_iter = kwargs.pop("host_iter", None)
DockerAPIClient._host_iter = DockerAPIClient._host_iter or host_iter
DockerAPIClient._client_args = args
DockerAPIClient._client_kwargs = kwargs
DockerAPIClient._init_client()
def __getattr__(self, attr):
- """Allow the calling of methods on this class as if it were a docker.APIClient instance.
- """
+ """Allow the calling of methods on this class as if it were a docker.APIClient instance."""
cattr = DockerAPIClient._unwrapped_attr(attr)
if callable(cattr):
return partial(DockerAPIClient._default_client_handler, attr)
@@ -327,16 +340,16 @@ class DockerAPIClient:
class DockerAPIInterface(DockerInterface):
- container_type = 'docker'
+ container_type = "docker"
# 'publish_port_random' and 'volumes' are special cases handled in _create_host_config()
host_config_option_map = {
- 'auto_remove': {},
- 'publish_all_ports': {},
- 'cpus': {'param': 'nano_cpus', 'map': lambda x: int(x * 1000000000)},
- 'memory': {'param': 'mem_limit'},
- 'binds': {},
- 'port_bindings': {},
+ "auto_remove": {},
+ "publish_all_ports": {},
+ "cpus": {"param": "nano_cpus", "map": lambda x: int(x * 1000000000)},
+ "memory": {"param": "mem_limit"},
+ "binds": {},
+ "port_bindings": {},
}
def validate_config(self):
@@ -347,14 +360,13 @@ class DockerAPIInterface(DockerInterface):
@property
def _client(self):
# TODO: add cert options to containers conf
- cert_path = os.environ.get('DOCKER_CERT_PATH') or None
+ cert_path = os.environ.get("DOCKER_CERT_PATH") or None
if not cert_path:
- cert_path = os.path.join(os.path.expanduser('~'), '.docker')
+ cert_path = os.path.join(os.path.expanduser("~"), ".docker")
if self._conf.force_tlsverify or self._conf.tls:
tls_config = docker.tls.TLSConfig(
- client_cert=(os.path.join(cert_path, 'cert.pem'),
- os.path.join(cert_path, 'key.pem')),
- ca_cert=os.path.join(cert_path, 'ca.pem'),
+ client_cert=(os.path.join(cert_path, "cert.pem"), os.path.join(cert_path, "key.pem")),
+ ca_cert=os.path.join(cert_path, "ca.pem"),
verify=self._conf.force_tlsverify,
)
else:
@@ -376,9 +388,9 @@ class DockerAPIInterface(DockerInterface):
@staticmethod
def _filter_by_id_or_name(id, name):
if id:
- return {'id': id}
+ return {"id": id}
elif name:
- return {'name': name}
+ return {"name": name}
return None
@staticmethod
@@ -388,11 +400,11 @@ class DockerAPIInterface(DockerInterface):
See :meth:`_create_docker_api_spec`.
"""
params = []
- if 'param' not in map_spec and 'params' not in map_spec:
+ if "param" not in map_spec and "params" not in map_spec:
params.append(key)
- elif 'param' in map_spec:
- params.append(map_spec['param'])
- params.extend(map_spec.get('params', ()))
+ elif "param" in map_spec:
+ params.append(map_spec["param"])
+ params.extend(map_spec.get("params", ()))
return params
@staticmethod
@@ -403,8 +415,8 @@ class DockerAPIInterface(DockerInterface):
See :meth:`_create_docker_api_spec`.
"""
params = {}
- if 'map' in map_spec:
- value = map_spec['map'](value)
+ if "map" in map_spec:
+ value = map_spec["map"](value)
for param in DockerAPIInterface._kwopt_to_param_names(map_spec, key):
params[param] = value
return params
@@ -475,15 +487,17 @@ class DockerAPIInterface(DockerInterface):
:returns: Instantiated ``spec_class`` object
:rtype: ``type(spec_class)``
"""
+
def _kwopt_to_arg(map_spec, key, value, param=None):
# determines whether the given param is a positional or keyword argument in docker-py and adds it to the
# list of arguments
- if isinstance(map_spec.get('param'), int):
- spec_opts.append((map_spec.get('param'), value))
+ if isinstance(map_spec.get("param"), int):
+ spec_opts.append((map_spec.get("param"), value))
elif param is not None:
spec_kwopts[param] = value
else:
spec_kwopts.update(DockerAPIInterface._kwopt_to_params(map_spec, key, value))
+
# positional arguments
spec_opts = []
# keyword arguments
@@ -491,20 +505,20 @@ class DockerAPIInterface(DockerInterface):
# retrieve the option map for the docker-py object we're creating
option_map = getattr(self, f"{option_map_name}_option_map")
# set defaults
- for key in filter(lambda k: option_map[k].get('default'), option_map.keys()):
+ for key in filter(lambda k: option_map[k].get("default"), option_map.keys()):
map_spec = option_map[key]
- _kwopt_to_arg(map_spec, key, map_spec['default'])
+ _kwopt_to_arg(map_spec, key, map_spec["default"])
# don't allow kwopts that start with _, those are reserved for "child" object params
- for kwopt in filter(lambda k: not k.startswith('_') and k in option_map, list(kwopts.keys())):
+ for kwopt in filter(lambda k: not k.startswith("_") and k in option_map, list(kwopts.keys())):
map_spec = option_map[kwopt]
_v = kwopts.pop(kwopt)
_kwopt_to_arg(map_spec, kwopt, _v)
# find any child objects that need to be created and recurse to create them
- for _sub_k in filter(lambda k: k.startswith('_') and 'spec_class' in option_map[k], option_map.keys()):
+ for _sub_k in filter(lambda k: k.startswith("_") and "spec_class" in option_map[k], option_map.keys()):
map_spec = option_map[_sub_k]
- param = _sub_k.lstrip('_')
- _sub_v = self._create_docker_api_spec(param, map_spec['spec_class'], kwopts)
- if _sub_v is not None or map_spec.get('required') or isinstance(map_spec.get('param'), int):
+ param = _sub_k.lstrip("_")
+ _sub_v = self._create_docker_api_spec(param, map_spec["spec_class"], kwopts)
+ if _sub_v is not None or map_spec.get("required") or isinstance(map_spec.get("param"), int):
_kwopt_to_arg(map_spec, None, _sub_v, param=param)
# sort positional args and make into a flat tuple
if spec_opts:
@@ -541,15 +555,15 @@ class DockerAPIInterface(DockerInterface):
:returns: The return value of `docker.APIClient.create_host_config()`
:rtype: dict
"""
- if 'publish_port_random' in kwopts:
- port = int(kwopts.pop('publish_port_random'))
- kwopts['port_bindings'] = {port: None}
- kwopts['ports'] = [port]
- if 'volumes' in kwopts:
- paths, binds = self._volumes_to_native(kwopts.pop('volumes'))
- kwopts['binds'] = binds
- kwopts['volumes'] = paths
- return self._create_docker_api_spec('host_config', self._client.create_host_config, kwopts)
+ if "publish_port_random" in kwopts:
+ port = int(kwopts.pop("publish_port_random"))
+ kwopts["port_bindings"] = {port: None}
+ kwopts["ports"] = [port]
+ if "volumes" in kwopts:
+ paths, binds = self._volumes_to_native(kwopts.pop("volumes"))
+ kwopts["binds"] = binds
+ kwopts["volumes"] = paths
+ return self._create_docker_api_spec("host_config", self._client.create_host_config, kwopts)
#
# docker subcommands
@@ -565,7 +579,7 @@ class DockerAPIInterface(DockerInterface):
host_config = self._create_host_config(kwopts)
log.debug("Docker container host configuration:\n%s", safe_dumps_formatted(host_config))
log.debug("Docker container creation parameters:\n%s", safe_dumps_formatted(kwopts))
- success_test = partial(self._first, self.ps, name=kwopts['name'], running=False)
+ success_test = partial(self._first, self.ps, name=kwopts["name"], running=False)
# this can raise exceptions, if necessary we could wrap them in a more generic "creation failed" exception class
container = self._client.create_container(
image,
@@ -573,10 +587,10 @@ class DockerAPIInterface(DockerInterface):
host_config=host_config,
success_test=success_test,
max_tries=5,
- **kwopts
+ **kwopts,
)
- container_id = container.get('Id')
- log.debug("Starting container: %s (%s)", kwopts['name'], str(container_id))
+ container_id = container.get("Id")
+ log.debug("Starting container: %s (%s)", kwopts["name"], str(container_id))
# start can safely be run more than once
self._client.start(container=container_id)
return DockerContainer.from_id(self, container_id)
diff --git a/lib/galaxy/containers/docker_decorators.py b/lib/galaxy/containers/docker_decorators.py
index ee5e9fa6fa2..a3e28de4860 100644
--- a/lib/galaxy/containers/docker_decorators.py
+++ b/lib/galaxy/containers/docker_decorators.py
@@ -28,22 +28,23 @@ def docker_columns(f):
if not output:
return parsed
for i, c in enumerate(header):
- if c != ' ' and spacect > 1:
+ if c != " " and spacect > 1:
colidx += 1
colstarts.append(i)
spacect = 0
- elif c == ' ':
+ elif c == " ":
spacect += 1
colstarts.append(None)
colheadings = []
for i in range(0, len(colstarts) - 1):
- colheadings.append(header[colstarts[i]:colstarts[i + 1]].strip())
+ colheadings.append(header[colstarts[i] : colstarts[i + 1]].strip())
for line in output[1:]:
row = {}
for i, key in enumerate(colheadings):
- row[key] = line[colstarts[i]:colstarts[i + 1]].strip()
+ row[key] = line[colstarts[i] : colstarts[i + 1]].strip()
parsed.append(row)
return parsed
+
return parse_docker_column_output
@@ -51,4 +52,5 @@ def docker_json(f):
@wraps(f)
def json_loads(*args, **kwargs):
return json.loads(f(*args, **kwargs))
+
return json_loads
diff --git a/lib/galaxy/containers/docker_model.py b/lib/galaxy/containers/docker_model.py
index 2d409e258db..a5f2656c46d 100644
--- a/lib/galaxy/containers/docker_model.py
+++ b/lib/galaxy/containers/docker_model.py
@@ -9,21 +9,21 @@ try:
import docker
except ImportError:
from galaxy.util.bunch import Bunch
+
docker = Bunch(errors=Bunch(NotFound=None))
from galaxy.containers import (
Container,
ContainerPort,
- ContainerVolume
+ ContainerVolume,
)
from galaxy.util import (
pretty_print_time_interval,
unicodify,
)
-
-CPUS_LABEL = '_galaxy_cpus'
-IMAGE_LABEL = '_galaxy_image'
+CPUS_LABEL = "_galaxy_cpus"
+IMAGE_LABEL = "_galaxy_image"
CPUS_CONSTRAINT = f"node.labels.{CPUS_LABEL}"
IMAGE_CONSTRAINT = f"node.labels.{IMAGE_LABEL}"
@@ -31,7 +31,6 @@ log = logging.getLogger(__name__)
class DockerAttributeContainer:
-
def __init__(self, members=None):
if members is None:
members = set()
@@ -47,7 +46,7 @@ class DockerAttributeContainer:
return hash(tuple(sorted(repr(x) for x in self._members)))
def __str__(self):
- return ', '.join(str(x) for x in self._members) or 'None'
+ return ", ".join(str(x) for x in self._members) or "None"
def __iter__(self):
return iter(self._members)
@@ -114,11 +113,10 @@ class DockerVolume(ContainerVolume):
def to_native(self):
host_path = self.host_path or self.path
- return (self.path, {host_path: {'bind': self.path, 'mode': self.mode}})
+ return (self.path, {host_path: {"bind": self.path, "mode": self.mode}})
class DockerContainer(Container):
-
def __init__(self, interface, id, name=None, inspect=None):
super().__init__(interface, id, name=name)
self._inspect = inspect
@@ -126,7 +124,7 @@ class DockerContainer(Container):
@classmethod
def from_id(cls, interface, id):
inspect = interface.inspect(id)
- return cls(interface, id, name=inspect['Name'], inspect=inspect)
+ return cls(interface, id, name=inspect["Name"], inspect=inspect)
@property
def ports(self):
@@ -141,30 +139,36 @@ class DockerContainer(Container):
# ]
rval = []
try:
- port_mappings = self.inspect['NetworkSettings']['Ports']
+ port_mappings = self.inspect["NetworkSettings"]["Ports"]
except KeyError:
- log.warning("Failed to get ports for container %s from `docker inspect` output at "
- "['NetworkSettings']['Ports']: %s: %s", self.id, exc_info=True)
+ log.warning(
+ "Failed to get ports for container %s from `docker inspect` output at "
+ "['NetworkSettings']['Ports']: %s: %s",
+ self.id,
+ exc_info=True,
+ )
return None
for port_name in port_mappings:
for binding in port_mappings[port_name]:
- rval.append(ContainerPort(
- int(port_name.split('/')[0]),
- port_name.split('/')[1],
- self.address,
- int(binding['HostPort']),
- ))
+ rval.append(
+ ContainerPort(
+ int(port_name.split("/")[0]),
+ port_name.split("/")[1],
+ self.address,
+ int(binding["HostPort"]),
+ )
+ )
return rval
@property
def address(self):
- if self._interface.host and self._interface.host.startswith('tcp://'):
- return self._interface.host.replace('tcp://', '').split(':', 1)[0]
+ if self._interface.host and self._interface.host.startswith("tcp://"):
+ return self._interface.host.replace("tcp://", "").split(":", 1)[0]
else:
- return 'localhost'
+ return "localhost"
def is_ready(self):
- return self.inspect['State']['Running']
+ return self.inspect["State"]["Running"]
def __eq__(self, other):
return self._id == other.id
@@ -183,7 +187,6 @@ class DockerContainer(Container):
class DockerService(Container):
-
def __init__(self, interface, id, name=None, image=None, inspect=None):
super().__init__(interface, id, name=name)
self._image = image
@@ -191,15 +194,15 @@ class DockerService(Container):
self._env = {}
self._tasks = []
if inspect:
- self._name = name or inspect['Spec']['Name']
- self._image = image or inspect['Spec']['TaskTemplate']['ContainerSpec']['Image']
+ self._name = name or inspect["Spec"]["Name"]
+ self._image = image or inspect["Spec"]["TaskTemplate"]["ContainerSpec"]["Image"]
@classmethod
def from_cli(cls, interface, s, task_list):
- service = cls(interface, s['ID'], name=s['NAME'], image=s['IMAGE'])
+ service = cls(interface, s["ID"], name=s["NAME"], image=s["IMAGE"])
for task_dict in task_list:
- if task_dict['NAME'].strip().startswith(r'\_'):
- continue # historical task
+ if task_dict["NAME"].strip().startswith(r"\_"):
+ continue # historical task
service.task_add(DockerTask.from_cli(interface, task_dict, service=service))
return service
@@ -225,29 +228,35 @@ class DockerService(Container):
# ]
rval = []
try:
- port_mappings = self.inspect['Endpoint']['Ports']
+ port_mappings = self.inspect["Endpoint"]["Ports"]
except (IndexError, KeyError):
- log.warning("Failed to get ports for container %s from `docker service inspect` output at "
- "['Endpoint']['Ports']: %s: %s", self.id, exc_info=True)
+ log.warning(
+ "Failed to get ports for container %s from `docker service inspect` output at "
+ "['Endpoint']['Ports']: %s: %s",
+ self.id,
+ exc_info=True,
+ )
return None
for binding in port_mappings:
- rval.append(ContainerPort(
- binding['TargetPort'],
- binding['Protocol'],
- self.address, # use the routing mesh
- binding['PublishedPort']
- ))
+ rval.append(
+ ContainerPort(
+ binding["TargetPort"],
+ binding["Protocol"],
+ self.address, # use the routing mesh
+ binding["PublishedPort"],
+ )
+ )
return rval
@property
def address(self):
- if self._interface.host and self._interface.host.startswith('tcp://'):
- return self._interface.host.replace('tcp://', '').split(':', 1)[0]
+ if self._interface.host and self._interface.host.startswith("tcp://"):
+ return self._interface.host.replace("tcp://", "").split(":", 1)[0]
else:
- return 'localhost'
+ return "localhost"
def is_ready(self):
- return self.in_state('Running', 'Running')
+ return self.in_state("Running", "Running")
def __eq__(self, other):
return self._id == other.id
@@ -278,7 +287,7 @@ class DockerService(Container):
state = None
for task in self.tasks:
state = task.state
- if task.desired_state == 'running':
+ if task.desired_state == "running":
break
return state
@@ -286,28 +295,26 @@ class DockerService(Container):
def env(self):
if not self._env:
try:
- for env_str in self.inspect['Spec']['TaskTemplate']['ContainerSpec']['Env']:
+ for env_str in self.inspect["Spec"]["TaskTemplate"]["ContainerSpec"]["Env"]:
try:
- self._env.update([env_str.split('=', 1)])
+ self._env.update([env_str.split("=", 1)])
except ValueError:
self._env[env_str] = None
except KeyError as exc:
- log.debug('Cannot retrieve container environment: KeyError: %s', unicodify(exc))
+ log.debug("Cannot retrieve container environment: KeyError: %s", unicodify(exc))
return self._env
@property
def terminal(self):
- """Same caveats as :meth:`state`.
- """
+ """Same caveats as :meth:`state`."""
for task in self.tasks:
- if task.desired_state == 'running':
+ if task.desired_state == "running":
return False
return True
@property
def node(self):
- """Same caveats as :meth:`state`.
- """
+ """Same caveats as :meth:`state`."""
for task in self.tasks:
if task.node is not None:
return task.node
@@ -316,13 +323,13 @@ class DockerService(Container):
@property
def image(self):
if self._image is None:
- self._image = self.inspect['Spec']['TaskTemplate']['ContainerSpec']['Image']
+ self._image = self.inspect["Spec"]["TaskTemplate"]["ContainerSpec"]["Image"]
return self._image
@property
def cpus(self):
try:
- cpus = self.inspect['Spec']['TaskTemplate']['Resources']['Limits']['NanoCPUs'] / 1000000000.0
+ cpus = self.inspect["Spec"]["TaskTemplate"]["Resources"]["Limits"]["NanoCPUs"] / 1000000000.0
if cpus == int(cpus):
cpus = int(cpus)
return cpus
@@ -331,13 +338,12 @@ class DockerService(Container):
@property
def constraints(self):
- constraints = self.inspect['Spec']['TaskTemplate']['Placement'].get('Constraints', [])
+ constraints = self.inspect["Spec"]["TaskTemplate"]["Placement"].get("Constraints", [])
return DockerServiceConstraints.from_constraint_string_list(constraints)
@property
def tasks(self):
- """A list of *all* tasks, including terminal ones.
- """
+ """A list of *all* tasks, including terminal ones."""
if not self._tasks:
self._tasks = []
for task in self._interface.service_tasks(self):
@@ -346,45 +352,40 @@ class DockerService(Container):
@property
def task_count(self):
- """A count of *all* tasks, including terminal ones.
- """
+ """A count of *all* tasks, including terminal ones."""
return len(self.tasks)
- def in_state(self, desired, current, tasks='any'):
- """Indicate if one of this service's tasks matches the desired state.
- """
+ def in_state(self, desired, current, tasks="any"):
+ """Indicate if one of this service's tasks matches the desired state."""
for task in self.tasks:
if task.in_state(desired, current):
- if tasks == 'any':
+ if tasks == "any":
# at least 1 task in desired state
return True
- elif tasks == 'all':
+ elif tasks == "all":
# at least 1 task not in desired state
return False
else:
- return False if tasks == 'any' else True
+ return False if tasks == "any" else True
def constraint_add(self, name, op, value):
self._interface.service_constraint_add(self.id, name, op, value)
def set_cpus(self):
- self.constraint_add(CPUS_LABEL, '==', self.cpus)
+ self.constraint_add(CPUS_LABEL, "==", self.cpus)
def set_image(self):
- self.constraint_add(IMAGE_LABEL, '==', self.image)
+ self.constraint_add(IMAGE_LABEL, "==", self.image)
class DockerServiceConstraint:
-
def __init__(self, name=None, op=None, value=None):
self._name = name
self._op = op
self._value = value
def __eq__(self, other):
- return self._name == other._name and \
- self._op == other._op and \
- self._value == other._value
+ return self._name == other._name and self._op == other._op and self._value == other._value
def __ne__(self, other):
return not self.__eq__(other)
@@ -393,20 +394,20 @@ class DockerServiceConstraint:
return hash((self._name, self._op, self._value))
def __repr__(self):
- return f'{self.__class__.__name__}({self._name}{self._op}{self._value})'
+ return f"{self.__class__.__name__}({self._name}{self._op}{self._value})"
def __str__(self):
- return f'{self._name}{self._op}{self._value}'
+ return f"{self._name}{self._op}{self._value}"
@staticmethod
def split_constraint_string(constraint_str):
- constraint = (constraint_str, '', '')
- for op in '==', '!=':
+ constraint = (constraint_str, "", "")
+ for op in "==", "!=":
t = constraint_str.partition(op)
if len(t[0]) < len(constraint[0]):
constraint = t
if constraint[0] == constraint_str:
- raise Exception(f'Unable to parse constraint string: {constraint_str}')
+ raise Exception(f"Unable to parse constraint string: {constraint_str}")
return [x.strip() for x in constraint]
@classmethod
@@ -428,10 +429,7 @@ class DockerServiceConstraint:
@property
def label(self):
- return DockerNodeLabel(
- name=self.name.replace('node.labels.', ''),
- value=self.value
- )
+ return DockerNodeLabel(name=self.name.replace("node.labels.", ""), value=self.value)
class DockerServiceConstraints(DockerAttributeContainer):
@@ -451,9 +449,7 @@ class DockerServiceConstraints(DockerAttributeContainer):
class DockerNode:
-
- def __init__(self, interface, id=None, name=None, status=None,
- availability=None, manager=False, inspect=None):
+ def __init__(self, interface, id=None, name=None, status=None, availability=None, manager=False, inspect=None):
self._interface = interface
self._id = id
self._name = name
@@ -462,16 +458,22 @@ class DockerNode:
self._manager = manager
self._inspect = inspect
if inspect:
- self._name = name or inspect['Description']['Hostname']
- self._status = status or inspect['Status']['State']
- self._availability = inspect['Spec']['Availability']
- self._manager = manager or inspect['Spec']['Role'] == 'manager'
+ self._name = name or inspect["Description"]["Hostname"]
+ self._status = status or inspect["Status"]["State"]
+ self._availability = inspect["Spec"]["Availability"]
+ self._manager = manager or inspect["Spec"]["Role"] == "manager"
self._tasks = []
@classmethod
def from_cli(cls, interface, n, task_list):
- node = cls(interface, id=n['ID'], name=n['HOSTNAME'], status=n['STATUS'],
- availability=n['AVAILABILITY'], manager=True if n['MANAGER STATUS'] else False)
+ node = cls(
+ interface,
+ id=n["ID"],
+ name=n["HOSTNAME"],
+ status=n["STATUS"],
+ availability=n["AVAILABILITY"],
+ manager=True if n["MANAGER STATUS"] else False,
+ )
for task_dict in task_list:
node.task_add(DockerTask.from_cli(interface, task_dict, node=node))
return node
@@ -498,7 +500,7 @@ class DockerNode:
@property
def version(self):
# this changes on update so don't cache
- return self._interface.node_inspect(self._id or self._name)['Version']['Index']
+ return self._interface.node_inspect(self._id or self._name)["Version"]["Index"]
@property
def inspect(self):
@@ -508,15 +510,15 @@ class DockerNode:
@property
def state(self):
- return (f'{self._status}-{self._availability}').lower()
+ return (f"{self._status}-{self._availability}").lower()
@property
def cpus(self):
- return self.inspect['Description']['Resources']['NanoCPUs'] / 1000000000
+ return self.inspect["Description"]["Resources"]["NanoCPUs"] / 1000000000
@property
def labels(self):
- labels = self.inspect['Spec'].get('Labels', {}) or {}
+ labels = self.inspect["Spec"].get("Labels", {}) or {}
return DockerNodeLabels.from_label_dictionary(labels)
def label_add(self, label, value):
@@ -534,14 +536,13 @@ class DockerNode:
self.label_add(label.name, label.value)
def _constraints_to_label_args(self, constraints):
- constraints = filter(lambda x: x.name.startswith('node.labels.') and x.op == '==', constraints)
- labels = map(lambda x: DockerNodeLabel(name=x.name.replace('node.labels.', '', 1), value=x.value), constraints)
+ constraints = filter(lambda x: x.name.startswith("node.labels.") and x.op == "==", constraints)
+ labels = map(lambda x: DockerNodeLabel(name=x.name.replace("node.labels.", "", 1), value=x.value), constraints)
return labels
@property
def tasks(self):
- """A list of *all* tasks, including terminal ones.
- """
+ """A list of *all* tasks, including terminal ones."""
if not self._tasks:
self._tasks = []
for task in self._interface.node_tasks(self):
@@ -560,15 +561,14 @@ class DockerNode:
@property
def task_count(self):
- """A count of *all* tasks, including terminal ones.
- """
+ """A count of *all* tasks, including terminal ones."""
return len(self.tasks)
def in_state(self, status, availability):
return self._status.lower() == status.lower() and self._availability.lower() == availability.lower()
def is_ok(self):
- return self.in_state('Ready', 'Active')
+ return self.in_state("Ready", "Active")
def is_managed(self):
return not self._manager
@@ -577,18 +577,16 @@ class DockerNode:
return not self._manager and self.is_ok() and self.task_count == 0
def drain(self):
- self._interface.node_update(self.id, availability='drain')
+ self._interface.node_update(self.id, availability="drain")
class DockerNodeLabel:
-
def __init__(self, name=None, value=None):
self._name = name
self._value = value
def __eq__(self, other):
- return self._name == other._name and \
- self._value == other._value
+ return self._name == other._name and self._value == other._value
def __ne__(self, other):
return not self.__eq__(other)
@@ -597,10 +595,10 @@ class DockerNodeLabel:
return hash((self._name, self._value))
def __repr__(self):
- return f'{self.__class__.__name__}({self._name}: {self._value})'
+ return f"{self.__class__.__name__}({self._name}: {self._value})"
def __str__(self):
- return f'{self._name}: {self._value}'
+ return f"{self._name}: {self._value}"
@property
def name(self):
@@ -612,15 +610,11 @@ class DockerNodeLabel:
@property
def constraint_string(self):
- return f'node.labels.{self.name}=={self.value}'
+ return f"node.labels.{self.name}=={self.value}"
@property
def constraint(self):
- return DockerServiceConstraint(
- name=f'node.labels.{self.name}',
- op='==',
- value=self.value
- )
+ return DockerServiceConstraint(name=f"node.labels.{self.name}", op="==", value=self.value)
class DockerNodeLabels(DockerAttributeContainer):
@@ -643,15 +637,26 @@ class DockerTask:
# these are the possible *current* state terminal states
terminal_states = (
- 'shutdown', # this is normally only a desired state but I've seen a task with it as current as well
- 'complete',
- 'failed',
- 'rejected',
- 'orphaned',
+ "shutdown", # this is normally only a desired state but I've seen a task with it as current as well
+ "complete",
+ "failed",
+ "rejected",
+ "orphaned",
)
- def __init__(self, interface, id=None, name=None, image=None, desired_state=None,
- state=None, error=None, ports=None, service=None, node=None):
+ def __init__(
+ self,
+ interface,
+ id=None,
+ name=None,
+ image=None,
+ desired_state=None,
+ state=None,
+ error=None,
+ ports=None,
+ service=None,
+ node=None,
+ ):
self._interface = interface
self._id = id
self._name = name
@@ -666,23 +671,41 @@ class DockerTask:
@classmethod
def from_cli(cls, interface, t, service=None, node=None):
- state = t['CURRENT STATE'].split()[0]
- return cls(interface, id=t['ID'], name=t['NAME'], image=t['IMAGE'],
- desired_state=t['DESIRED STATE'], state=state, error=t['ERROR'],
- ports=t['PORTS'], service=service, node=node)
+ state = t["CURRENT STATE"].split()[0]
+ return cls(
+ interface,
+ id=t["ID"],
+ name=t["NAME"],
+ image=t["IMAGE"],
+ desired_state=t["DESIRED STATE"],
+ state=state,
+ error=t["ERROR"],
+ ports=t["PORTS"],
+ service=service,
+ node=node,
+ )
@classmethod
def from_api(cls, interface, t, service=None, node=None):
- service = service or interface.service(id=t.get('ServiceID'))
- node = node or interface.node(id=t.get('NodeID'))
+ service = service or interface.service(id=t.get("ServiceID"))
+ node = node or interface.node(id=t.get("NodeID"))
if service:
name = f"{service.name}.{str(t['Slot'])}"
else:
- name = t['ID']
- image = t['Spec']['ContainerSpec']['Image'].split('@', 1)[0], # remove pin
- return cls(interface, id=t['ID'], name=name, image=image, desired_state=t['DesiredState'],
- state=t['Status']['State'], ports=t['Status']['PortStatus'], error=t['Status']['Message'],
- service=service, node=node)
+ name = t["ID"]
+ image = (t["Spec"]["ContainerSpec"]["Image"].split("@", 1)[0],) # remove pin
+ return cls(
+ interface,
+ id=t["ID"],
+ name=name,
+ image=image,
+ desired_state=t["DesiredState"],
+ state=t["Status"]["State"],
+ ports=t["Status"]["PortStatus"],
+ error=t["Status"]["Message"],
+ service=service,
+ node=node,
+ )
@property
def id(self):
@@ -700,15 +723,16 @@ class DockerTask:
except docker.errors.NotFound:
# This shouldn't be possible, appears to be some kind of Swarm bug (the node claims to have a task that
# does not actually exist anymore, nor does its service exist).
- log.error('Task could not be inspected because Docker claims it does not exist: %s (%s)',
- self.name, self.id)
+ log.error(
+ "Task could not be inspected because Docker claims it does not exist: %s (%s)", self.name, self.id
+ )
return None
return self._inspect
@property
def slot(self):
try:
- return self.inspect['Slot']
+ return self.inspect["Slot"]
except TypeError:
return None
@@ -716,7 +740,7 @@ class DockerTask:
def node(self):
if not self._node:
try:
- self._node = self._interface.node(id=self.inspect['NodeID'])
+ self._node = self._interface.node(id=self.inspect["NodeID"])
except TypeError:
return None
return self._node
@@ -725,7 +749,7 @@ class DockerTask:
def service(self):
if not self._service:
try:
- self._service = self._interface.service(id=self.inspect['ServiceID'])
+ self._service = self._interface.service(id=self.inspect["ServiceID"])
except TypeError:
return None
return self._service
@@ -733,7 +757,7 @@ class DockerTask:
@property
def cpus(self):
try:
- cpus = self.inspect['Spec']['Resources']['Reservations']['NanoCPUs'] / 1000000000.0
+ cpus = self.inspect["Spec"]["Resources"]["Reservations"]["NanoCPUs"] / 1000000000.0
if cpus == int(cpus):
cpus = int(cpus)
return cpus
@@ -744,7 +768,7 @@ class DockerTask:
@property
def state(self):
- return (f'{self._desired_state}-{self._state}').lower()
+ return (f"{self._desired_state}-{self._state}").lower()
@property
def current_state(self):
@@ -758,10 +782,10 @@ class DockerTask:
def current_state_time(self):
# Docker API returns a stamp w/ higher second precision than Python takes
try:
- stamp = self.inspect['Status']['Timestamp']
+ stamp = self.inspect["Status"]["Timestamp"]
except TypeError:
return None
- return pretty_print_time_interval(time=stamp[:stamp.index('.') + 7], precise=True, utc=stamp[-1] == 'Z')
+ return pretty_print_time_interval(time=stamp[: stamp.index(".") + 7], precise=True, utc=stamp[-1] == "Z")
@property
def desired_state(self):
@@ -773,7 +797,7 @@ class DockerTask:
@property
def terminal(self):
- return self.desired_state == 'shutdown' and self.current_state in self.terminal_states
+ return self.desired_state == "shutdown" and self.current_state in self.terminal_states
def in_state(self, desired, current):
return self.desired_state == desired.lower() and self.current_state == current.lower()
diff --git a/lib/galaxy/containers/docker_swarm.py b/lib/galaxy/containers/docker_swarm.py
index 6ac26c547df..840215950f9 100644
--- a/lib/galaxy/containers/docker_swarm.py
+++ b/lib/galaxy/containers/docker_swarm.py
@@ -6,31 +6,41 @@ import logging
import os.path
import subprocess
from functools import partial
-from typing import Any, Dict, Optional
+from typing import (
+ Any,
+ Dict,
+ Optional,
+)
try:
import docker.types
except ImportError:
from galaxy.util.bunch import Bunch
- docker = Bunch(types=Bunch(
- ContainerSpec=None,
- RestartPolicy=None,
- Resources=None,
- Placement=None,
- ))
+
+ docker = Bunch(
+ types=Bunch(
+ ContainerSpec=None,
+ RestartPolicy=None,
+ Resources=None,
+ Placement=None,
+ )
+ )
from galaxy.containers.docker import (
DockerAPIInterface,
DockerCLIInterface,
- DockerInterface
+ DockerInterface,
+)
+from galaxy.containers.docker_decorators import (
+ docker_columns,
+ docker_json,
)
-from galaxy.containers.docker_decorators import docker_columns, docker_json
from galaxy.containers.docker_model import (
CPUS_CONSTRAINT,
DockerNode,
DockerService,
DockerTask,
- IMAGE_CONSTRAINT
+ IMAGE_CONSTRAINT,
)
from galaxy.exceptions import ContainerRunError
from galaxy.util import unicodify
@@ -40,25 +50,22 @@ log = logging.getLogger(__name__)
SWARM_MANAGER_PATH = os.path.abspath(
os.path.join(
- os.path.dirname(__file__),
- os.path.pardir,
- os.path.pardir,
- os.path.pardir,
- 'scripts',
- 'docker_swarm_manager.py'))
+ os.path.dirname(__file__), os.path.pardir, os.path.pardir, os.path.pardir, "scripts", "docker_swarm_manager.py"
+ )
+)
class DockerSwarmInterface(DockerInterface):
container_class = DockerService
conf_defaults: Dict[str, Optional[Any]] = {
- 'ignore_volumes': False,
- 'node_prefix': None,
- 'service_create_image_constraint': False,
- 'service_create_cpus_constraint': False,
- 'resolve_image_digest': False,
- 'managed': True,
- 'manager_autostart': True,
+ "ignore_volumes": False,
+ "node_prefix": None,
+ "service_create_image_constraint": False,
+ "service_create_cpus_constraint": False,
+ "resolve_image_digest": False,
+ "managed": True,
+ "manager_autostart": True,
}
publish_port_list_required = True
supports_volumes = False
@@ -68,48 +75,45 @@ class DockerSwarmInterface(DockerInterface):
self._node_prefix = self._conf.node_prefix
def run_in_container(self, command, image=None, **kwopts):
- """Run a service like a detached container
- """
- kwopts['replicas'] = 1
- kwopts['restart_condition'] = 'none'
- if kwopts.get('publish_all_ports', False):
+ """Run a service like a detached container"""
+ kwopts["replicas"] = 1
+ kwopts["restart_condition"] = "none"
+ if kwopts.get("publish_all_ports", False):
# not supported for services
# TODO: inspect image (or query registry if possible) for port list
- if kwopts.get('publish_port_random', False) or kwopts.get('ports', False):
+ if kwopts.get("publish_port_random", False) or kwopts.get("ports", False):
# assume this covers for publish_all_ports
- del kwopts['publish_all_ports']
+ del kwopts["publish_all_ports"]
else:
raise ContainerRunError(
"Publishing all ports is not supported in Docker swarm"
" mode, use `publish_port_random` or `ports`",
image=image,
- command=command
+ command=command,
)
- if not kwopts.get('detach', True):
+ if not kwopts.get("detach", True):
raise ContainerRunError(
- "Running attached containers is not supported in Docker swarm mode",
- image=image,
- command=command
+ "Running attached containers is not supported in Docker swarm mode", image=image, command=command
)
- elif kwopts.get('detach', None):
- del kwopts['detach']
- if kwopts.get('volumes', None):
+ elif kwopts.get("detach", None):
+ del kwopts["detach"]
+ if kwopts.get("volumes", None):
if self._conf.ignore_volumes:
log.warning(
"'volumes' kwopt is set and not supported in Docker swarm "
"mode, volumes will not be passed (set 'ignore_volumes: "
- "False' in containers config to fail instead): %s" % kwopts['volumes']
+ "False' in containers config to fail instead): %s" % kwopts["volumes"]
)
else:
raise ContainerRunError(
"'volumes' kwopt is set and not supported in Docker swarm "
"mode (set 'ignore_volumes: True' in containers config to "
- "warn instead): %s" % kwopts['volumes'],
+ "warn instead): %s" % kwopts["volumes"],
image=image,
- command=command
+ command=command,
)
# ensure the volumes key is removed from kwopts
- kwopts.pop('volumes', None)
+ kwopts.pop("volumes", None)
service = self.service_create(command, image=image, **kwopts)
self._run_swarm_manager()
return service
@@ -122,10 +126,18 @@ class DockerSwarmInterface(DockerInterface):
if self._conf.managed and self._conf.manager_autostart:
try:
# sys.exectuable would be preferable to using $PATH, but sys.executable is probably uwsgi
- subprocess.check_call(['python', SWARM_MANAGER_PATH, '--containers-config-file',
- self.containers_config_file, '--swarm', self.key])
+ subprocess.check_call(
+ [
+ "python",
+ SWARM_MANAGER_PATH,
+ "--containers-config-file",
+ self.containers_config_file,
+ "--swarm",
+ self.key,
+ ]
+ )
except subprocess.CalledProcessError as exc:
- log.error('Failed to launch swarm manager: %s', unicodify(exc))
+ log.error("Failed to launch swarm manager: %s", unicodify(exc))
def _get_image(self, image):
"""Get the image string, either from the argument, or from the
@@ -143,7 +155,9 @@ class DockerSwarmInterface(DockerInterface):
"""
if not image:
image = self._conf.image
- assert image is not None, "No image supplied as parameter and no image set as default in config, cannot create service"
+ assert (
+ image is not None
+ ), "No image supplied as parameter and no image set as default in config, cannot create service"
if self._conf.resolve_image_digest:
image = self.image_repodigest(image)
return image
@@ -163,7 +177,7 @@ class DockerSwarmInterface(DockerInterface):
def services(self, id=None, name=None):
for service_dict in self.service_ls(id=id, name=name):
- service_id = service_dict['ID']
+ service_id = service_dict["ID"]
service = DockerService(self, service_id, inspect=service_dict)
if service.name.startswith(self._name_prefix):
yield service
@@ -174,7 +188,7 @@ class DockerSwarmInterface(DockerInterface):
except StopIteration:
return None
- def services_in_state(self, desired, current, tasks='any'):
+ def services_in_state(self, desired, current, tasks="any"):
for service in self.services():
if service.in_state(desired, current, tasks=tasks):
yield service
@@ -185,7 +199,7 @@ class DockerSwarmInterface(DockerInterface):
def nodes(self, id=None, name=None):
for node_dict in self.node_ls(id=id, name=name):
- node_id = node_dict['ID']
+ node_id = node_dict["ID"]
node = DockerNode(self, node_id, inspect=node_dict)
if self._node_prefix and not node.name.startswith(self._node_prefix):
continue
@@ -211,22 +225,22 @@ class DockerSwarmInterface(DockerInterface):
#
def services_waiting(self):
- return self.services_in_state('Running', 'Pending')
+ return self.services_in_state("Running", "Pending")
def services_waiting_by_constraints(self):
- return self._objects_by_attribute(self.services_waiting(), 'constraints')
+ return self._objects_by_attribute(self.services_waiting(), "constraints")
def services_completed(self):
- return self.services_in_state('Shutdown', 'Complete', tasks='all')
+ return self.services_in_state("Shutdown", "Complete", tasks="all")
def services_terminal(self):
return [s for s in self.services() if s.terminal]
def nodes_active(self):
- return self.nodes_in_state('Ready', 'Active')
+ return self.nodes_in_state("Ready", "Active")
def nodes_active_by_constraints(self):
- return self._objects_by_attribute(self.nodes_active(), 'labels_as_constraints')
+ return self._objects_by_attribute(self.nodes_active(), "labels_as_constraints")
#
# operations
@@ -239,7 +253,12 @@ class DockerSwarmInterface(DockerInterface):
cleaned_service_ids.extend(self.service_rm([x.id for x in completed_services]))
terminal_services = list(self.services_terminal())
for service in terminal_services:
- log.warning('cleaned service in abnormal terminal state: %s (%s). state: %s', service.name, service.id, service.state)
+ log.warning(
+ "cleaned service in abnormal terminal state: %s (%s). state: %s",
+ service.name,
+ service.id,
+ service.state,
+ )
if terminal_services:
cleaned_service_ids.extend(self.service_rm([x.id for x in terminal_services]))
return filter(lambda x: x.id in cleaned_service_ids, completed_services + terminal_services)
@@ -247,23 +266,23 @@ class DockerSwarmInterface(DockerInterface):
class DockerSwarmCLIInterface(DockerSwarmInterface, DockerCLIInterface):
- container_type = 'docker_swarm_cli'
+ container_type = "docker_swarm_cli"
option_map = {
# `service create` options
- 'constraint': {'flag': '--constraint', 'type': 'list_of_kovtrips'},
- 'replicas': {'flag': '--replicas', 'type': 'string'},
- 'restart_condition': {'flag': '--restart-condition', 'type': 'string'},
- 'environment': {'flag': '--env', 'type': 'list_of_kvpairs'},
- 'name': {'flag': '--name', 'type': 'string'},
- 'publish_port_random': {'flag': '--publish', 'type': 'string'},
- 'cpu_limit': {'flag': '--limit-cpu', 'type': 'string'},
- 'mem_limit': {'flag': '--limit-memory', 'type': 'string'},
- 'cpu_reservation': {'flag': '--reserve-cpu', 'type': 'string'},
- 'mem_reservation': {'flag': '--reserve-memory', 'type': 'string'},
+ "constraint": {"flag": "--constraint", "type": "list_of_kovtrips"},
+ "replicas": {"flag": "--replicas", "type": "string"},
+ "restart_condition": {"flag": "--restart-condition", "type": "string"},
+ "environment": {"flag": "--env", "type": "list_of_kvpairs"},
+ "name": {"flag": "--name", "type": "string"},
+ "publish_port_random": {"flag": "--publish", "type": "string"},
+ "cpu_limit": {"flag": "--limit-cpu", "type": "string"},
+ "mem_limit": {"flag": "--limit-memory", "type": "string"},
+ "cpu_reservation": {"flag": "--reserve-cpu", "type": "string"},
+ "mem_reservation": {"flag": "--reserve-memory", "type": "string"},
# `service update` options
- 'label_add': {'flag': '--label-add', 'type': 'list_of_kvpairs'},
- 'label_rm': {'flag': '--label-rm', 'type': 'list_of_kvpairs'},
- 'availability': {'flag': '--availability', 'type': 'string'},
+ "label_add": {"flag": "--label-add", "type": "list_of_kvpairs"},
+ "label_rm": {"flag": "--label-rm", "type": "list_of_kvpairs"},
+ "availability": {"flag": "--availability", "type": "string"},
}
#
@@ -272,8 +291,8 @@ class DockerSwarmCLIInterface(DockerSwarmInterface, DockerCLIInterface):
def services(self, id=None, name=None):
for service_dict in self.service_ls(id=id, name=name):
- service_id = service_dict['ID']
- service_name = service_dict['NAME']
+ service_id = service_dict["ID"]
+ service_name = service_dict["NAME"]
if not service_name.startswith(self._name_prefix):
continue
task_list = self.service_ps(service_id)
@@ -281,17 +300,17 @@ class DockerSwarmCLIInterface(DockerSwarmInterface, DockerCLIInterface):
def service_tasks(self, service):
for task_dict in self.service_ps(service.id):
- if task_dict['NAME'].strip().startswith(r'\_'):
- continue # historical task
+ if task_dict["NAME"].strip().startswith(r"\_"):
+ continue # historical task
yield DockerTask.from_cli(self, task_dict, service=service)
def nodes(self, id=None, name=None):
for node_dict in self.node_ls(id=id, name=name):
- node_id = node_dict['ID'].strip(' *')
- node_name = node_dict['HOSTNAME']
+ node_id = node_dict["ID"].strip(" *")
+ node_name = node_dict["HOSTNAME"]
if self._node_prefix and not node_name.startswith(self._node_prefix):
continue
- task_list = filter(lambda x: x['NAME'].startswith(self._name_prefix), self.node_ps(node_id))
+ task_list = filter(lambda x: x["NAME"].startswith(self._name_prefix), self.node_ps(node_id))
yield DockerNode.from_cli(self, node_dict, task_list)
#
@@ -299,62 +318,64 @@ class DockerSwarmCLIInterface(DockerSwarmInterface, DockerCLIInterface):
#
def service_create(self, command, image=None, **kwopts):
- if ('service_create_image_constraint' in self._conf or 'service_create_cpus_constraint' in self._conf) and 'constraint' not in kwopts:
- kwopts['constraint'] = []
+ if (
+ "service_create_image_constraint" in self._conf or "service_create_cpus_constraint" in self._conf
+ ) and "constraint" not in kwopts:
+ kwopts["constraint"] = []
image = self._get_image(image)
if self._conf.service_create_image_constraint:
- kwopts['constraint'].append((IMAGE_CONSTRAINT, '==', image))
+ kwopts["constraint"].append((IMAGE_CONSTRAINT, "==", image))
if self._conf.service_create_cpus_constraint:
- cpus = kwopts.get('reserve_cpus', kwopts.get('limit_cpus', '1'))
- kwopts['constraint'].append((CPUS_CONSTRAINT, '==', cpus))
+ cpus = kwopts.get("reserve_cpus", kwopts.get("limit_cpus", "1"))
+ kwopts["constraint"].append((CPUS_CONSTRAINT, "==", cpus))
if self._conf.cpus:
- kwopts['cpu_limit'] = self._conf.cpus
- kwopts['cpu_reservation'] = self._conf.cpus
+ kwopts["cpu_limit"] = self._conf.cpus
+ kwopts["cpu_reservation"] = self._conf.cpus
if self._conf.memory:
- kwopts['mem_limit'] = self._conf.memory
- kwopts['mem_reservation'] = self._conf.memory
+ kwopts["mem_limit"] = self._conf.memory
+ kwopts["mem_reservation"] = self._conf.memory
self.set_kwopts_name(kwopts)
- args = '{kwopts} {image} {command}'.format(
+ args = "{kwopts} {image} {command}".format(
kwopts=self._stringify_kwopts(kwopts),
- image=image if image else '',
- command=command if command else '',
+ image=image if image else "",
+ command=command if command else "",
).strip()
- service_id = self._run_docker(subcommand='service create', args=args, verbose=True)
+ service_id = self._run_docker(subcommand="service create", args=args, verbose=True)
return DockerService.from_id(self, service_id)
@docker_json
def service_inspect(self, service_id):
- return self._run_docker(subcommand='service inspect', args=service_id)[0]
+ return self._run_docker(subcommand="service inspect", args=service_id)[0]
@docker_columns
def service_ls(self, id=None, name=None):
- return self._run_docker(subcommand='service ls', args=self._filter_by_id_or_name(id, name))
+ return self._run_docker(subcommand="service ls", args=self._filter_by_id_or_name(id, name))
@docker_columns
def service_ps(self, service_id):
- return self._run_docker(subcommand='service ps', args=f'--no-trunc {service_id}')
+ return self._run_docker(subcommand="service ps", args=f"--no-trunc {service_id}")
def service_rm(self, service_ids):
- service_ids = ' '.join(service_ids)
- return self._run_docker(subcommand='service rm', args=service_ids).splitlines()
+ service_ids = " ".join(service_ids)
+ return self._run_docker(subcommand="service rm", args=service_ids).splitlines()
@docker_json
def node_inspect(self, node_id):
- return self._run_docker(subcommand='node inspect', args=node_id)[0]
+ return self._run_docker(subcommand="node inspect", args=node_id)[0]
@docker_columns
def node_ls(self, id=None, name=None):
- return self._run_docker(subcommand='node ls', args=self._filter_by_id_or_name(id, name))
+ return self._run_docker(subcommand="node ls", args=self._filter_by_id_or_name(id, name))
@docker_columns
def node_ps(self, node_id):
- return self._run_docker(subcommand='node ps', args=f'--no-trunc {node_id}')
+ return self._run_docker(subcommand="node ps", args=f"--no-trunc {node_id}")
def node_update(self, node_id, **kwopts):
- return self._run_docker(subcommand='node update', args='{kwopts} {node_id}'.format(
- kwopts=self._stringify_kwopts(kwopts),
- node_id=node_id
- ))
+ return self._run_docker(
+ subcommand="node update",
+ args="{kwopts} {node_id}".format(kwopts=self._stringify_kwopts(kwopts), node_id=node_id),
+ )
@docker_json
def task_inspect(self, task_id):
@@ -363,51 +384,51 @@ class DockerSwarmCLIInterface(DockerSwarmInterface, DockerCLIInterface):
class DockerSwarmAPIInterface(DockerSwarmInterface, DockerAPIInterface):
- container_type = 'docker_swarm'
+ container_type = "docker_swarm"
placement_option_map = {
- 'constraint': {'param': 'constraints'},
+ "constraint": {"param": "constraints"},
}
service_mode_option_map = {
- 'service_mode': {'param': 0, 'default': 'replicated'},
- 'replicas': {'default': 1},
+ "service_mode": {"param": 0, "default": "replicated"},
+ "replicas": {"default": 1},
}
endpoint_spec_option_map: Dict[str, Dict] = {
- 'ports': {},
+ "ports": {},
}
resources_option_map = {
- 'cpus': {'params': ('cpu_limit', 'cpu_reservation'), 'map': lambda x: int(x * 1000000000)},
- 'memory': {'params': ('mem_limit', 'mem_reservation')},
+ "cpus": {"params": ("cpu_limit", "cpu_reservation"), "map": lambda x: int(x * 1000000000)},
+ "memory": {"params": ("mem_limit", "mem_reservation")},
}
container_spec_option_map = {
- 'image': {'param': 0},
- 'command': {},
- 'environment': {'param': 'env'},
- 'labels': {},
+ "image": {"param": 0},
+ "command": {},
+ "environment": {"param": "env"},
+ "labels": {},
}
restart_policy_option_map = {
- 'restart_condition': {'param': 'condition', 'default': 'none'},
- 'restart_delay': {'param': 'delay'},
- 'restart_max_attempts': {'param': 'max_attemps'},
+ "restart_condition": {"param": "condition", "default": "none"},
+ "restart_delay": {"param": "delay"},
+ "restart_max_attempts": {"param": "max_attemps"},
}
task_template_option_map = {
- '_container_spec': {'spec_class': docker.types.ContainerSpec, 'required': True},
- '_resources': {'spec_class': docker.types.Resources},
- '_restart_policy': {'spec_class': docker.types.RestartPolicy},
- '_placement': {'spec_class': docker.types.Placement},
+ "_container_spec": {"spec_class": docker.types.ContainerSpec, "required": True},
+ "_resources": {"spec_class": docker.types.Resources},
+ "_restart_policy": {"spec_class": docker.types.RestartPolicy},
+ "_placement": {"spec_class": docker.types.Placement},
}
node_spec_option_map = {
- 'availability': {'param': 'Availability'},
- 'name': {'param': 'Name'},
- 'role': {'param': 'Role'},
- 'labels': {'param': 'Labels'},
+ "availability": {"param": "Availability"},
+ "name": {"param": "Name"},
+ "role": {"param": "Role"},
+ "labels": {"param": "Labels"},
}
@staticmethod
def create_random_port_spec(port):
return {
- 'Protocol': 'tcp',
- 'PublishedPort': None,
- 'TargetPort': port,
+ "Protocol": "tcp",
+ "PublishedPort": None,
+ "TargetPort": port,
}
#
@@ -422,27 +443,27 @@ class DockerSwarmAPIInterface(DockerSwarmInterface, DockerAPIInterface):
if self._conf[opt]:
kwopts[opt] = self._conf[opt]
# image is part of the container spec
- kwopts['image'] = self._get_image(image)
+ kwopts["image"] = self._get_image(image)
# service constraints
- kwopts['constraint'] = kwopts.get('constraint', [])
+ kwopts["constraint"] = kwopts.get("constraint", [])
if self._conf.service_create_image_constraint:
- kwopts['constraint'].append(f"{IMAGE_CONSTRAINT}=={image}")
+ kwopts["constraint"].append(f"{IMAGE_CONSTRAINT}=={image}")
if self._conf.service_create_cpus_constraint:
- cpus = kwopts.get('reserve_cpus', kwopts.get('limit_cpus', '1'))
- kwopts['constraint'].append(f"{CPUS_CONSTRAINT}=={cpus}")
+ cpus = kwopts.get("reserve_cpus", kwopts.get("limit_cpus", "1"))
+ kwopts["constraint"].append(f"{CPUS_CONSTRAINT}=={cpus}")
# ports
- if 'publish_port_random' in kwopts:
- kwopts['ports'] = [DockerSwarmAPIInterface.create_random_port_spec(kwopts.pop('publish_port_random'))]
+ if "publish_port_random" in kwopts:
+ kwopts["ports"] = [DockerSwarmAPIInterface.create_random_port_spec(kwopts.pop("publish_port_random"))]
# create specs
- service_mode = self._create_docker_api_spec('service_mode', docker.types.ServiceMode, kwopts)
- endpoint_spec = self._create_docker_api_spec('endpoint_spec', docker.types.EndpointSpec, kwopts)
- task_template = self._create_docker_api_spec('task_template', docker.types.TaskTemplate, kwopts)
+ service_mode = self._create_docker_api_spec("service_mode", docker.types.ServiceMode, kwopts)
+ endpoint_spec = self._create_docker_api_spec("endpoint_spec", docker.types.EndpointSpec, kwopts)
+ task_template = self._create_docker_api_spec("task_template", docker.types.TaskTemplate, kwopts)
self.set_kwopts_name(kwopts)
log.debug("Docker service task template:\n%s", safe_dumps_formatted(task_template))
log.debug("Docker service endpoint specification:\n%s", safe_dumps_formatted(endpoint_spec))
log.debug("Docker service mode:\n%s", safe_dumps_formatted(service_mode))
log.debug("Docker service creation parameters:\n%s", safe_dumps_formatted(kwopts))
- success_test = partial(self._first, self.service_ls, name=kwopts['name'])
+ success_test = partial(self._first, self.service_ls, name=kwopts["name"])
# this can raise exceptions, if necessary we could wrap them in a more generic "creation failed" exception class
service = self._client.create_service(
task_template,
@@ -450,9 +471,10 @@ class DockerSwarmAPIInterface(DockerSwarmInterface, DockerAPIInterface):
endpoint_spec=endpoint_spec,
success_test=success_test,
max_tries=5,
- **kwopts)
- service_id = service.get('ID')
- log.debug('Created service: %s (%s)', kwopts['name'], service_id)
+ **kwopts,
+ )
+ service_id = service.get("ID")
+ log.debug("Created service: %s (%s)", kwopts["name"], service_id)
return DockerService.from_id(self, service_id)
def service_inspect(self, service_id):
@@ -463,7 +485,7 @@ class DockerSwarmAPIInterface(DockerSwarmInterface, DockerAPIInterface):
# roughly `docker service ps`
def service_ps(self, service_id):
- return self.task_ls(filters={'service': service_id})
+ return self.task_ls(filters={"service": service_id})
def service_rm(self, service_ids):
r = []
@@ -480,15 +502,15 @@ class DockerSwarmAPIInterface(DockerSwarmInterface, DockerAPIInterface):
# roughly `docker node ps`
def node_ps(self, node_id):
- return self.task_ls(filters={'node': node_id})
+ return self.task_ls(filters={"node": node_id})
def node_update(self, node_id, **kwopts):
node = DockerNode.from_id(self, node_id)
- spec = node.inspect['Spec']
- if 'label_add' in kwopts:
- kwopts['labels'] = spec.get('Labels', {})
- kwopts['labels'].update(kwopts.pop('label_add'))
- spec.update(self._create_docker_api_spec('node_spec', dict, kwopts))
+ spec = node.inspect["Spec"]
+ if "label_add" in kwopts:
+ kwopts["labels"] = spec.get("Labels", {})
+ kwopts["labels"].update(kwopts.pop("label_add"))
+ spec.update(self._create_docker_api_spec("node_spec", dict, kwopts))
return self._client.update_node(node.id, node.version, node_spec=spec)
def task_inspect(self, task_id):
diff --git a/lib/galaxy/datatypes/_schema.py b/lib/galaxy/datatypes/_schema.py
index 6efb7cc4565..50c1774b614 100644
--- a/lib/galaxy/datatypes/_schema.py
+++ b/lib/galaxy/datatypes/_schema.py
@@ -7,44 +7,23 @@ from typing import (
from pydantic import (
BaseModel,
Field,
- HttpUrl
+ HttpUrl,
)
class CompositeFileInfo(BaseModel):
- name: str = Field(
- ..., # Mark this field as required
- title="Name",
- description="The name of this composite file"
- )
- optional: bool = Field(
- title="Optional",
- description="" # TODO add description
- )
- mimetype: Optional[str] = Field(
- title="MIME type",
- description="The MIME type of this file"
- )
+ name: str = Field(..., title="Name", description="The name of this composite file") # Mark this field as required
+ optional: bool = Field(title="Optional", description="") # TODO add description
+ mimetype: Optional[str] = Field(title="MIME type", description="The MIME type of this file")
description: Optional[str] = Field(
- title="Description",
- description="Summary description of the purpouse of this file"
+ title="Description", description="Summary description of the purpouse of this file"
)
substitute_name_with_metadata: Optional[str] = Field(
- title="Substitute name with metadata",
- description="" # TODO add description
- )
- is_binary: bool = Field(
- title="Is binary",
- description="Whether this file is a binary file"
- )
- to_posix_lines: bool = Field(
- title="To posix lines",
- description="" # TODO add description
- )
- space_to_tab: bool = Field(
- title="Spaces to tabulation",
- description="" # TODO add description
+ title="Substitute name with metadata", description="" # TODO add description
)
+ is_binary: bool = Field(title="Is binary", description="Whether this file is a binary file")
+ to_posix_lines: bool = Field(title="To posix lines", description="") # TODO add description
+ space_to_tab: bool = Field(title="Spaces to tabulation", description="") # TODO add description
class DatatypeDetails(BaseModel):
@@ -52,26 +31,21 @@ class DatatypeDetails(BaseModel):
..., # Mark this field as required
title="Extension",
description="The data type’s Dataset file extension",
- example="bed"
- )
- description: Optional[str] = Field(
- title="Description",
- description="A summary description for this data type"
+ example="bed",
)
+ description: Optional[str] = Field(title="Description", description="A summary description for this data type")
description_url: Optional[HttpUrl] = Field(
title="Description URL",
description="The URL to a detailed description for this datatype",
- example="https://wiki.galaxyproject.org/Learn/Datatypes#Bed"
+ example="https://wiki.galaxyproject.org/Learn/Datatypes#Bed",
)
display_in_upload: bool = Field(
default=False,
title="Display in upload",
- description="If True, the associated file extension will be displayed in the `File Format` select list in the `Upload File from your computer` tool in the `Get Data` tool section of the tool panel"
+ description="If True, the associated file extension will be displayed in the `File Format` select list in the `Upload File from your computer` tool in the `Get Data` tool section of the tool panel",
)
composite_files: Optional[List[CompositeFileInfo]] = Field(
- default=None,
- title="Composite files",
- description="A collection of files composing this data type"
+ default=None, title="Composite files", description="A collection of files composing this data type"
)
@@ -123,7 +97,4 @@ class DatatypeConverter(BaseModel):
class DatatypeConverterList(BaseModel):
- __root__: List[DatatypeConverter] = Field(
- title='List of data type converters',
- default=[]
- )
+ __root__: List[DatatypeConverter] = Field(title="List of data type converters", default=[])
diff --git a/lib/galaxy/datatypes/annotation.py b/lib/galaxy/datatypes/annotation.py
index 8d2e70d87b3..b8eb217bc66 100644
--- a/lib/galaxy/datatypes/annotation.py
+++ b/lib/galaxy/datatypes/annotation.py
@@ -2,7 +2,10 @@ import logging
import tarfile
from galaxy.datatypes.binary import CompressedArchive
-from galaxy.datatypes.data import get_file_peek, Text
+from galaxy.datatypes.data import (
+ get_file_peek,
+ Text,
+)
from galaxy.datatypes.sniff import (
build_sniff_from_prefix,
FilePrefix,
@@ -22,8 +25,8 @@ class SnapHmm(Text):
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "SNAP HMM model"
else:
- dataset.peek = 'file does not exist'
- dataset.blurb = 'file purged from disc'
+ dataset.peek = "file does not exist"
+ dataset.blurb = "file purged from disc"
def display_peek(self, dataset):
try:
@@ -35,13 +38,14 @@ class SnapHmm(Text):
"""
SNAP model files start with zoeHMM
"""
- return file_prefix.startswith('zoeHMM')
+ return file_prefix.startswith("zoeHMM")
class Augustus(CompressedArchive):
"""
- Class describing an Augustus prediction model
+ Class describing an Augustus prediction model
"""
+
file_ext = "augustus"
edam_data = "data_0950"
compressed = True
@@ -51,8 +55,8 @@ class Augustus(CompressedArchive):
dataset.peek = "Augustus model"
dataset.blurb = nice_size(dataset.get_size())
else:
- dataset.peek = 'file does not exist'
- dataset.blurb = 'file purged from disk'
+ dataset.peek = "file does not exist"
+ dataset.blurb = "file purged from disk"
def display_peek(self, dataset):
try:
@@ -65,19 +69,21 @@ class Augustus(CompressedArchive):
Augustus archives always contain the same files
"""
if filename and tarfile.is_tarfile(filename):
- with tarfile.open(filename, 'r') as temptar:
+ with tarfile.open(filename, "r") as temptar:
for f in temptar:
if not f.isfile():
continue
- if f.name.endswith('_exon_probs.pbl') \
- or f.name.endswith('_igenic_probs.pbl') \
- or f.name.endswith('_intron_probs.pbl') \
- or f.name.endswith('_metapars.cfg') \
- or f.name.endswith('_metapars.utr.cfg') \
- or f.name.endswith('_parameters.cfg') \
- or f.name.endswith('_parameters.cgp.cfg') \
- or f.name.endswith('_utr_probs.pbl') \
- or f.name.endswith('_weightmatrix.txt'):
+ if (
+ f.name.endswith("_exon_probs.pbl")
+ or f.name.endswith("_igenic_probs.pbl")
+ or f.name.endswith("_intron_probs.pbl")
+ or f.name.endswith("_metapars.cfg")
+ or f.name.endswith("_metapars.utr.cfg")
+ or f.name.endswith("_parameters.cfg")
+ or f.name.endswith("_parameters.cgp.cfg")
+ or f.name.endswith("_utr_probs.pbl")
+ or f.name.endswith("_weightmatrix.txt")
+ ):
return True
else:
return False
diff --git a/lib/galaxy/datatypes/anvio.py b/lib/galaxy/datatypes/anvio.py
index 9e0d9828b57..51c8f9eecd7 100644
--- a/lib/galaxy/datatypes/anvio.py
+++ b/lib/galaxy/datatypes/anvio.py
@@ -19,8 +19,9 @@ class AnvioComposite(Html):
Base class to use for Anvi'o composite datatypes.
Generally consist of a sqlite database, plus optional additional files
"""
+
file_ext = "anvio_composite"
- composite_type = 'auto_primary_file'
+ composite_type = "auto_primary_file"
def generate_primary_file(self, dataset=None):
"""
@@ -32,12 +33,12 @@ class AnvioComposite(Html):
if defined_files:
rval.append("This composite dataset is composed of the following defined files:")
for composite_name, composite_file in defined_files:
- opt_text = ''
+ opt_text = ""
if composite_file.optional:
- opt_text = ' (optional)'
- missing_text = ''
+ opt_text = " (optional)"
+ missing_text = ""
if not os.path.exists(os.path.join(dataset.extra_files_path, composite_name)):
- missing_text = ' (missing)'
+ missing_text = " (missing)"
rval.append(f'- {composite_name}{opt_text}{missing_text}
')
rval.append("
")
defined_files = map(lambda x: x[0], defined_files)
@@ -51,24 +52,24 @@ class AnvioComposite(Html):
rval.append("This composite dataset contains these undefined files:")
for rel_path in extra_files:
rval.append(f'- {rel_path}
')
- rval.append('
')
+ rval.append("")
if not (defined_files or extra_files):
rval.append("This composite dataset does not contain any files!