diff --git a/config/plugins/webhooks/demo/tool_list/__init__.py b/config/plugins/webhooks/demo/tool_list/__init__.py index 31db2b819c1..db8e58ceeca 100644 --- a/config/plugins/webhooks/demo/tool_list/__init__.py +++ b/config/plugins/webhooks/demo/tool_list/__init__.py @@ -3,7 +3,7 @@ import yaml def main(trans, webhook, params): data = {} - data['tools'] = [] + data["tools"] = [] unique_tools = [] tools = trans.app.toolbox.tools() @@ -16,24 +16,23 @@ def main(trans, webhook, params): except AttributeError: continue - if (ts_data['name'] + ts_data['installed_changeset_revision'] not in - unique_tools): + if ts_data["name"] + ts_data["installed_changeset_revision"] not in unique_tools: - unique_tools.append( - ts_data['name'] + ts_data['installed_changeset_revision'] + unique_tools.append(ts_data["name"] + ts_data["installed_changeset_revision"]) + + data["tools"].append( + { + "name": ts_data["name"], + "owner": ts_data["owner"], + "tool_panel_section_label": panel[1], + "tool_shed_url": ts_data["tool_shed"], + "install_tool_dependencies": True, + "install_repository_dependencies": True, + "install_resolver_dependencies": True, + "revisions": [ts_data["installed_changeset_revision"]], + } ) - - data['tools'].append({ - 'name': ts_data['name'], - 'owner': ts_data['owner'], - 'tool_panel_section_label': panel[1], - 'tool_shed_url': ts_data['tool_shed'], - 'install_tool_dependencies': True, - 'install_repository_dependencies': True, - 'install_resolver_dependencies': True, - 'revisions': [ts_data['installed_changeset_revision']] - }) if panel[0]: - data['tools'][-1]['tool_panel_section_id'] = panel[0] + data["tools"][-1]["tool_panel_section_id"] = panel[0] - return {'yaml': yaml.safe_dump(data, default_flow_style=False)} + return {"yaml": yaml.safe_dump(data, default_flow_style=False)} diff --git a/config/plugins/webhooks/demo/tour_generator/__init__.py b/config/plugins/webhooks/demo/tour_generator/__init__.py index 8e4d77e5bfc..c132aff0501 100644 --- a/config/plugins/webhooks/demo/tour_generator/__init__.py +++ b/config/plugins/webhooks/demo/tour_generator/__init__.py @@ -28,13 +28,12 @@ class TourGenerator: section of the provided tool. """ if not self._tool.tests: - raise ValueError('Tests are not defined.') + raise ValueError("Tests are not defined.") self._test = self._tool.tests[0] # All inputs with the type 'data' - self._data_inputs = {x.name: x for x in self._tool.input_params if - x.type == 'data'} + self._data_inputs = {x.name: x for x in self._tool.input_params if x.type == "data"} # Datasets from the section test_datasets = { @@ -45,20 +44,17 @@ class TourGenerator: # Conditional datasets for name in self._test.inputs.keys(): - if '|' in name: - input_name = name.split('|')[1] + if "|" in name: + input_name = name.split("|")[1] if input_name in self._data_inputs.keys(): - test_datasets.update({ - input_name: self._test.inputs[name][0] - }) + test_datasets.update({input_name: self._test.inputs[name][0]}) if not test_datasets.keys(): not_supported_input_types = [ - k for k, v in self._tool.inputs.items() if - v.type == 'repeat' or v.type == 'data_collection' + k for k, v in self._tool.inputs.items() if v.type == "repeat" or v.type == "data_collection" ] if not_supported_input_types: - raise ValueError('Not supported input types.') + raise ValueError("Not supported input types.") else: # Some tests don't have data inputs at all, # so we can generate a tour without them @@ -73,16 +69,15 @@ class TourGenerator: if not input_path: raise ValueError('Test dataset "%s" doesn\'t exist.' % input_name) - upload_tool = self._trans.app.toolbox.get_tool('upload1') + upload_tool = self._trans.app.toolbox.get_tool("upload1") filename = os.path.basename(input_path) - with open(input_path, 'rb') as f: + with open(input_path, "rb") as f: content = f.read() headers = { - 'content-disposition': - 'form-data; name="{}"; filename="{}"'.format( - 'files_0|file_data', filename - ), + "content-disposition": 'form-data; name="{}"; filename="{}"'.format( + "files_0|file_data", filename + ), } input_file = cgi_FieldStorage(headers=headers) @@ -90,76 +85,68 @@ class TourGenerator: input_file.file.write(content) inputs = { - 'dbkey': '?', # is it always a question mark? - 'file_type': input.extensions[0], - 'files_0|type': 'upload_dataset', - 'files_0|space_to_tab': None, - 'files_0|to_posix_lines': 'Yes', - 'files_0|file_data': input_file, + "dbkey": "?", # is it always a question mark? + "file_type": input.extensions[0], + "files_0|type": "upload_dataset", + "files_0|space_to_tab": None, + "files_0|to_posix_lines": "Yes", + "files_0|file_data": input_file, } params = Params(inputs, sanitize=False) incoming = params.__dict__ - output = upload_tool.handle_input(self._trans, incoming, - history=None) + output = upload_tool.handle_input(self._trans, incoming, history=None) - job_errors = output.get('job_errors', []) + job_errors = output.get("job_errors", []) if job_errors: # self._errors.extend(job_errors) - raise ValueError('Cannot upload a dataset.') + raise ValueError("Cannot upload a dataset.") else: - self._hids.update({ - input_name: output['out_data'][0][1].hid - }) + self._hids.update({input_name: output["out_data"][0][1].hid}) def _generate_tour(self): - """ Generate a tour. """ - tour_name = self._tool.name + ' Tour' + """Generate a tour.""" + tour_name = self._tool.name + " Tour" test_inputs = self._test.inputs.keys() - steps = [{ - 'title': tour_name, - 'content': 'This short tour will guide you through the ' - + self._tool.name + ' tool.', - 'orphan': True - }] + steps = [ + { + "title": tour_name, + "content": "This short tour will guide you through the " + self._tool.name + " tool.", + "orphan": True, + } + ] for name, input in self._tool.inputs.items(): cond_case_steps = [] - if input.type == 'repeat': + if input.type == "repeat": continue - step = { - 'title': input.label, - 'element': '[tour_id=%s]' % name, - 'placement': 'right', - 'content': '' - } + step = {"title": input.label, "element": "[tour_id=%s]" % name, "placement": "right", "content": ""} - if input.type == 'text': + if input.type == "text": if name in test_inputs: param = self._test.inputs[name] - step['content'] = 'Enter value(s): %s' % param + step["content"] = "Enter value(s): %s" % param else: - step['content'] = 'Enter a value' + step["content"] = "Enter a value" - elif input.type == 'integer' or input.type == 'float': + elif input.type == "integer" or input.type == "float": if name in test_inputs: num_param = self._test.inputs[name][0] - step['content'] = 'Enter number: %s' % num_param + step["content"] = "Enter number: %s" % num_param else: - step['content'] = 'Enter a number' + step["content"] = "Enter a number" - elif input.type == 'boolean': + elif input.type == "boolean": if name in test_inputs: - choice = 'Yes' if self._test.inputs[name][0] is True \ - else 'No' - step['content'] = 'Choose %s' % choice + choice = "Yes" if self._test.inputs[name][0] is True else "No" + step["content"] = "Choose %s" % choice else: - step['content'] = 'Choose Yes/No' + step["content"] = "Choose Yes/No" - elif input.type == 'select': + elif input.type == "select": params = [] if name in test_inputs: for option in input.static_options: @@ -167,26 +154,23 @@ class TourGenerator: if test_option == option[1]: params.append(option[0]) if params: - select_param = ', '.join(params) - step['content'] = 'Select parameter(s): %s' % \ - select_param + select_param = ", ".join(params) + step["content"] = "Select parameter(s): %s" % select_param else: - step['content'] = 'Select a parameter' + step["content"] = "Select a parameter" - elif input.type == 'data': + elif input.type == "data": if name in test_inputs: hid = self._hids[name] dataset = self._test.inputs[name][0] - step['content'] = 'Select dataset: {}: {}'.format( - hid, dataset - ) + step["content"] = "Select dataset: {}: {}".format(hid, dataset) else: - step['content'] = 'Select a dataset' + step["content"] = "Select a dataset" - elif input.type == 'conditional': - param_id = f'{input.name}|{input.test_param.name}' - step['title'] = input.test_param.label - step['element'] = '[tour_id="%s"]' % param_id + elif input.type == "conditional": + param_id = f"{input.name}|{input.test_param.name}" + step["title"] = input.test_param.label + step["element"] = '[tour_id="%s"]' % param_id params = [] if param_id in self._test.inputs.keys(): @@ -203,60 +187,60 @@ class TourGenerator: cases[key] = value.label for case_id, case_title in cases.items(): - tour_id = f'{input.name}|{case_id}' + tour_id = f"{input.name}|{case_id}" if tour_id in self._test.inputs.keys(): if case_id in self._data_inputs.keys(): hid = self._hids[case_id] dataset = self._test.inputs[tour_id][0] - step_msg = 'Select dataset: %s: %s' % \ - (hid, dataset) + step_msg = "Select dataset: %s: %s" % (hid, dataset) else: - case_params = ', '.join( - self._test.inputs[tour_id]) - step_msg = 'Select parameter(s): ' + \ - '%s' % case_params - cond_case_steps.append({ - 'title': case_title, - 'element': '[tour_id="%s"]' % tour_id, - 'placement': 'right', - 'content': step_msg - }) + case_params = ", ".join(self._test.inputs[tour_id]) + step_msg = "Select parameter(s): " + "%s" % case_params + cond_case_steps.append( + { + "title": case_title, + "element": '[tour_id="%s"]' % tour_id, + "placement": "right", + "content": step_msg, + } + ) if params: - cond_param = ', '.join(params) - step['content'] = 'Select parameter(s): %s' % \ - cond_param + cond_param = ", ".join(params) + step["content"] = "Select parameter(s): %s" % cond_param else: - step['content'] = 'Select a parameter' + step["content"] = "Select a parameter" - elif input.type == 'data_column': + elif input.type == "data_column": if name in test_inputs: column_param = self._test.inputs[name][0] - step['content'] = 'Select Column: %s' % column_param + step["content"] = "Select Column: %s" % column_param else: - step['content'] = 'Select a column' + step["content"] = "Select a column" else: - step['content'] = 'Select a parameter' + step["content"] = "Select a parameter" steps.append(step) if cond_case_steps: steps.extend(cond_case_steps) # add conditional input steps # Add the last step - steps.append({ - 'title': 'Execute tool', - 'content': 'Click Execute button to run the tool.', - 'element': '#execute', - 'placement': 'bottom', - # 'postclick': ['#execute'] - }) + steps.append( + { + "title": "Execute tool", + "content": "Click Execute button to run the tool.", + "element": "#execute", + "placement": "bottom", + # 'postclick': ['#execute'] + } + ) self._tour = { - 'title_default': tour_name, - 'name': tour_name, - 'description': self._tool.name + ' ' + self._tool.description, - 'steps': steps + "title_default": tour_name, + "name": tour_name, + "description": self._tool.name + " " + self._tool.description, + "steps": steps, } def get_data(self): @@ -264,26 +248,22 @@ class TourGenerator: Return a dictionary with the uploaded datasets' history ids and the generated tour. """ - return { - 'useDatasets': self._use_datasets, - 'hids': list(self._hids.values()), - 'tour': self._tour - } + return {"useDatasets": self._use_datasets, "hids": list(self._hids.values()), "tour": self._tour} def main(trans, webhook, params): - error = '' + error = "" data = {} try: - if not params or 'tool_id' not in params.keys(): - raise KeyError('Tool id is missing.') + if not params or "tool_id" not in params.keys(): + raise KeyError("Tool id is missing.") - if not params or 'tool_version' not in params.keys(): - raise KeyError('Tool version is missing.') + if not params or "tool_version" not in params.keys(): + raise KeyError("Tool version is missing.") - tool_id = params['tool_id'] - tool_version = params['tool_version'] + tool_id = params["tool_id"] + tool_version = params["tool_version"] tour_generator = TourGenerator(trans, tool_id, tool_version) data = tour_generator.get_data() @@ -291,4 +271,4 @@ def main(trans, webhook, params): error = str(e) log.exception(e) - return {'success': not error, 'error': error, 'data': data} + return {"success": not error, "error": error, "data": data} diff --git a/contrib/galaxy_config_merger.py b/contrib/galaxy_config_merger.py index 887e43434f4..f7d60f183aa 100644 --- a/contrib/galaxy_config_merger.py +++ b/contrib/galaxy_config_merger.py @@ -1,5 +1,5 @@ #! /usr/bin/env python -''' +""" galaxy_config_merger.py Created by Anne Pajon on 31 Jan 2012 @@ -22,7 +22,7 @@ FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY OF THE ORIGINAL WORK IS WITH YOU. Script for merging specific local Galaxy config galaxy.ini.cri with default Galaxy galaxy.ini.sample -''' +""" import configparser import logging @@ -32,16 +32,18 @@ import sys def main(): # logging configuration - logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) + logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO) # get the options parser = optparse.OptionParser() parser.add_option("-s", "--sample", dest="sample", action="store", help="path to Galaxy galaxy.ini.sample file") parser.add_option("-c", "--config", dest="config", action="store", help="path to your own galaxy.ini file") - parser.add_option("-o", "--output", dest="output", action="store", help="path to the new merged galaxy.ini.new file") + parser.add_option( + "-o", "--output", dest="output", action="store", help="path to the new merged galaxy.ini.new file" + ) (options, args) = parser.parse_args() - for option in ['sample', 'config']: + for option in ["sample", "config"]: if getattr(options, option) is None: print("Please supply a --%s parameter.\n" % (option)) parser.print_help() @@ -63,18 +65,24 @@ def main(): for (name, value) in config.items(section): if not config_sample.has_option(section, name): if not "#%s" % name in config_sample_content: - logging.warning(f"-MISSING- section [{section}] option '{name}' not found in sample file. It will be ignored.") + logging.warning( + f"-MISSING- section [{section}] option '{name}' not found in sample file. It will be ignored." + ) else: - logging.info(f"-notset- section [{section}] option '{name}' not set in sample file. It will be added.") + logging.info( + f"-notset- section [{section}] option '{name}' not set in sample file. It will be added." + ) config_sample.set(section, name, value) else: if not config_sample.get(section, name) == value: - logging.info(f"- diff - section [{section}] option '{name}' has different value ('{config_sample.get(section, name)}':'{value}'). It will be modified.") + logging.info( + f"- diff - section [{section}] option '{name}' has different value ('{config_sample.get(section, name)}':'{value}'). It will be modified." + ) config_sample.set(section, name, value) logging.info("---------- DIFFERENCE ANALYSIS END ----------") if options.output: - outputfile = open(options.output, 'w') + outputfile = open(options.output, "w") config_sample.write(outputfile) outputfile.close() else: @@ -83,5 +91,5 @@ def main(): logging.info("read Galaxy galaxy.ini.sample for detailed information.") -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/cron/add_manual_builds.py b/cron/add_manual_builds.py index 8a4197eef5b..5138442e145 100644 --- a/cron/add_manual_builds.py +++ b/cron/add_manual_builds.py @@ -21,7 +21,7 @@ def add_manual_builds(input_file, build_file, chr_dir): existing_builds.append(line.replace("\n", "").replace("\r", "").split("\t")[0]) except Exception: continue - build_file_out = open(build_file, 'a') + build_file_out = open(build_file, "a") for line in open(input_file): try: fields = line.replace("\n", "").replace("\r", "").split("\t") @@ -35,7 +35,7 @@ def add_manual_builds(input_file, build_file, chr_dir): chrs = [] print(build + "\t" + name + " (" + build + ")", file=build_file_out) if chrs: # create len file if provided chrom lens - chr_len_out = open(os.path.join(chr_dir, build + ".len"), 'w') + chr_len_out = open(os.path.join(chr_dir, build + ".len"), "w") for chr in chrs: print(chr.replace("=", "\t"), file=chr_len_out) chr_len_out.close() diff --git a/cron/build_chrom_db.py b/cron/build_chrom_db.py index 7289dc72202..615e791e5c2 100644 --- a/cron/build_chrom_db.py +++ b/cron/build_chrom_db.py @@ -17,24 +17,26 @@ import os import sys from urllib.parse import urlencode -import requests - import parse_builds # noqa: I100,I202 +import requests def getchrominfo(url, db): tableURL = "http://genome-test.gi.ucsc.edu/cgi-bin/hgTables?" - URL = tableURL + urlencode({ - "clade": "", - "org": "", - "db": db, - "hgta_outputType": "primaryTable", - "hgta_group": "allTables", - "hgta_table": "chromInfo", - "hgta_track": db, - "hgta_regionType": "", - "position": "", - "hgta_doTopSubmit": "get info"}) + URL = tableURL + urlencode( + { + "clade": "", + "org": "", + "db": db, + "hgta_outputType": "primaryTable", + "hgta_group": "allTables", + "hgta_table": "chromInfo", + "hgta_track": db, + "hgta_regionType": "", + "position": "", + "hgta_doTopSubmit": "get info", + } + ) page = requests.get(URL).text for i, line in enumerate(page.splitlines()): line = line.rstrip("\r\n") diff --git a/cron/cleanup_datasets.py b/cron/cleanup_datasets.py index 184be129957..f20362d6b36 100644 --- a/cron/cleanup_datasets.py +++ b/cron/cleanup_datasets.py @@ -2,5 +2,7 @@ import sys -sys.exit("This script has been deprecated, replaced by the set of scripts in /scripts/cleanup_datsets/." - "See https://wiki.galaxyproject.org/Admin/Config/Performance/Purge%20Histories%20and%20Datasets for more information.") +sys.exit( + "This script has been deprecated, replaced by the set of scripts in /scripts/cleanup_datsets/." + "See https://wiki.galaxyproject.org/Admin/Config/Performance/Purge%20Histories%20and%20Datasets for more information." +) diff --git a/cron/parse_builds.py b/cron/parse_builds.py index 45361c3aeeb..dcd2005c537 100644 --- a/cron/parse_builds.py +++ b/cron/parse_builds.py @@ -29,7 +29,7 @@ def getbuilds(url): print("#Harvested from " + url) print("?\tunspecified (?)") for dsn in tree: - build = dsn.find("SOURCE").attrib['id'] + build = dsn.find("SOURCE").attrib["id"] description = dsn.find("DESCRIPTION").text.replace(" - Genome at UCSC", "").replace(" Genome at UCSC", "") fields = description.split(" ") diff --git a/cron/parse_builds_3_sites.py b/cron/parse_builds_3_sites.py index c1eb4d0a147..52ad1bd6acd 100644 --- a/cron/parse_builds_3_sites.py +++ b/cron/parse_builds_3_sites.py @@ -7,10 +7,12 @@ import xml.etree.ElementTree as ElementTree import requests -sites = ['http://genome.ucsc.edu/cgi-bin/', - 'http://archaea.ucsc.edu/cgi-bin/', - 'http://genome-test.gi.ucsc.edu/cgi-bin/'] -names = ['main', 'archaea', 'test'] +sites = [ + "http://genome.ucsc.edu/cgi-bin/", + "http://archaea.ucsc.edu/cgi-bin/", + "http://genome-test.gi.ucsc.edu/cgi-bin/", +] +names = ["main", "archaea", "test"] def main(): @@ -32,7 +34,7 @@ def main(): print("#Harvested from", site) for dsn in tree: - build = dsn.find("SOURCE").attrib['id'] + build = dsn.find("SOURCE").attrib["id"] builds.append(build) build_dict = {} for build in builds: diff --git a/doc/parse_gx_xsd.py b/doc/parse_gx_xsd.py index 382c7eee8ce..a74557fdf20 100644 --- a/doc/parse_gx_xsd.py +++ b/doc/parse_gx_xsd.py @@ -28,7 +28,7 @@ def main(): toc_list.append(tag.build_toc_entry()) content_list.append(tag.build_help()) elif not found_tag: - print(line, end='') + print(line, end="") else: raise Exception("No normal text allowed after the first $tag") print("## Contents\n") @@ -36,11 +36,10 @@ def main(): print(el) print("\n") for el in content_list: - print(el, end='') + print(el, end="") class Tag: - def __init__(self, line): assert line.startswith("$tag:") line_parts = line.split(" ") @@ -60,9 +59,9 @@ class Tag: @property def _anchor(self): anchor = self.title - for _ in ['|', '_']: - anchor = anchor.replace(_, '-') - return '#' + anchor + for _ in ["|", "_"]: + anchor = anchor.replace(_, "-") + return "#" + anchor @property def _pretty_title(self): @@ -93,18 +92,24 @@ def _build_tag(tag, hide_attributes): text = _replace_attribute_list(tag, text, attributes) for line in text.splitlines(): if line.startswith("$assertions"): - assertions_tag = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}complexType[@name='TestAssertions']") + assertions_tag = xmlschema_doc.find( + "//{http://www.w3.org/2001/XMLSchema}complexType[@name='TestAssertions']" + ) assertions_buffer = StringIO() assertions_buffer.write(_doc_or_none(assertions_tag)) assertions_buffer.write("\n\n") - assertion_groups = assertions_tag.xpath("xs:choice/xs:group", namespaces={'xs': 'http://www.w3.org/2001/XMLSchema'}) + assertion_groups = assertions_tag.xpath( + "xs:choice/xs:group", namespaces={"xs": "http://www.w3.org/2001/XMLSchema"} + ) for group in assertion_groups: - ref = group.attrib['ref'] + ref = group.attrib["ref"] assertion_tag = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}group[@name='" + ref + "']") doc = _doc_or_none(assertion_tag) assertions_buffer.write(f"### {doc}\n\n") - elements = assertion_tag.findall("{http://www.w3.org/2001/XMLSchema}choice/{http://www.w3.org/2001/XMLSchema}element") + elements = assertion_tag.findall( + "{http://www.w3.org/2001/XMLSchema}choice/{http://www.w3.org/2001/XMLSchema}element" + ) for element in elements: doc = _doc_or_none(element) if doc is None: @@ -121,9 +126,12 @@ def _build_tag(tag, hide_attributes): best_practices = _get_bp_link(annotation_el) if best_practices: tag_help.write("\n\n### Best Practices\n") - tag_help.write(""" + tag_help.write( + """ Find the Intergalactic Utilities Commision suggested best practices for this -element [here](%s).""" % best_practices) +element [here](%s).""" + % best_practices + ) tag_help.write(_build_attributes_table(tag, attributes, hide_attributes)) return tag_help.getvalue() @@ -139,7 +147,9 @@ def _replace_attribute_list(tag, text, attributes): else: attribute_names = attributes_str.split(",") header_level = int(header_level) - text = text.replace(line, _build_attributes_table(tag, attributes, attribute_names=attribute_names, header_level=header_level)) + text = text.replace( + line, _build_attributes_table(tag, attributes, attribute_names=attribute_names, header_level=header_level) + ) return text @@ -155,7 +165,7 @@ def _build_attributes_table(tag, attributes, hide_attributes=False, attribute_na attribute_table = StringIO() attribute_table.write("\n\n") if attributes and not hide_attributes: - header_prefix = '#' * header_level + header_prefix = "#" * header_level attribute_table.write("\n%s Attributes\n\n" % header_prefix) attribute_table.write("Attribute | Details | Required\n") attribute_table.write("--- | --- | ---\n") @@ -176,25 +186,42 @@ def _build_attributes_table(tag, attributes, hide_attributes=False, attribute_na details = details.replace("\n", " ").strip() best_practices = _get_bp_link(annotation_el) if best_practices: - details += """ Find the Intergalactic Utilities Commision suggested best practices for this element [here](%s).""" % best_practices + details += ( + """ Find the Intergalactic Utilities Commision suggested best practices for this element [here](%s).""" + % best_practices + ) attribute_table.write(f"``{name}`` | {details} | {use}\n") return attribute_table.getvalue() def _find_attributes(tag): - raw_attributes = tag.findall("{http://www.w3.org/2001/XMLSchema}attribute") or \ - tag.findall("{http://www.w3.org/2001/XMLSchema}complexType/{http://www.w3.org/2001/XMLSchema}attribute") or \ - tag.findall("{http://www.w3.org/2001/XMLSchema}complexContent/{http://www.w3.org/2001/XMLSchema}extension/{http://www.w3.org/2001/XMLSchema}attribute") or \ - tag.findall("{http://www.w3.org/2001/XMLSchema}simpleContent/{http://www.w3.org/2001/XMLSchema}extension/{http://www.w3.org/2001/XMLSchema}attribute") - attribute_groups = tag.findall("{http://www.w3.org/2001/XMLSchema}attributeGroup") or \ - tag.findall("{http://www.w3.org/2001/XMLSchema}complexType/{http://www.w3.org/2001/XMLSchema}attributeGroup") or \ - tag.findall("{http://www.w3.org/2001/XMLSchema}complexContent/{http://www.w3.org/2001/XMLSchema}extension/{http://www.w3.org/2001/XMLSchema}attributeGroup") or \ - tag.findall("{http://www.w3.org/2001/XMLSchema}simpleContent/{http://www.w3.org/2001/XMLSchema}extension/{http://www.w3.org/2001/XMLSchema}attributeGroup") + raw_attributes = ( + tag.findall("{http://www.w3.org/2001/XMLSchema}attribute") + or tag.findall("{http://www.w3.org/2001/XMLSchema}complexType/{http://www.w3.org/2001/XMLSchema}attribute") + or tag.findall( + "{http://www.w3.org/2001/XMLSchema}complexContent/{http://www.w3.org/2001/XMLSchema}extension/{http://www.w3.org/2001/XMLSchema}attribute" + ) + or tag.findall( + "{http://www.w3.org/2001/XMLSchema}simpleContent/{http://www.w3.org/2001/XMLSchema}extension/{http://www.w3.org/2001/XMLSchema}attribute" + ) + ) + attribute_groups = ( + tag.findall("{http://www.w3.org/2001/XMLSchema}attributeGroup") + or tag.findall("{http://www.w3.org/2001/XMLSchema}complexType/{http://www.w3.org/2001/XMLSchema}attributeGroup") + or tag.findall( + "{http://www.w3.org/2001/XMLSchema}complexContent/{http://www.w3.org/2001/XMLSchema}extension/{http://www.w3.org/2001/XMLSchema}attributeGroup" + ) + or tag.findall( + "{http://www.w3.org/2001/XMLSchema}simpleContent/{http://www.w3.org/2001/XMLSchema}extension/{http://www.w3.org/2001/XMLSchema}attributeGroup" + ) + ) attributes = list(raw_attributes) for attribute_group in attribute_groups: attribute_group_name = attribute_group.get("ref") - attribute_group_def = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}attributeGroup/[@name='%s']" % attribute_group_name) + attribute_group_def = xmlschema_doc.find( + "//{http://www.w3.org/2001/XMLSchema}attributeGroup/[@name='%s']" % attribute_group_name + ) attributes.extend(_find_attributes(attribute_group_def)) return attributes @@ -222,5 +249,5 @@ def _doc_or_none(tag): return doc_el.text -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/doc/source/conf.py b/doc/source/conf.py index c983beef6a1..8e364417cb7 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -15,6 +15,7 @@ import os import sys import sphinx_rtd_theme + # Library to make .md to slideshow from recommonmark.transform import AutoStructify @@ -30,35 +31,35 @@ SKIP_RELEASES = os.environ.get("GALAXY_DOCS_SKIP_RELEASES", False) == "1" # REQUIRED GALAXY INCLUDES -sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib'))) +sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "lib"))) # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +# sys.path.insert(0, os.path.abspath('.')) # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['recommonmark', 'sphinx.ext.intersphinx', 'sphinx_markdown_tables'] +extensions = ["recommonmark", "sphinx.ext.intersphinx", "sphinx_markdown_tables"] if not SKIP_SOURCE: # TODO: Add https://pypi.org/project/sphinx-autodoc-typehints - extensions += ['sphinx.ext.doctest', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.autodoc'] + extensions += ["sphinx.ext.doctest", "sphinx.ext.todo", "sphinx.ext.coverage", "sphinx.ext.autodoc"] if not SKIP_VIEW_CODE: - extensions.append('sphinx.ext.viewcode') + extensions.append("sphinx.ext.viewcode") # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # Configure default autodoc's action -autodoc_default_flags = ['members', 'undoc-members'] +autodoc_default_flags = ["members", "undoc-members"] # Prevent alphabetical reordering of module members. -autodoc_member_order = 'bysource' +autodoc_member_order = "bysource" def dont_skip_init(app, what, name, obj, skip, options): @@ -70,26 +71,30 @@ def dont_skip_init(app, what, name, obj, skip, options): def setup(app): if not SKIP_SOURCE: app.connect("autodoc-skip-member", dont_skip_init) - app.add_config_value('recommonmark_config', { - 'enable_auto_doc_ref': False, - 'enable_auto_toc_tree': False, - 'enable_inline_math': False, # https://github.com/rtfd/recommonmark/pull/124 - }, True) + app.add_config_value( + "recommonmark_config", + { + "enable_auto_doc_ref": False, + "enable_auto_toc_tree": False, + "enable_inline_math": False, # https://github.com/rtfd/recommonmark/pull/124 + }, + True, + ) app.add_transform(AutoStructify) # The suffix of source filenames. -source_suffix = ['.rst', '.md'] +source_suffix = [".rst", ".md"] # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = 'Galaxy Project' -copyright = str(datetime.datetime.now().year) + ', Galaxy Committers' +project = "Galaxy Project" +copyright = str(datetime.datetime.now().year) + ", Galaxy Committers" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -97,33 +102,34 @@ copyright = str(datetime.datetime.now().year) + ', Galaxy Committers' # # The short X.Y version. from galaxy.version import VERSION, VERSION_MAJOR + version = VERSION_MAJOR # The full version, including alpha/beta/rc tags. release = VERSION # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['**/_*.rst'] +exclude_patterns = ["**/_*.rst"] if SKIP_SOURCE: - exclude_patterns.extend(['lib']) + exclude_patterns.extend(["lib"]) if SKIP_RELEASES: - exclude_patterns.extend(['releases']) + exclude_patterns.extend(["releases"]) # The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). @@ -131,34 +137,34 @@ add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # Intersphinx mapping to Python documentation intersphinx_mapping = { - 'python': ('https://docs.python.org/3', None), - 'requests': ("https://requests.readthedocs.io/en/master/", None), + "python": ("https://docs.python.org/3", None), + "requests": ("https://requests.readthedocs.io/en/master/", None), } # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. html_theme_options = { - 'collapse_navigation': False, - 'display_version': True, - 'navigation_depth': 2, - 'canonical_url': 'https://docs.galaxyproject.org/en/master/', + "collapse_navigation": False, + "display_version": True, + "navigation_depth": 2, + "canonical_url": "https://docs.galaxyproject.org/en/master/", } # Add any paths that contain custom themes here, relative to this directory. @@ -166,68 +172,68 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = 'Galaxydoc' +htmlhelp_basename = "Galaxydoc" # -- Options for LaTeX output -------------------------------------------------- @@ -235,10 +241,8 @@ htmlhelp_basename = 'Galaxydoc' latex_elements = { # The paper size ('letterpaper' or 'a4paper'). #'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). #'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. #'preamble': '', } @@ -246,42 +250,38 @@ latex_elements = { # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'Galaxy.tex', 'Galaxy Code Documentation', - 'Galaxy Team', 'manual'), + ("index", "Galaxy.tex", "Galaxy Code Documentation", "Galaxy Team", "manual"), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output -------------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'galaxy', 'Galaxy Documentation', - ['Galaxy Team'], 1) -] +man_pages = [("index", "galaxy", "Galaxy Documentation", ["Galaxy Team"], 1)] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------------ @@ -290,19 +290,25 @@ man_pages = [ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'Galaxy', 'Galaxy Documentation', - 'Galaxy Team', 'Galaxy', 'Data intensive biology for everyone.', - 'Miscellaneous'), + ( + "index", + "Galaxy", + "Galaxy Documentation", + "Galaxy Team", + "Galaxy", + "Data intensive biology for everyone.", + "Miscellaneous", + ), ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' # -- ReadTheDocs.org Settings ------------------------------------------------ @@ -315,8 +321,8 @@ class Mock: @classmethod def __getattr__(cls, name): - if name in ('__file__', '__path__'): - return '/dev/null' + if name in ("__file__", "__path__"): + return "/dev/null" elif name[0] == name[0].upper(): mockType = type(name, (), {}) mockType.__module__ = __name__ @@ -326,6 +332,6 @@ class Mock: # adding pbs_python, DRMAA_python, markupsafe, and drmaa here had no effect. -MOCK_MODULES = ['tables', 'decorator'] +MOCK_MODULES = ["tables", "decorator"] for mod_name in MOCK_MODULES: sys.modules[mod_name] = Mock() diff --git a/doc/source/conf.versioning.py b/doc/source/conf.versioning.py index fb8c7eb94b1..3e45f6db296 100644 --- a/doc/source/conf.versioning.py +++ b/doc/source/conf.versioning.py @@ -7,7 +7,7 @@ from distutils.version import LooseVersion from subprocess import check_output # This is set in the Jenkins matrix config -TARGET_GIT_BRANCH = os.environ.get('TARGET_BRANCH', 'dev') # noqa: F821 +TARGET_GIT_BRANCH = os.environ.get("TARGET_BRANCH", "dev") # noqa: F821 # Version message templates OLD_BANNER = """This document is for an old release of Galaxy.""" @@ -17,58 +17,51 @@ BANNER_APPEND = """ You can alternatively view this pa exists or view the top of the latest release's documentation.""" # Minimum version for linking to docs -MIN_DOC_VERSION = LooseVersion('17.05') +MIN_DOC_VERSION = LooseVersion("17.05") # Enable simpleversioning -extensions += ['sphinxcontrib.simpleversioning'] # noqa: F821 +extensions += ["sphinxcontrib.simpleversioning"] # noqa: F821 # -- sphinxcontrib-simpleversioning Settings --------------------------------- -simpleversioning_path_template = '/en/{version}/{pagename}' -simpleversioning_stable_version = 'master' +simpleversioning_path_template = "/en/{version}/{pagename}" +simpleversioning_stable_version = "master" simpleversioning_current_version = TARGET_GIT_BRANCH simpleversioning_versions = [ - {'id': 'latest', 'name': 'dev'}, - {'id': 'master', 'name': 'stable'}, + {"id": "latest", "name": "dev"}, + {"id": "master", "name": "stable"}, # Additional versions added below ] # Used for determining the latest stable release so the banner can be added to older releases. _stable = None # Use tags to determine versions - a stable version will have a branch before it's released, but not a tag. -tags = check_output(('git', 'tag')).decode().splitlines() +tags = check_output(("git", "tag")).decode().splitlines() for _tag in reversed(tags): - if _tag.startswith('v') and _tag.count('.') == 1: + if _tag.startswith("v") and _tag.count(".") == 1: # this version is released _ver = _tag[1:] if not _stable: _stable = _ver if LooseVersion(_ver) >= MIN_DOC_VERSION: - simpleversioning_versions.append( - {'id': 'release_%s' % _ver, 'name': _ver} - ) + simpleversioning_versions.append({"id": "release_%s" % _ver, "name": _ver}) -if re.fullmatch(r'release_\d{2}\.\d{2}', TARGET_GIT_BRANCH): +if re.fullmatch(r"release_\d{2}\.\d{2}", TARGET_GIT_BRANCH): if _stable: # The current stable release will go here but fail the next conditional, avoiding either banner. - if TARGET_GIT_BRANCH != 'release_%s' % _stable: + if TARGET_GIT_BRANCH != "release_%s" % _stable: simpleversioning_show_banner = True - _target_ver = TARGET_GIT_BRANCH[len('release_'):] + _target_ver = TARGET_GIT_BRANCH[len("release_") :] if LooseVersion(_target_ver) > LooseVersion(_stable): # Pre-release # Insert it between master and _stable - simpleversioning_versions.insert( - 2, - {'id': TARGET_GIT_BRANCH, 'name': _target_ver} - ) + simpleversioning_versions.insert(2, {"id": TARGET_GIT_BRANCH, "name": _target_ver}) simpleversioning_banner_message = PRE_BANNER + BANNER_APPEND else: simpleversioning_banner_message = OLD_BANNER + BANNER_APPEND -elif TARGET_GIT_BRANCH != 'master': - if TARGET_GIT_BRANCH != 'dev': +elif TARGET_GIT_BRANCH != "master": + if TARGET_GIT_BRANCH != "dev": # Feature branch - simpleversioning_versions.append( - {'id': TARGET_GIT_BRANCH, 'name': TARGET_GIT_BRANCH} - ) + simpleversioning_versions.append({"id": TARGET_GIT_BRANCH, "name": TARGET_GIT_BRANCH}) simpleversioning_show_banner = True simpleversioning_banner_message = DEV_BANNER + BANNER_APPEND diff --git a/lib/galaxy/__init__.py b/lib/galaxy/__init__.py index 8c478621de6..70b3b112b71 100644 --- a/lib/galaxy/__init__.py +++ b/lib/galaxy/__init__.py @@ -3,4 +3,5 @@ Galaxy root package -- this is a namespace package. """ from pkgutil import extend_path + __path__ = extend_path(__path__, __name__) # type: ignore[has-type] diff --git a/lib/galaxy/actions/library.py b/lib/galaxy/actions/library.py index 04be31f36ab..fdd365f0db2 100644 --- a/lib/galaxy/actions/library.py +++ b/lib/galaxy/actions/library.py @@ -22,19 +22,19 @@ from galaxy.tools.parameters import populate_state from galaxy.util.path import ( safe_contains, safe_relpath, - unsafe_walk + unsafe_walk, ) log = logging.getLogger(__name__) def validate_server_directory_upload(trans, server_dir): - if server_dir in [None, 'None', '']: + if server_dir in [None, "None", ""]: raise RequestParameterInvalidException("Invalid or unspecified server_dir parameter") if trans.user_is_admin: import_dir = trans.app.config.library_import_dir - import_dir_desc = 'library_import_dir' + import_dir_desc = "library_import_dir" if not import_dir: raise ConfigDoesNotAllowException('"library_import_dir" is not set in the Galaxy configuration') else: @@ -43,17 +43,31 @@ def validate_server_directory_upload(trans, server_dir): raise ConfigDoesNotAllowException('"user_library_import_dir" is not set in the Galaxy configuration') if server_dir != trans.user.email: import_dir = os.path.join(import_dir, trans.user.email) - import_dir_desc = 'user_library_import_dir' + import_dir_desc = "user_library_import_dir" full_dir = os.path.join(import_dir, server_dir) unsafe = None if safe_relpath(server_dir): username = trans.user.username if trans.app.config.user_library_import_check_permissions else None - if import_dir_desc == 'user_library_import_dir' and safe_contains(import_dir, full_dir, allowlist=trans.app.config.user_library_import_symlink_allowlist): - for unsafe in unsafe_walk(full_dir, allowlist=[import_dir] + trans.app.config.user_library_import_symlink_allowlist, username=username): - log.error('User attempted to import a path that resolves to a path outside of their import dir: %s -> %s', unsafe, os.path.realpath(unsafe)) + if import_dir_desc == "user_library_import_dir" and safe_contains( + import_dir, full_dir, allowlist=trans.app.config.user_library_import_symlink_allowlist + ): + for unsafe in unsafe_walk( + full_dir, + allowlist=[import_dir] + trans.app.config.user_library_import_symlink_allowlist, + username=username, + ): + log.error( + "User attempted to import a path that resolves to a path outside of their import dir: %s -> %s", + unsafe, + os.path.realpath(unsafe), + ) else: - log.error('User attempted to import a directory path that resolves to a path outside of their import dir: %s -> %s', server_dir, os.path.realpath(full_dir)) + log.error( + "User attempted to import a directory path that resolves to a path outside of their import dir: %s -> %s", + server_dir, + os.path.realpath(full_dir), + ) unsafe = True if unsafe: raise RequestParameterInvalidException("Invalid server_dir specified") @@ -66,7 +80,7 @@ def validate_path_upload(trans): raise ConfigDoesNotAllowException('"allow_path_paste" is not set to True in the Galaxy configuration file') if not trans.user_is_admin: - raise AdminRequiredException('Uploading files via filesystem paths can only be performed by administrators') + raise AdminRequiredException("Uploading files via filesystem paths can only be performed by administrators") class LibraryActions: @@ -76,10 +90,10 @@ class LibraryActions: def _upload_dataset(self, trans, folder_id: str, replace_dataset: Optional[LibraryDataset] = None, **kwd): # Set up the traditional tool state/params - cntrller = 'api' - tool_id = 'upload1' + cntrller = "api" + tool_id = "upload1" message = None - file_type = kwd.get('file_type') + file_type = kwd.get("file_type") try: upload_common.validate_datatype_extension(datatypes_registry=trans.app.datatypes_registry, ext=file_type) except RequestParameterInvalidException as e: @@ -93,13 +107,13 @@ class LibraryActions: if input.type == "upload_dataset": dataset_upload_inputs.append(input) # Library-specific params - server_dir = kwd.get('server_dir', '') - upload_option = kwd.get('upload_option', 'upload_file') + server_dir = kwd.get("server_dir", "") + upload_option = kwd.get("upload_option", "upload_file") response_code = 200 - if upload_option == 'upload_directory': + if upload_option == "upload_directory": full_dir, import_dir_desc = validate_server_directory_upload(trans, server_dir) - message = 'Select a directory' - elif upload_option == 'upload_paths': + message = "Select a directory" + elif upload_option == "upload_paths": # Library API already checked this - following check isn't actually needed. validate_path_upload(trans) # Some error handling should be added to this method. @@ -112,28 +126,38 @@ class LibraryActions: message = "Unable to parse upload parameters, please report this error." # Proceed with (mostly) regular upload processing if we're still errorless if response_code == 200: - if upload_option == 'upload_file': + if upload_option == "upload_file": tool_params = upload_common.persist_uploads(tool_params, trans) - uploaded_datasets = upload_common.get_uploaded_datasets(trans, cntrller, tool_params, dataset_upload_inputs, library_bunch=library_bunch) - elif upload_option == 'upload_directory': - uploaded_datasets, response_code, message = self._get_server_dir_uploaded_datasets(trans, kwd, full_dir, import_dir_desc, library_bunch, response_code, message) - elif upload_option == 'upload_paths': - uploaded_datasets, response_code, message = self._get_path_paste_uploaded_datasets(trans, kwd, library_bunch, response_code, message) - if upload_option == 'upload_file' and not uploaded_datasets: + uploaded_datasets = upload_common.get_uploaded_datasets( + trans, cntrller, tool_params, dataset_upload_inputs, library_bunch=library_bunch + ) + elif upload_option == "upload_directory": + uploaded_datasets, response_code, message = self._get_server_dir_uploaded_datasets( + trans, kwd, full_dir, import_dir_desc, library_bunch, response_code, message + ) + elif upload_option == "upload_paths": + uploaded_datasets, response_code, message = self._get_path_paste_uploaded_datasets( + trans, kwd, library_bunch, response_code, message + ) + if upload_option == "upload_file" and not uploaded_datasets: response_code = 400 - message = 'Select a file, enter a URL or enter text' + message = "Select a file, enter a URL or enter text" if response_code != 200: return (response_code, message) json_file_path = upload_common.create_paramfile(trans, uploaded_datasets) data_list = [ud.data for ud in uploaded_datasets] job_params = {} - job_params['link_data_only'] = json.dumps(kwd.get('link_data_only', 'copy_files')) - job_params['uuid'] = json.dumps(kwd.get('uuid', None)) - job, output = upload_common.create_job(trans, tool_params, tool, json_file_path, data_list, folder=library_bunch.folder, job_params=job_params) + job_params["link_data_only"] = json.dumps(kwd.get("link_data_only", "copy_files")) + job_params["uuid"] = json.dumps(kwd.get("uuid", None)) + job, output = upload_common.create_job( + trans, tool_params, tool, json_file_path, data_list, folder=library_bunch.folder, job_params=job_params + ) trans.app.job_manager.enqueue(job, tool=tool) return output - def _get_server_dir_uploaded_datasets(self, trans, params, full_dir, import_dir_desc, library_bunch, response_code, message): + def _get_server_dir_uploaded_datasets( + self, trans, params, full_dir, import_dir_desc, library_bunch, response_code, message + ): dir_response = self._get_server_dir_files(params, full_dir, import_dir_desc) files = dir_response[0] if not files: @@ -141,7 +165,9 @@ class LibraryActions: uploaded_datasets = [] for file in files: name = os.path.basename(file) - uploaded_datasets.append(self._make_library_uploaded_dataset(trans, params, name, file, 'server_dir', library_bunch)) + uploaded_datasets.append( + self._make_library_uploaded_dataset(trans, params, name, file, "server_dir", library_bunch) + ) return uploaded_datasets, 200, None def _get_server_dir_files(self, params, full_dir, import_dir_desc): @@ -150,8 +176,8 @@ class LibraryActions: for entry in os.listdir(full_dir): # Only import regular files path = os.path.join(full_dir, entry) - link_data_only = params.get('link_data_only', 'copy_files') - if os.path.islink(full_dir) and link_data_only == 'link_to_files': + link_data_only = params.get("link_data_only", "copy_files") + if os.path.islink(full_dir) and link_data_only == "link_to_files": # If we're linking instead of copying and the # sub-"directory" in the import dir is actually a symlink, # dereference the symlink, but not any of its contents. @@ -160,7 +186,7 @@ class LibraryActions: path = os.path.join(link_path, entry) else: path = os.path.abspath(os.path.join(link_path, entry)) - elif os.path.islink(path) and os.path.isfile(path) and link_data_only == 'link_to_files': + elif os.path.islink(path) and os.path.isfile(path) and link_data_only == "link_to_files": # If we're linking instead of copying and the "file" in the # sub-directory of the import dir is actually a symlink, # dereference the symlink (one dereference only, Vasili). @@ -182,13 +208,15 @@ class LibraryActions: return files, None, None def _get_path_paste_uploaded_datasets(self, trans, params, library_bunch, response_code, message): - preserve_dirs = util.string_as_bool(params.get('preserve_dirs', False)) + preserve_dirs = util.string_as_bool(params.get("preserve_dirs", False)) uploaded_datasets = [] (files_and_folders, _response_code, _message) = self._get_path_files_and_folders(params, preserve_dirs) if _response_code: return (uploaded_datasets, _response_code, _message) for (path, name, folder) in files_and_folders: - uploaded_datasets.append(self._make_library_uploaded_dataset(trans, params, name, path, 'path_paste', library_bunch, folder)) + uploaded_datasets.append( + self._make_library_uploaded_dataset(trans, params, name, path, "path_paste", library_bunch, folder) + ) return uploaded_datasets, 200, None def _get_path_files_and_folders(self, params, preserve_dirs): @@ -210,17 +238,21 @@ class LibraryActions: for file in files: file_path = os.path.abspath(os.path.join(basedir, file)) if preserve_dirs: - in_folder = os.path.dirname(file_path.replace(path, '', 1).lstrip('/')) + in_folder = os.path.dirname(file_path.replace(path, "", 1).lstrip("/")) else: in_folder = None files_and_folders.append((file_path, file, in_folder)) return files_and_folders def _paths_list(self, params): - return [(line.strip(), os.path.abspath(line.strip())) for line in params.get('filesystem_paths', '').splitlines() if line.strip()] + return [ + (line.strip(), os.path.abspath(line.strip())) + for line in params.get("filesystem_paths", "").splitlines() + if line.strip() + ] def _check_path_paste_params(self, params): - if params.get('filesystem_paths', '') == '': + if params.get("filesystem_paths", "") == "": message = "No paths entered in the upload form" response_code = 400 return None, response_code, message @@ -235,36 +267,36 @@ class LibraryActions: return None def _make_library_uploaded_dataset(self, trans, params, name, path, type, library_bunch, in_folder=None): - link_data_only = params.get('link_data_only', 'copy_files') - uuid_str = params.get('uuid', None) - file_type = params.get('file_type', None) + link_data_only = params.get("link_data_only", "copy_files") + uuid_str = params.get("uuid", None) + file_type = params.get("file_type", None) library_bunch.replace_dataset = None # not valid for these types of upload uploaded_dataset = util.bunch.Bunch() new_name = name # Remove compressed file extensions, if any, but only if # we're copying files into Galaxy's file space. - if link_data_only == 'copy_files': - if new_name.endswith('.gz'): - new_name = new_name.rstrip('.gz') - elif new_name.endswith('.zip'): - new_name = new_name.rstrip('.zip') + if link_data_only == "copy_files": + if new_name.endswith(".gz"): + new_name = new_name.rstrip(".gz") + elif new_name.endswith(".zip"): + new_name = new_name.rstrip(".zip") uploaded_dataset.name = new_name uploaded_dataset.path = path uploaded_dataset.type = type uploaded_dataset.ext = None uploaded_dataset.file_type = file_type - uploaded_dataset.dbkey = params.get('dbkey', None) - uploaded_dataset.to_posix_lines = params.get('to_posix_lines', None) - uploaded_dataset.space_to_tab = params.get('space_to_tab', None) - uploaded_dataset.tag_using_filenames = params.get('tag_using_filenames', False) - uploaded_dataset.tags = params.get('tags', None) - uploaded_dataset.purge_source = getattr(trans.app.config, 'ftp_upload_purge', True) + uploaded_dataset.dbkey = params.get("dbkey", None) + uploaded_dataset.to_posix_lines = params.get("to_posix_lines", None) + uploaded_dataset.space_to_tab = params.get("space_to_tab", None) + uploaded_dataset.tag_using_filenames = params.get("tag_using_filenames", False) + uploaded_dataset.tags = params.get("tags", None) + uploaded_dataset.purge_source = getattr(trans.app.config, "ftp_upload_purge", True) if in_folder: uploaded_dataset.in_folder = in_folder - uploaded_dataset.data = upload_common.new_upload(trans, 'api', uploaded_dataset, library_bunch) + uploaded_dataset.data = upload_common.new_upload(trans, "api", uploaded_dataset, library_bunch) uploaded_dataset.link_data_only = link_data_only uploaded_dataset.uuid = uuid_str - if link_data_only == 'link_to_files': + if link_data_only == "link_to_files": uploaded_dataset.data.link_to(path) trans.sa_session.add_all((uploaded_dataset.data, uploaded_dataset.data.dataset)) trans.sa_session.flush() @@ -274,15 +306,16 @@ class LibraryActions: is_admin = trans.user_is_admin current_user_roles = trans.get_current_user_roles() try: - parent_folder = trans.sa_session.query(trans.app.model.LibraryFolder).get(trans.security.decode_id(parent_id)) + parent_folder = trans.sa_session.query(trans.app.model.LibraryFolder).get( + trans.security.decode_id(parent_id) + ) except Exception: parent_folder = None # Check the library which actually contains the user-supplied parent folder, not the user-supplied # library, which could be anything. self._check_access(trans, is_admin, parent_folder, current_user_roles) self._check_add(trans, is_admin, parent_folder, current_user_roles) - new_folder = trans.app.model.LibraryFolder(name=kwd.get('name', ''), - description=kwd.get('description', '')) + new_folder = trans.app.model.LibraryFolder(name=kwd.get("name", ""), description=kwd.get("description", "")) # We are associating the last used genome build with folders, so we will always # initialize a new folder with the first dbkey in genome builds list which is currently # ? unspecified (?) @@ -300,7 +333,10 @@ class LibraryActions: if not item: message = f"Invalid history dataset ({escape(str(item))}) specified." raise ObjectNotFound(message) - elif not trans.app.security_agent.can_access_dataset(current_user_roles, item.dataset) and item.history.user == trans.user: + elif ( + not trans.app.security_agent.can_access_dataset(current_user_roles, item.dataset) + and item.history.user == trans.user + ): message = f"You do not have permission to access the history dataset with id ({str(item.id)})." raise ItemAccessibilityException(message) else: @@ -308,13 +344,15 @@ class LibraryActions: if not item: message = f"Invalid library item ({escape(str(item))}) specified." raise ObjectNotFound(message) - elif not (is_admin or trans.app.security_agent.can_access_library_item(current_user_roles, item, trans.user)): + elif not ( + is_admin or trans.app.security_agent.can_access_library_item(current_user_roles, item, trans.user) + ): if isinstance(item, trans.model.Library): - item_type = 'data library' + item_type = "data library" elif isinstance(item, trans.model.LibraryFolder): - item_type = 'folder' + item_type = "folder" else: - item_type = '(unknown item type)' + item_type = "(unknown item type)" message = f"You do not have permission to access the {escape(item_type)} with id ({str(item.id)})." raise ItemAccessibilityException(message) diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 0b131b4b3dc..f0647767282 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -2,13 +2,22 @@ import logging import signal import sys import time -from typing import Any, Callable, List, Tuple +from typing import ( + Any, + Callable, + List, + Tuple, +) import galaxy.model import galaxy.model.security import galaxy.queues import galaxy.security -from galaxy import auth, config, jobs +from galaxy import ( + auth, + config, + jobs, +) from galaxy.config_watchers import ConfigWatchers from galaxy.containers import build_container_interfaces from galaxy.datatypes.registry import Registry @@ -43,11 +52,14 @@ from galaxy.queue_worker import ( GalaxyQueueWorker, send_local_control_task, ) -from galaxy.quota import get_quota_agent, QuotaAgent +from galaxy.quota import ( + get_quota_agent, + QuotaAgent, +) from galaxy.security.idencoding import IdEncodingHelper from galaxy.security.vault import ( Vault, - VaultFactory + VaultFactory, ) from galaxy.tool_shed.galaxy_install.installed_repository_manager import InstalledRepositoryManager from galaxy.tool_shed.galaxy_install.update_repository_manager import UpdateRepositoryManager @@ -55,12 +67,15 @@ from galaxy.tool_util.deps.views import DependencyResolversView from galaxy.tool_util.verify.test_data import TestDataResolver from galaxy.tools.cache import ( ToolCache, - ToolShedRepositoryCache + ToolShedRepositoryCache, ) from galaxy.tools.data_manager.manager import DataManagers from galaxy.tools.error_reports import ErrorReports from galaxy.tools.special_tools import load_lib_tools -from galaxy.tours import build_tours_registry, ToursRegistry +from galaxy.tours import ( + build_tours_registry, + ToursRegistry, +) from galaxy.util import ( ExecutionTimer, heartbeat, @@ -72,11 +87,18 @@ from galaxy.visualization.genomes import Genomes from galaxy.visualization.plugins.registry import VisualizationsRegistry from galaxy.web import url_for from galaxy.web.proxy import ProxyManager -from galaxy.web_stack import application_stack_instance, ApplicationStack +from galaxy.web_stack import ( + application_stack_instance, + ApplicationStack, +) from galaxy.webhooks import WebhooksRegistry from galaxy.workflow.trs_proxy import TrsProxy from .di import Container -from .structured_app import BasicSharedApp, MinimalManagerApp, StructuredApp +from .structured_app import ( + BasicSharedApp, + MinimalManagerApp, + StructuredApp, +) log = logging.getLogger(__name__) app = None @@ -106,7 +128,13 @@ class SentryClientMixin: self.sentry_client = None if self.config.sentry_dsn: event_level = self.config.sentry_event_level.upper() - assert event_level in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], f"Invalid sentry event level '{self.config.sentry.event_level}'" + assert event_level in [ + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL", + ], f"Invalid sentry event level '{self.config.sentry.event_level}'" def postfork_sentry_client(): import sentry_sdk @@ -114,12 +142,12 @@ class SentryClientMixin: sentry_logging = LoggingIntegration( level=logging.INFO, # Capture info and above as breadcrumbs - event_level=getattr(logging, event_level) # Send errors as events + event_level=getattr(logging, event_level), # Send errors as events ) self.sentry_client = sentry_sdk.init( self.config.sentry_dsn, release=f"{self.config.version_major}.{self.config.version_minor}", - integrations=[sentry_logging] + integrations=[sentry_logging], ) self.application_stack.register_postfork_function(postfork_sentry_client) @@ -141,7 +169,7 @@ class MinimalGalaxyApplication(BasicSharedApp, config.ConfiguresGalaxyMixin, Hal # an appropriately configured logger in configure_logging below. logging.basicConfig(level=logging.DEBUG) log.debug("python path is: %s", ", ".join(sys.path)) - self.name = 'galaxy' + self.name = "galaxy" self.is_webapp = False self.new_installation = False # Read config file and check for errors @@ -150,7 +178,7 @@ class MinimalGalaxyApplication(BasicSharedApp, config.ConfiguresGalaxyMixin, Hal if configure_logging: config.configure_logging(self.config) self._configure_object_store(fsmon=True) - config_file = kwargs.get('global_conf', {}).get('__file__', None) + config_file = kwargs.get("global_conf", {}).get("__file__", None) if config_file: log.debug('Using "galaxy.ini" config file: %s', config_file) self._configure_models(check_migrate_databases=self.config.check_migrate_databases, config_file=config_file) @@ -165,7 +193,8 @@ class MinimalGalaxyApplication(BasicSharedApp, config.ConfiguresGalaxyMixin, Hal def configure_fluent_log(self): if self.config.fluent_log: from galaxy.util.custom_logging.fluent_log import FluentTraceLogger - self.trace_logger = FluentTraceLogger('galaxy', self.config.fluent_host, self.config.fluent_port) + + self.trace_logger = FluentTraceLogger("galaxy", self.config.fluent_host, self.config.fluent_port) else: self.trace_logger = None @@ -182,12 +211,16 @@ class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication): def __init__(self, **kwargs): super().__init__(**kwargs) self._register_singleton(MinimalManagerApp, self) - self.execution_timer_factory = self._register_singleton(ExecutionTimerFactory, ExecutionTimerFactory(self.config)) + self.execution_timer_factory = self._register_singleton( + ExecutionTimerFactory, ExecutionTimerFactory(self.config) + ) self.configure_fluent_log() self.application_stack = self._register_singleton(ApplicationStack, application_stack_instance(app=self)) # Initialize job metrics manager, needs to be in place before # config so per-destination modifications can be made. - self.job_metrics = self._register_singleton(JobMetrics, JobMetrics(self.config.job_metrics_config_file, app=self)) + self.job_metrics = self._register_singleton( + JobMetrics, JobMetrics(self.config.job_metrics_config_file, app=self) + ) # Initialize the job management configuration self.job_config = self._register_singleton(jobs.JobConfiguration) @@ -206,17 +239,22 @@ class GalaxyManagerApplication(MinimalManagerApp, MinimalGalaxyApplication): self.library_datasets_manager = self._register_singleton(LibraryDatasetsManager) self.role_manager = self._register_singleton(RoleManager) from galaxy.jobs.manager import JobManager + self.job_manager = self._register_singleton(JobManager) # ConfiguredFileSources - self.file_sources = self._register_singleton(ConfiguredFileSources, ConfiguredFileSources.from_app_config(self.config)) + self.file_sources = self._register_singleton( + ConfiguredFileSources, ConfiguredFileSources.from_app_config(self.config) + ) self.vault = self._register_singleton(Vault, VaultFactory.from_app(self)) # We need the datatype registry for running certain tasks that modify HDAs, and to build the registry we need # to setup the installed repositories ... this is not ideal self._configure_tool_config_files() - self.installed_repository_manager = self._register_singleton(InstalledRepositoryManager, InstalledRepositoryManager(self)) + self.installed_repository_manager = self._register_singleton( + InstalledRepositoryManager, InstalledRepositoryManager(self) + ) self._configure_datatypes_registry(self.installed_repository_manager) self._register_singleton(Registry, self.datatypes_registry) galaxy.model.set_datatypes_registry(self.datatypes_registry) @@ -244,7 +282,7 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication): self._register_singleton(StructuredApp, self) # A lot of postfork initialization depends on the server name, ensure it is set immediately after forking before other postfork functions self.application_stack.register_postfork_function(self.application_stack.set_postfork_server_name, self) - self.config.reload_sanitize_allowlist(explicit='sanitize_allowlist_file' in kwargs) + self.config.reload_sanitize_allowlist(explicit="sanitize_allowlist_file" in kwargs) self.amqp_internal_connection_obj = galaxy.queues.connection_from_config(self.config) # queue_worker *can* be initialized with a queue, but here we don't # want to and we'll allow postfork to bind and start it. @@ -252,8 +290,12 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication): self._configure_tool_shed_registry() - self.dependency_resolvers_view = self._register_singleton(DependencyResolversView, DependencyResolversView(self)) - self.test_data_resolver = self._register_singleton(TestDataResolver, TestDataResolver(file_dirs=self.config.tool_test_data_directories)) + self.dependency_resolvers_view = self._register_singleton( + DependencyResolversView, DependencyResolversView(self) + ) + self.test_data_resolver = self._register_singleton( + TestDataResolver, TestDataResolver(file_dirs=self.config.tool_test_data_directories) + ) self.dynamic_tool_manager = self._register_singleton(DynamicToolManager) self.api_keys_manager = self._register_singleton(ApiKeyManager) @@ -268,7 +310,9 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication): self.data_provider_registry = self._register_singleton(DataProviderRegistry) # Initialize error report plugins. - self.error_reports = self._register_singleton(ErrorReports, ErrorReports(self.config.error_report_file, app=self)) + self.error_reports = self._register_singleton( + ErrorReports, ErrorReports(self.config.error_report_file, app=self) + ) # Setup a Tool Cache self.tool_cache = self._register_singleton(ToolCache) @@ -279,7 +323,9 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication): # Load Data Manager self.data_managers = self._register_singleton(DataManagers) # Load the update repository manager. - self.update_repository_manager = self._register_singleton(UpdateRepositoryManager, UpdateRepositoryManager(self)) + self.update_repository_manager = self._register_singleton( + UpdateRepositoryManager, UpdateRepositoryManager(self) + ) # Load proprietary datatype converters and display applications. self.installed_repository_manager.load_proprietary_converters_and_display_applications() # Load datatype display applications defined in local datatypes_conf.xml @@ -292,10 +338,14 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication): load_lib_tools(self.toolbox) self.toolbox.persist_cache(register_postfork=True) # visualizations registry: associates resources with visualizations, controls how to render - self.visualizations_registry = self._register_singleton(VisualizationsRegistry, VisualizationsRegistry( - self, - directories_setting=self.config.visualization_plugins_directory, - template_cache_dir=self.config.template_cache_path)) + self.visualizations_registry = self._register_singleton( + VisualizationsRegistry, + VisualizationsRegistry( + self, + directories_setting=self.config.visualization_plugins_directory, + template_cache_dir=self.config.template_cache_path, + ), + ) # Tours registry tour_registry = build_tours_registry(self.config.tour_config_dir) self.tour_registry = tour_registry @@ -305,8 +355,8 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication): # Load security policy. self.security_agent = self.model.security_agent self.host_security_agent = galaxy.model.security.HostAgent( - model=self.security_agent.model, - permitted_actions=self.security_agent.permitted_actions) + model=self.security_agent.model, permitted_actions=self.security_agent.permitted_actions + ) # Load quota management. self.quota_agent = self._register_singleton(QuotaAgent, get_quota_agent(self.config, self.model)) # Heartbeat for thread profiling @@ -317,9 +367,7 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication): if self.config.use_heartbeat: if heartbeat.Heartbeat: self.heartbeat = heartbeat.Heartbeat( - self.config, - period=self.config.heartbeat_interval, - fname=self.config.heartbeat_log + self.config, period=self.config.heartbeat_interval, fname=self.config.heartbeat_log ) self.heartbeat.daemon = True self.application_stack.register_postfork_function(self.heartbeat.start) @@ -327,15 +375,15 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication): self.authnz_manager = None if self.config.enable_oidc: from galaxy.authnz import managers - self.authnz_manager = managers.AuthnzManager(self, - self.config.oidc_config_file, - self.config.oidc_backends_config_file) + + self.authnz_manager = managers.AuthnzManager( + self, self.config.oidc_config_file, self.config.oidc_backends_config_file + ) self.containers = {} if self.config.enable_beta_containers_interface: self.containers = build_container_interfaces( - self.config.containers_config_file, - containers_conf=self.config.containers_conf + self.config.containers_config_file, containers_conf=self.config.containers_conf ) if not self.config.enable_celery_tasks and self.config.history_audit_table_prune_interval > 0: @@ -344,7 +392,8 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication): name="HistoryAuditTablePruneTask", interval=self.config.history_audit_table_prune_interval, immediate_start=False, - time_execution=True) + time_execution=True, + ) self.application_stack.register_postfork_function(self.prune_history_audit_task.start) self.haltables.append(("HistoryAuditTablePruneTask", self.prune_history_audit_task.shutdown)) # Start the job manager @@ -355,6 +404,7 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication): self.proxy_manager = ProxyManager(self.config) from galaxy.workflow import scheduling_manager + # Must be initialized after job_config. self.workflow_scheduling_manager = scheduling_manager.WorkflowSchedulingManager(self) @@ -372,9 +422,7 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication): handlers[signal.SIGUSR1] = self.heartbeat.dump_signal_handler self._configure_signal_handlers(handlers) - self.database_heartbeat = DatabaseHeartbeat( - application_stack=self.application_stack - ) + self.database_heartbeat = DatabaseHeartbeat(application_stack=self.application_stack) self.database_heartbeat.add_change_callback(self.watchers.change_state) self.application_stack.register_postfork_function(self.database_heartbeat.start) @@ -382,7 +430,9 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication): self.application_stack.register_postfork_function(self.application_stack.start) self.application_stack.register_postfork_function(self.queue_worker.bind_and_start) # Delay toolbox index until after startup - self.application_stack.register_postfork_function(lambda: send_local_control_task(self, 'rebuild_toolbox_search_index')) + self.application_stack.register_postfork_function( + lambda: send_local_control_task(self, "rebuild_toolbox_search_index") + ) # Inject url_for for components to more easily optionally depend # on url_for. @@ -420,32 +470,33 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication): @property def is_job_handler(self) -> bool: - return (self.config.track_jobs_in_database and self.job_config.is_handler) or not self.config.track_jobs_in_database + return ( + self.config.track_jobs_in_database and self.job_config.is_handler + ) or not self.config.track_jobs_in_database class StatsdStructuredExecutionTimer(StructuredExecutionTimer): - def __init__(self, galaxy_statsd_client, *args, **kwds): self.galaxy_statsd_client = galaxy_statsd_client super().__init__(*args, **kwds) def to_str(self, **kwd): - self.galaxy_statsd_client.timing(self.timer_id, self.elapsed * 1000., kwd) + self.galaxy_statsd_client.timing(self.timer_id, self.elapsed * 1000.0, kwd) return super().to_str(**kwd) class ExecutionTimerFactory: - def __init__(self, config): statsd_host = getattr(config, "statsd_host", None) if statsd_host: from galaxy.web.framework.middleware.statsd import GalaxyStatsdClient + self.galaxy_statsd_client = GalaxyStatsdClient( statsd_host, - getattr(config, 'statsd_port', 8125), - getattr(config, 'statsd_prefix', 'galaxy'), - getattr(config, 'statsd_influxdb', False), - getattr(config, 'statsd_mock_calls', False), + getattr(config, "statsd_port", 8125), + getattr(config, "statsd_prefix", "galaxy"), + getattr(config, "statsd_influxdb", False), + getattr(config, "statsd_mock_calls", False), ) else: self.galaxy_statsd_client = None diff --git a/lib/galaxy/app_unittest_utils/celery_helper.py b/lib/galaxy/app_unittest_utils/celery_helper.py index fa4657ef065..d70e0eafb94 100644 --- a/lib/galaxy/app_unittest_utils/celery_helper.py +++ b/lib/galaxy/app_unittest_utils/celery_helper.py @@ -3,6 +3,7 @@ from functools import wraps def rebind_container_to_task(app): import galaxy.app + galaxy.app.app = app from galaxy.celery import ( CELERY_TASKS, @@ -15,6 +16,6 @@ def rebind_container_to_task(app): for task in CELERY_TASKS: task_fn = getattr(tasks, task, None) if task_fn: - task_fn = getattr(task_fn, '__wrapped__', task_fn) + task_fn = getattr(task_fn, "__wrapped__", task_fn) container_bound_task = magic_bind_dynamic(task_fn) setattr(tasks, task, container_bound_task) diff --git a/lib/galaxy/app_unittest_utils/galaxy_mock.py b/lib/galaxy/app_unittest_utils/galaxy_mock.py index 98e3574b028..53ece3ff6fa 100644 --- a/lib/galaxy/app_unittest_utils/galaxy_mock.py +++ b/lib/galaxy/app_unittest_utils/galaxy_mock.py @@ -18,9 +18,16 @@ from galaxy.model import tags from galaxy.model.base import SharedModelMapping from galaxy.model.mapping import GalaxyModelMapping from galaxy.model.scoped_session import galaxy_scoped_session -from galaxy.model.unittest_utils import GalaxyDataTestApp, GalaxyDataTestConfig +from galaxy.model.unittest_utils import ( + GalaxyDataTestApp, + GalaxyDataTestConfig, +) from galaxy.security import idencoding -from galaxy.structured_app import BasicSharedApp, MinimalManagerApp, StructuredApp +from galaxy.structured_app import ( + BasicSharedApp, + MinimalManagerApp, + StructuredApp, +) from galaxy.tool_util.deps.containers import NullContainerFinder from galaxy.tools.data import ToolDataTableManager from galaxy.util import StructuredExecutionTimer @@ -33,34 +40,34 @@ from .celery_helper import rebind_container_to_task # ============================================================================= def buildMockEnviron(**kwargs): environ = { - 'CONTENT_LENGTH': '0', - 'CONTENT_TYPE': '', - 'HTTP_ACCEPT': '*/*', - 'HTTP_ACCEPT_ENCODING': 'gzip, deflate', - 'HTTP_ACCEPT_LANGUAGE': 'en-US,en;q=0.8,zh;q=0.5,ja;q=0.3', - 'HTTP_CACHE_CONTROL': 'no-cache', - 'HTTP_CONNECTION': 'keep-alive', - 'HTTP_DNT': '1', - 'HTTP_HOST': 'localhost:8000', - 'HTTP_ORIGIN': 'http://localhost:8000', - 'HTTP_PRAGMA': 'no-cache', - 'HTTP_REFERER': 'http://localhost:8000', - 'HTTP_USER_AGENT': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:43.0) Gecko/20100101 Firefox/43.0', - 'PATH_INFO': '/', - 'QUERY_STRING': '', - 'REMOTE_ADDR': '127.0.0.1', - 'REQUEST_METHOD': 'GET', - 'SCRIPT_NAME': '', - 'SERVER_NAME': '127.0.0.1', - 'SERVER_PORT': '8080', - 'SERVER_PROTOCOL': 'HTTP/1.1' + "CONTENT_LENGTH": "0", + "CONTENT_TYPE": "", + "HTTP_ACCEPT": "*/*", + "HTTP_ACCEPT_ENCODING": "gzip, deflate", + "HTTP_ACCEPT_LANGUAGE": "en-US,en;q=0.8,zh;q=0.5,ja;q=0.3", + "HTTP_CACHE_CONTROL": "no-cache", + "HTTP_CONNECTION": "keep-alive", + "HTTP_DNT": "1", + "HTTP_HOST": "localhost:8000", + "HTTP_ORIGIN": "http://localhost:8000", + "HTTP_PRAGMA": "no-cache", + "HTTP_REFERER": "http://localhost:8000", + "HTTP_USER_AGENT": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:43.0) Gecko/20100101 Firefox/43.0", + "PATH_INFO": "/", + "QUERY_STRING": "", + "REMOTE_ADDR": "127.0.0.1", + "REQUEST_METHOD": "GET", + "SCRIPT_NAME": "", + "SERVER_NAME": "127.0.0.1", + "SERVER_PORT": "8080", + "SERVER_PROTOCOL": "HTTP/1.1", } environ.update(**kwargs) return environ class MockApp(di.Container, GalaxyDataTestApp): - config: 'MockAppConfig' + config: "MockAppConfig" def __init__(self, config=None, **kwargs): super().__init__() @@ -70,7 +77,7 @@ class MockApp(di.Container, GalaxyDataTestApp): self[MinimalManagerApp] = self self[StructuredApp] = self self[idencoding.IdEncodingHelper] = self.security - self.name = kwargs.get('name', 'galaxy') + self.name = kwargs.get("name", "galaxy") self[SharedModelMapping] = self.model self[GalaxyModelMapping] = self.model self[galaxy_scoped_session] = self.model.context @@ -79,10 +86,7 @@ class MockApp(di.Container, GalaxyDataTestApp): self[tags.GalaxyTagHandler] = self.tag_handler self.quota_agent = quota.DatabaseQuotaAgent(self.model) self.job_config = Bunch( - dynamic_params=None, - destinations={}, - use_messaging=False, - assign_handler=lambda *args, **kwargs: None + dynamic_params=None, destinations={}, use_messaging=False, assign_handler=lambda *args, **kwargs: None ) self.tool_data_tables = ToolDataTableManager(tool_data_path=self.config.tool_data_path) self.dataset_collections_service = None @@ -103,6 +107,7 @@ class MockApp(di.Container, GalaxyDataTestApp): def url_for(*args, **kwds): return "/mock/url" + self.url_for = url_for def wait_for_toolbox_reload(self, toolbox): @@ -120,19 +125,18 @@ class MockLock: class MockAppConfig(GalaxyDataTestConfig, CommonConfigurationMixin): - class MockSchema(Bunch): pass def __init__(self, **kwargs): super().__init__(**kwargs) self.schema = self.MockSchema() - self.use_remote_user = kwargs.get('use_remote_user', False) + self.use_remote_user = kwargs.get("use_remote_user", False) self.enable_celery_tasks = False - self.tool_data_path = os.path.join(self.root, 'tool-data') + self.tool_data_path = os.path.join(self.root, "tool-data") self.galaxy_data_manager_data_path = self.tool_data_path self.tool_dependency_dir = None - self.metadata_strategy = 'directory' + self.metadata_strategy = "directory" self.user_activation_on = False self.new_user_dataset_access_role_default_private = False @@ -152,8 +156,8 @@ class MockAppConfig(GalaxyDataTestConfig, CommonConfigurationMixin): self.redact_email_in_job_name = False # Follow two required by GenomeBuilds - self.len_file_path = os.path.join('tool-data', 'shared', 'ucsc', 'chrom') - self.builds_file_path = os.path.join('tool-data', 'shared', 'ucsc', 'builds.txt.sample') + self.len_file_path = os.path.join("tool-data", "shared", "ucsc", "chrom") + self.builds_file_path = os.path.join("tool-data", "shared", "ucsc", "builds.txt.sample") self.shed_tool_config_file = "config/shed_tool_conf.xml" self.shed_tool_config_file_set = False @@ -165,7 +169,7 @@ class MockAppConfig(GalaxyDataTestConfig, CommonConfigurationMixin): # set by MockDir self.enable_tool_document_cache = False - self.tool_cache_data_dir = os.path.join(self.root, 'tool_cache') + self.tool_cache_data_dir = os.path.join(self.root, "tool_cache") self.delay_tool_initialization = True self.external_chown_script = None self.check_job_script_integrity = False @@ -173,9 +177,9 @@ class MockAppConfig(GalaxyDataTestConfig, CommonConfigurationMixin): self.check_job_script_integrity_sleep = 0 self.default_panel_view = "default" - self.panel_views_dir = '' + self.panel_views_dir = "" self.panel_views = {} - self.edam_panel_views = '' + self.edam_panel_views = "" self.config_file = None @@ -189,7 +193,7 @@ class MockAppConfig(GalaxyDataTestConfig, CommonConfigurationMixin): self.enable_tool_shed_check = False self.monitor_thread_join_timeout = 1 self.integrated_tool_panel_config = None - self.vault_config_file = kwargs.get('vault_config_file') + self.vault_config_file = kwargs.get("vault_config_file") self.max_discovered_files = 10000 @property @@ -198,23 +202,21 @@ class MockAppConfig(GalaxyDataTestConfig, CommonConfigurationMixin): def __getattr__(self, name): # Handle the automatic [option]_set options: for tests, assume none are set - if name == 'is_set': + if name == "is_set": return lambda x: False # Handle the automatic config file _set options - if name.endswith('_file_set'): + if name.endswith("_file_set"): return False raise AttributeError(name) class MockWebapp: - def __init__(self, security: idencoding.IdEncodingHelper, **kwargs): - self.name = kwargs.get('name', 'galaxy') + self.name = kwargs.get("name", "galaxy") self.security = security class MockTrans: - def __init__(self, app=None, user=None, history=None, **kwargs): self.app = app or MockApp(**kwargs) self.model = self.app.model @@ -273,14 +275,13 @@ class MockTrans: class MockVisualizationsRegistry: - BUILT_IN_VISUALIZATIONS = ['trackster'] + BUILT_IN_VISUALIZATIONS = ["trackster"] def get_visualizations(self, trans, target): return [] class MockDir: - def __init__(self, structure_dict, where=None): self.structure_dict = structure_dict self.create_root(structure_dict, where) @@ -301,7 +302,7 @@ class MockDir: self.create_structure(subdir_path, v) def create_file(self, path, contents): - with open(path, 'w') as newfile: + with open(path, "w") as newfile: newfile.write(contents) def remove(self): diff --git a/lib/galaxy/app_unittest_utils/tools_support.py b/lib/galaxy/app_unittest_utils/tools_support.py index 349bc291350..c36ac90911b 100644 --- a/lib/galaxy/app_unittest_utils/tools_support.py +++ b/lib/galaxy/app_unittest_utils/tools_support.py @@ -17,14 +17,12 @@ from galaxy.tool_util.parser import get_tool_source from galaxy.tools import create_tool_from_source from galaxy.util.bunch import Bunch - datatypes_registry = galaxy.datatypes.registry.Registry() datatypes_registry.load_datatypes() galaxy.model.set_datatypes_registry(datatypes_registry) class UsesApp: - def setup_app(self): self.test_directory = tempfile.mkdtemp() self.app = MockApp() @@ -37,7 +35,7 @@ class UsesApp: # Simple tool with just one text parameter and output. -SIMPLE_TOOL_CONTENTS = ''' +SIMPLE_TOOL_CONTENTS = """ echo "$param1" < $out1 @@ -46,11 +44,11 @@ SIMPLE_TOOL_CONTENTS = ''' -''' +""" # A tool with data parameters (kind of like cat1) my favorite test tool :) -SIMPLE_CAT_TOOL_CONTENTS = ''' +SIMPLE_CAT_TOOL_CONTENTS = """ cat "$param1" #for $r in $repeat# "$r.param2" #end for# < $out1 @@ -62,11 +60,10 @@ SIMPLE_CAT_TOOL_CONTENTS = ''' -''' +""" class MockActionI: - def execute(self, tool, trans, **kwds): pass @@ -122,7 +119,6 @@ class UsesTools(UsesApp): class MockContext: - def __init__(self, model_objects=None): self.expunged_all = False self.flushed = False @@ -144,7 +140,6 @@ class MockContext: class MockQuery: - def __init__(self, class_objects): self.class_objects = class_objects @@ -155,4 +150,4 @@ class MockQuery: return self.class_objects.get(id, None) -__all__ = ('UsesApp', ) +__all__ = ("UsesApp",) diff --git a/lib/galaxy/auth/__init__.py b/lib/galaxy/auth/__init__.py index 064ae8af2af..1f4aa9986c6 100644 --- a/lib/galaxy/auth/__init__.py +++ b/lib/galaxy/auth/__init__.py @@ -3,7 +3,10 @@ Contains implementations of the authentication logic. """ import logging -from galaxy.auth.util import get_authenticators, parse_auth_results +from galaxy.auth.util import ( + get_authenticators, + parse_auth_results, +) from galaxy.exceptions import Conflict from galaxy.util import string_as_bool @@ -11,30 +14,29 @@ log = logging.getLogger(__name__) class AuthManager: - def __init__(self, config): self.redact_username_in_logs = config.redact_username_in_logs - self.authenticators = get_authenticators(config.auth_config_file, config.is_set('auth_config_file')) + self.authenticators = get_authenticators(config.auth_config_file, config.is_set("auth_config_file")) def check_registration_allowed(self, email, username, password): """Checks if the provided email/username is allowed to register.""" - message = '' - status = 'done' + message = "" + status = "done" for provider, options in self.active_authenticators(email, username, password): allow_reg = _get_allow_register(options) - if allow_reg == 'challenge': + if allow_reg == "challenge": auth_results = provider.authenticate(email, username, password, options) if auth_results[0] is True: break if auth_results[0] is None: - message = 'Invalid email address/username or password.' - status = 'error' + message = "Invalid email address/username or password." + status = "error" break elif allow_reg is True: break elif allow_reg is False: - message = 'Account registration not required for your account. Please simply login.' - status = 'error' + message = "Account registration not required for your account. Please simply login." + status = "error" break return message, status @@ -43,22 +45,18 @@ class AuthManager: Checks the username/email & password using auth providers in order. If a match is found, returns the 'auto-register' option for that provider. """ - if '@' in login: + if "@" in login: email = login username = None else: email = None username = login - auth_return = { - "auto_reg": False, - "email": "", - "username": "" - } + auth_return = {"auto_reg": False, "email": "", "username": ""} for provider, options in self.active_authenticators(email, username, password): if provider is None: log.debug(f"Unable to find module: {options}") else: - options['no_password_check'] = no_password_check + options["no_password_check"] = no_password_check auth_results = provider.authenticate(email, username, password, options) if auth_results[0] is True: try: @@ -98,10 +96,10 @@ class AuthManager: if string_as_bool(options.get("allow-password-change", False)): return else: - return 'Password change not supported.' + return "Password change not supported." elif auth_result is None: break # end authentication (skip rest) - return 'Invalid current password.' + return "Invalid current password." def active_authenticators(self, email, username, password): """Yields AuthProvider instances for the provided configfile that match the @@ -112,11 +110,11 @@ class AuthManager: filter_template = authenticator.filter_template if filter_template: filter_str = filter_template.format(email=email, username=username, password=password) - passed_filter = eval(filter_str, {"__builtins__": None}, {'str': str}) + passed_filter = eval(filter_str, {"__builtins__": None}, {"str": str}) if not passed_filter: continue # skip to next options = authenticator.options - options['redact_username_in_logs'] = self.redact_username_in_logs + options["redact_username_in_logs"] = self.redact_username_in_logs yield authenticator.plugin, options except Exception: log.exception("Active Authenticators Failure") @@ -124,9 +122,9 @@ class AuthManager: def _get_allow_register(d): - s = d.get('allow-register', True) + s = d.get("allow-register", True) lower_s = str(s).lower() - if lower_s == 'challenge': + if lower_s == "challenge": return lower_s else: return string_as_bool(s) diff --git a/lib/galaxy/auth/providers/__init__.py b/lib/galaxy/auth/providers/__init__.py index 220fdf24898..f85051dabcb 100644 --- a/lib/galaxy/auth/providers/__init__.py +++ b/lib/galaxy/auth/providers/__init__.py @@ -11,7 +11,7 @@ class AuthProvider(metaclass=abc.ABCMeta): @abc.abstractproperty def plugin_type(self): - """ Short string providing labelling this plugin """ + """Short string providing labelling this plugin""" @abc.abstractmethod def authenticate(self, email, username, password, options): diff --git a/lib/galaxy/auth/providers/alwaysreject.py b/lib/galaxy/auth/providers/alwaysreject.py index e39990ae977..b4ffa043fbe 100644 --- a/lib/galaxy/auth/providers/alwaysreject.py +++ b/lib/galaxy/auth/providers/alwaysreject.py @@ -14,13 +14,14 @@ class AlwaysReject(AuthProvider): """A simple authenticator that just accepts users (does not care about their password). """ - plugin_type = 'alwaysreject' + + plugin_type = "alwaysreject" def authenticate(self, email, username, password, options): """ See abstract method documentation. """ - return (None, '', '') + return (None, "", "") def authenticate_user(self, user, password, options): """ @@ -30,4 +31,4 @@ class AlwaysReject(AuthProvider): return None -__all__ = ('AlwaysReject', ) +__all__ = ("AlwaysReject",) diff --git a/lib/galaxy/auth/providers/ldap_ad.py b/lib/galaxy/auth/providers/ldap_ad.py index dac936f7386..fc7eef2af5b 100644 --- a/lib/galaxy/auth/providers/ldap_ad.py +++ b/lib/galaxy/auth/providers/ldap_ad.py @@ -43,11 +43,16 @@ def _parse_ldap_options(options_unparsed): try: key, value = opt.split("=") except ValueError: - log.warning("LDAP authenticate: Invalid syntax '%s' inside element. Syntax should be option1=value1,option2=value2", opt) + log.warning( + "LDAP authenticate: Invalid syntax '%s' inside element. Syntax should be option1=value1,option2=value2", + opt, + ) continue if not key.startswith(prefix): - log.warning("LDAP authenticate: Invalid LDAP option '%s'. '%s' doesn't start with prefix '%s'", opt, key, prefix) + log.warning( + "LDAP authenticate: Invalid LDAP option '%s'. '%s' doesn't start with prefix '%s'", opt, key, prefix + ) continue try: key = getattr(ldap, key) @@ -58,7 +63,9 @@ def _parse_ldap_options(options_unparsed): try: value = getattr(ldap, value) except AttributeError: - log.warning("LDAP authenticate: Invalid LDAP option '%s'. '%s' is not available in module ldap", opt, value) + log.warning( + "LDAP authenticate: Invalid LDAP option '%s'. '%s' is not available in module ldap", opt, value + ) continue pair = (key, value) log.debug("LDAP authenticate: Valid LDAP option pair '%s' -> '%s=%s'", opt, *pair) @@ -76,8 +83,9 @@ class LDAP(AuthProvider): those fields first. After that it will bind to LDAP with the username (formatted as specified). """ - plugin_type = 'ldap' - role_search_option = 'auto-register-roles' + + plugin_type = "ldap" + role_search_option = "auto-register-roles" def __init__(self): super().__init__() @@ -86,32 +94,36 @@ class LDAP(AuthProvider): def check_config(self, username, email, options): ok = True - if options.get('continue-on-failure', 'False') == 'False': + if options.get("continue-on-failure", "False") == "False": failure_mode = None # reject and do not continue else: failure_mode = False # reject but continue - if string_as_bool(options.get('login-use-username', False)): + if string_as_bool(options.get("login-use-username", False)): if not username: - log.debug('LDAP authenticate: username must be used to login, cannot be None') + log.debug("LDAP authenticate: username must be used to login, cannot be None") return ok, failure_mode else: if not email: - log.debug('LDAP authenticate: email must be used to login, cannot be None') + log.debug("LDAP authenticate: email must be used to login, cannot be None") return ok, failure_mode - auto_create_roles = string_as_bool(options.get('auto-create-roles', False)) - auto_create_groups = string_as_bool(options.get('auto-create-groups', False)) + auto_create_roles = string_as_bool(options.get("auto-create-roles", False)) + auto_create_groups = string_as_bool(options.get("auto-create-groups", False)) self.auto_create_roles_or_groups = auto_create_roles or auto_create_groups - auto_assign_roles_to_groups_only = string_as_bool(options.get('auto-assign-roles-to-groups-only', False)) + auto_assign_roles_to_groups_only = string_as_bool(options.get("auto-assign-roles-to-groups-only", False)) if auto_assign_roles_to_groups_only and not (auto_create_roles and auto_create_groups): - raise ConfigurationError("If 'auto-assign-roles-to-groups-only' is True, auto-create-roles and " - "auto-create-groups have to be True as well.") + raise ConfigurationError( + "If 'auto-assign-roles-to-groups-only' is True, auto-create-roles and " + "auto-create-groups have to be True as well." + ) self.role_search_attribute = options.get(self.role_search_option) if self.auto_create_roles_or_groups and self.role_search_attribute is None: - raise ConfigurationError("If 'auto-create-roles' or 'auto-create-groups' is True, a '%s' attribute has to" - " be provided." % self.role_search_option) + raise ConfigurationError( + "If 'auto-create-roles' or 'auto-create-groups' is True, a '%s' attribute has to" + " be provided." % self.role_search_option + ) return ok, failure_mode @@ -123,10 +135,10 @@ class LDAP(AuthProvider): if not config_ok: return failure_mode, None - params = {'email': email, 'username': username} + params = {"email": email, "username": username} try: - ldap_options_raw = _get_subs(options, 'ldap-options', params) + ldap_options_raw = _get_subs(options, "ldap-options", params) except ConfigurationError: ldap_options = () else: @@ -139,34 +151,37 @@ class LDAP(AuthProvider): for opt in ldap_options: ldap.set_option(*opt) except Exception: - log.exception('LDAP authenticate: set_option exception') + log.exception("LDAP authenticate: set_option exception") return (failure_mode, None) - if 'search-fields' in options: + if "search-fields" in options: try: - conn = ldap.initialize(_get_subs(options, 'server', params)) + conn = ldap.initialize(_get_subs(options, "server", params)) conn.protocol_version = 3 - if 'search-user' in options: + if "search-user" in options: conn.simple_bind_s( - _get_subs(options, 'search-user', params), - _get_subs(options, 'search-password', params)) + _get_subs(options, "search-user", params), _get_subs(options, "search-password", params) + ) else: conn.simple_bind_s() # setup search - attributes = {_.strip().format(**params) for _ in options['search-fields'].split(',')} - if 'search-memberof-filter' in options: - attributes.add('memberOf') + attributes = {_.strip().format(**params) for _ in options["search-fields"].split(",")} + if "search-memberof-filter" in options: + attributes.add("memberOf") suser = conn.search_ext_s( - _get_subs(options, 'search-base', params), + _get_subs(options, "search-base", params), ldap.SCOPE_SUBTREE, - _get_subs(options, 'search-filter', params), attributes, - timeout=60, sizelimit=1) + _get_subs(options, "search-filter", params), + attributes, + timeout=60, + sizelimit=1, + ) # parse results if suser is None or len(suser) == 0: - log.warning('LDAP authenticate: search returned no results') + log.warning("LDAP authenticate: search returned no results") return (failure_mode, None) dn, attrs = suser[0] log.debug("LDAP authenticate: dn is %s", dn) @@ -175,7 +190,7 @@ class LDAP(AuthProvider): if self.role_search_attribute and attr == self.role_search_attribute[1:-1]: # strip curly brackets # keep role names as list params[self.role_search_option] = [unicodify(_) for _ in attrs[attr]] - elif attr == 'memberOf': + elif attr == "memberOf": params[attr] = [unicodify(_) for _ in attrs[attr]] elif attr in attrs: params[attr] = unicodify(attrs[attr][0]) @@ -183,12 +198,13 @@ class LDAP(AuthProvider): params[attr] = "" if self.auto_create_roles_or_groups and self.role_search_option not in params: - raise ConfigurationError("Missing or mismatching LDAP parameters for %s. Make sure the %s is " - "included in the 'search-fields'." % - (self.role_search_option, self.role_search_attribute)) - params['dn'] = dn + raise ConfigurationError( + "Missing or mismatching LDAP parameters for %s. Make sure the %s is " + "included in the 'search-fields'." % (self.role_search_option, self.role_search_attribute) + ) + params["dn"] = dn except Exception: - log.exception('LDAP authenticate: search exception') + log.exception("LDAP authenticate: search exception") return (failure_mode, None) return failure_mode, params @@ -197,7 +213,7 @@ class LDAP(AuthProvider): """ See abstract method documentation. """ - if not options['redact_username_in_logs']: + if not options["redact_username_in_logs"]: log.debug("LDAP authenticate: email is %s", email) log.debug("LDAP authenticate: username is %s", username) @@ -205,39 +221,41 @@ class LDAP(AuthProvider): failure_mode, params = self.ldap_search(email, username, options) if not params: - return failure_mode, '', '' + return failure_mode, "", "" # allow to skip authentication to allow for pre-populating users - if not options.get('no_password_check', False): - params['password'] = password + if not options.get("no_password_check", False): + params["password"] = password if not self._authenticate(params, options): - return failure_mode, '', '' + return failure_mode, "", "" # check whether the user is a member of a specified group/domain/... - if 'search-memberof-filter' in options: - search_filter = _get_subs(options, 'search-memberof-filter', params) - if not any(search_filter in ad_node_name for ad_node_name in params['memberOf']): - return failure_mode, '', '' + if "search-memberof-filter" in options: + search_filter = _get_subs(options, "search-memberof-filter", params) + if not any(search_filter in ad_node_name for ad_node_name in params["memberOf"]): + return failure_mode, "", "" attributes = {} if self.auto_create_roles_or_groups: - attributes['roles'] = params[self.role_search_option] - return (True, - _get_subs(options, 'auto-register-email', params), - transform_publicname(_get_subs(options, 'auto-register-username', params)), - attributes) + attributes["roles"] = params[self.role_search_option] + return ( + True, + _get_subs(options, "auto-register-email", params), + transform_publicname(_get_subs(options, "auto-register-username", params)), + attributes, + ) def _authenticate(self, params, options): """ Do the actual authentication by binding as the user to check their credentials """ try: - conn = ldap.initialize(_get_subs(options, 'server', params)) + conn = ldap.initialize(_get_subs(options, "server", params)) conn.protocol_version = 3 - bind_user = _get_subs(options, 'bind-user', params) - bind_password = _get_subs(options, 'bind-password', params) + bind_user = _get_subs(options, "bind-user", params) + bind_password = _get_subs(options, "bind-password", params) except Exception: - log.exception('LDAP authenticate: initialize exception') + log.exception("LDAP authenticate: initialize exception") return False try: conn.simple_bind_s(bind_user, bind_password) @@ -248,13 +266,13 @@ class LDAP(AuthProvider): pass else: if whoami is None: - raise RuntimeError('LDAP authenticate: anonymous bind') - if not options['redact_username_in_logs']: + raise RuntimeError("LDAP authenticate: anonymous bind") + if not options["redact_username_in_logs"]: log.debug("LDAP authenticate: whoami is %s", whoami) except Exception as e: - log.info('LDAP authenticate: bind exception: %s', unicodify(e)) + log.info("LDAP authenticate: bind exception: %s", unicodify(e)) return False - log.debug('LDAP authentication successful') + log.debug("LDAP authentication successful") return True def authenticate_user(self, user, password, options): @@ -265,9 +283,10 @@ class LDAP(AuthProvider): class ActiveDirectory(LDAP): - """ Effectively just an alias for LDAP auth, but may contain active directory specific - logic in the future. """ - plugin_type = 'activedirectory' + """Effectively just an alias for LDAP auth, but may contain active directory specific + logic in the future.""" + + plugin_type = "activedirectory" -__all__ = ('LDAP', 'ActiveDirectory') +__all__ = ("LDAP", "ActiveDirectory") diff --git a/lib/galaxy/auth/providers/localdb.py b/lib/galaxy/auth/providers/localdb.py index d17d865a7dc..168800a3e06 100644 --- a/lib/galaxy/auth/providers/localdb.py +++ b/lib/galaxy/auth/providers/localdb.py @@ -12,13 +12,14 @@ log = logging.getLogger(__name__) class LocalDB(AuthProvider): """Authenticate users against the local Galaxy database (as per usual).""" - plugin_type = 'localdb' + + plugin_type = "localdb" def authenticate(self, email, username, password, options): """ See abstract method documentation. """ - return (False, '', '') # it can never auto-create based of localdb (chicken-egg) + return (False, "", "") # it can never auto-create based of localdb (chicken-egg) def authenticate_user(self, user, password, options): """ @@ -29,4 +30,4 @@ class LocalDB(AuthProvider): return user_ok -__all__ = ('LocalDB', ) +__all__ = ("LocalDB",) diff --git a/lib/galaxy/auth/providers/pam_auth.py b/lib/galaxy/auth/providers/pam_auth.py index 69a56186935..48794157bd2 100644 --- a/lib/galaxy/auth/providers/pam_auth.py +++ b/lib/galaxy/auth/providers/pam_auth.py @@ -8,7 +8,7 @@ import shlex from galaxy.util import ( commands, - string_as_bool + string_as_bool, ) from ..providers import AuthProvider @@ -56,35 +56,37 @@ Configuration example (for internal authentication, use email for user details): class PAM(AuthProvider): - plugin_type = 'PAM' + plugin_type = "PAM" def authenticate(self, email, username, password, options): pam_username = None auto_register_username = None auto_register_email = None force_fail = False - if not options['redact_username_in_logs']: - log.debug(f"use username: {options.get('login-use-username')} use email {options.get('login-use-email', False)} email {email} username {username}") + if not options["redact_username_in_logs"]: + log.debug( + f"use username: {options.get('login-use-username')} use email {options.get('login-use-email', False)} email {email} username {username}" + ) # check email based login first because if email exists in Galaxy DB # we will be given the "public name" as username - if string_as_bool(options.get('login-use-email', False)) and email is not None: - if '@' in email: - (email_user, email_domain) = email.split('@') + if string_as_bool(options.get("login-use-email", False)) and email is not None: + if "@" in email: + (email_user, email_domain) = email.split("@") pam_username = email_user - if email_domain == options.get('maildomain', None): + if email_domain == options.get("maildomain", None): auto_register_email = email if username is not None: auto_register_username = username else: auto_register_username = email_user else: - log.debug('PAM authenticate: warning: email does not match configured PAM maildomain') + log.debug("PAM authenticate: warning: email does not match configured PAM maildomain") # no need to fail: if auto-register is not enabled, this # might still be a valid user else: - log.debug('PAM authenticate: email must be used to login, but no valid email found') + log.debug("PAM authenticate: email must be used to login, but no valid email found") force_fail = True - elif string_as_bool(options.get('login-use-username', False)): + elif string_as_bool(options.get("login-use-username", False)): # if we get here via authenticate_user then # user will be "public name" and # email address will be as per registered user @@ -92,42 +94,44 @@ class PAM(AuthProvider): pam_username = username if email is not None: auto_register_email = email - elif options.get('maildomain', None) is not None: + elif options.get("maildomain", None) is not None: # we can register a user with this username and mail domain # if auto registration is enabled auto_register_email = f"{username}@{options['maildomain']}" auto_register_username = username else: - log.debug('PAM authenticate: username login selected but no username provided') + log.debug("PAM authenticate: username login selected but no username provided") force_fail = True else: - log.debug('PAM authenticate: could not find username for PAM') + log.debug("PAM authenticate: could not find username for PAM") force_fail = True if force_fail: - return None, '', '' + return None, "", "" - pam_service = options.get('pam-service', 'galaxy') - use_helper = string_as_bool(options.get('use-external-helper', False)) + pam_service = options.get("pam-service", "galaxy") + use_helper = string_as_bool(options.get("use-external-helper", False)) log.debug(f"PAM auth: will use external helper: {use_helper}") authenticated = False if use_helper: - authentication_helper = options.get('authentication-helper-script', '/bin/false').strip() + authentication_helper = options.get("authentication-helper-script", "/bin/false").strip() log.debug(f"PAM auth: external helper script: {authentication_helper}") - if not authentication_helper.startswith('/'): + if not authentication_helper.startswith("/"): # don't accept relative path authenticated = False else: - auth_cmd = shlex.split(f'/usr/bin/sudo -n {authentication_helper}') + auth_cmd = shlex.split(f"/usr/bin/sudo -n {authentication_helper}") log.debug(f"PAM auth: external helper cmd: {auth_cmd}") - message = f'{pam_service}\n{pam_username}\n{password}\n' + message = f"{pam_service}\n{pam_username}\n{password}\n" try: output = commands.execute(auth_cmd, input=message) except commands.CommandLineException as e: - if e.stderr != '': - log.debug(f"PAM auth: external authentication script had errors: status {e.returncode} error {e.stderr}") + if e.stderr != "": + log.debug( + f"PAM auth: external authentication script had errors: status {e.returncode} error {e.stderr}" + ) output = e.stdout - if output.strip() == 'True': + if output.strip() == "True": authenticated = True else: authenticated = False @@ -135,21 +139,25 @@ class PAM(AuthProvider): try: import pam except ImportError: - log.debug('PAM authenticate: could not load pam module, PAM authentication disabled') - return None, '', '' + log.debug("PAM authenticate: could not load pam module, PAM authentication disabled") + return None, "", "" p_auth = pam.pam() authenticated = p_auth.authenticate(pam_username, password, service=pam_service) if authenticated: - log.debug(f"PAM authentication successful for {'redacted' if options['redact_username_in_logs'] else pam_username}") + log.debug( + f"PAM authentication successful for {'redacted' if options['redact_username_in_logs'] else pam_username}" + ) return True, auto_register_email, auto_register_username else: - log.debug(f"PAM authentication failed for {'redacted' if options['redact_username_in_logs'] else pam_username}") - return False, '', '' + log.debug( + f"PAM authentication failed for {'redacted' if options['redact_username_in_logs'] else pam_username}" + ) + return False, "", "" def authenticate_user(self, user, password, options): return self.authenticate(user.email, user.username, password, options)[0] -__all__ = ('PAM', ) +__all__ = ("PAM",) diff --git a/lib/galaxy/auth/util.py b/lib/galaxy/auth/util.py index 7f9f0f2dd1b..78168fda7de 100644 --- a/lib/galaxy/auth/util.py +++ b/lib/galaxy/auth/util.py @@ -12,7 +12,6 @@ from galaxy.util import ( string_as_bool, ) - log = logging.getLogger(__name__) AUTH_CONF_XML = """ @@ -26,11 +25,11 @@ AUTH_CONF_XML = """ """ -Authenticator = namedtuple('Authenticator', ['plugin', 'filter_template', 'options']) +Authenticator = namedtuple("Authenticator", ["plugin", "filter_template", "options"]) def get_authenticators(auth_config_file, auth_config_file_set): - __plugins_dict = plugin_config.plugins_dict(galaxy.auth.providers, 'plugin_type') + __plugins_dict = plugin_config.plugins_dict(galaxy.auth.providers, "plugin_type") # parse XML try: ct = parse_xml(auth_config_file) @@ -44,21 +43,23 @@ def get_authenticators(auth_config_file, auth_config_file_set): authenticators = [] # process authenticators for auth_elem in conf_root: - type_elem_text = auth_elem.find('type').text + type_elem_text = auth_elem.find("type").text plugin_class = __plugins_dict.get(type_elem_text) if not plugin_class: - raise Exception(f"Authenticator type '{type_elem_text}' not recognized, should be one of {', '.join(__plugins_dict)}") + raise Exception( + f"Authenticator type '{type_elem_text}' not recognized, should be one of {', '.join(__plugins_dict)}" + ) plugin = plugin_class() # check filterelem - filter_elem = auth_elem.find('filter') + filter_elem = auth_elem.find("filter") if filter_elem is not None: filter_template = str(filter_elem.text) else: filter_template = None # extract options - options_elem = auth_elem.find('options') + options_elem = auth_elem.find("options") options = {} if options_elem is not None: for opt in options_elem: @@ -77,23 +78,24 @@ def parse_auth_results(trans, auth_results, options): auth_result, auto_email, auto_username = auth_results[:3] auto_username = str(auto_username).lower() # make username unique - if validate_publicname(trans, auto_username) != '': + if validate_publicname(trans, auto_username) != "": i = 1 while i <= 10: # stop after 10 tries - if validate_publicname(trans, "%s-%i" % (auto_username, i)) == '': + if validate_publicname(trans, "%s-%i" % (auto_username, i)) == "": auto_username = "%s-%i" % (auto_username, i) break i += 1 else: raise Conflict("Cannot make unique username") log.debug(f"Email: {auto_email}, auto-register with username: {auto_username}") - auth_return["auto_reg"] = string_as_bool(options.get('auto-register', False)) + auth_return["auto_reg"] = string_as_bool(options.get("auto-register", False)) auth_return["email"] = auto_email auth_return["username"] = auto_username - auth_return["auto_create_roles"] = string_as_bool(options.get('auto-create-roles', False)) - auth_return["auto_create_groups"] = string_as_bool(options.get('auto-create-groups', False)) + auth_return["auto_create_roles"] = string_as_bool(options.get("auto-create-roles", False)) + auth_return["auto_create_groups"] = string_as_bool(options.get("auto-create-groups", False)) auth_return["auto_assign_roles_to_groups_only"] = string_as_bool( - options.get('auto-assign-roles-to-groups-only', False)) + options.get("auto-assign-roles-to-groups-only", False) + ) if len(auth_results) == 4: auth_return["attributes"] = auth_results[3] diff --git a/lib/galaxy/authnz/custos_authnz.py b/lib/galaxy/authnz/custos_authnz.py index 579006246b2..f3aa897a7c1 100644 --- a/lib/galaxy/authnz/custos_authnz.py +++ b/lib/galaxy/authnz/custos_authnz.py @@ -3,7 +3,10 @@ import hashlib import json import logging import os -from datetime import datetime, timedelta +from datetime import ( + datetime, + timedelta, +) from urllib.parse import quote import jwt @@ -11,55 +14,61 @@ import requests from oauthlib.common import generate_nonce from requests_oauthlib import OAuth2Session -from galaxy import exceptions -from galaxy import util -from galaxy.model import CustosAuthnzToken, User +from galaxy import ( + exceptions, + util, +) +from galaxy.model import ( + CustosAuthnzToken, + User, +) from ..authnz import IdentityProvider log = logging.getLogger(__name__) -STATE_COOKIE_NAME = 'galaxy-oidc-state' -NONCE_COOKIE_NAME = 'galaxy-oidc-nonce' -KEYCLOAK_BACKENDS = {'custos', 'cilogon', 'keycloak'} +STATE_COOKIE_NAME = "galaxy-oidc-state" +NONCE_COOKIE_NAME = "galaxy-oidc-nonce" +KEYCLOAK_BACKENDS = {"custos", "cilogon", "keycloak"} class CustosAuthnz(IdentityProvider): def __init__(self, provider, oidc_config, oidc_backend_config, idphint=None): provider = provider.lower() - self.config = {'provider': provider} - self.config['verify_ssl'] = oidc_config['VERIFY_SSL'] - self.config['url'] = oidc_backend_config['url'] - self.config['client_id'] = oidc_backend_config['client_id'] - self.config['client_secret'] = oidc_backend_config['client_secret'] - self.config['redirect_uri'] = oidc_backend_config['redirect_uri'] - self.config['ca_bundle'] = oidc_backend_config.get('ca_bundle', None) - self.config['extra_params'] = { - 'kc_idp_hint': oidc_backend_config.get('idphint', 'oidc' if self.config['provider'] in ['custos', 'keycloak'] else 'cilogon') + self.config = {"provider": provider} + self.config["verify_ssl"] = oidc_config["VERIFY_SSL"] + self.config["url"] = oidc_backend_config["url"] + self.config["client_id"] = oidc_backend_config["client_id"] + self.config["client_secret"] = oidc_backend_config["client_secret"] + self.config["redirect_uri"] = oidc_backend_config["redirect_uri"] + self.config["ca_bundle"] = oidc_backend_config.get("ca_bundle", None) + self.config["extra_params"] = { + "kc_idp_hint": oidc_backend_config.get( + "idphint", "oidc" if self.config["provider"] in ["custos", "keycloak"] else "cilogon" + ) } - if provider == 'cilogon': + if provider == "cilogon": self._load_config_for_cilogon() - elif provider == 'custos': + elif provider == "custos": self._load_config_for_custos() - elif provider == 'keycloak': + elif provider == "keycloak": self._load_config_for_keycloak() def _decode_token_no_signature(self, token): - return jwt.decode(token, audience=self.config['client_id'], options={"verify_signature": False}) + return jwt.decode(token, audience=self.config["client_id"], options={"verify_signature": False}) def authenticate(self, trans, idphint=None): - base_authorize_url = self.config['authorization_endpoint'] - scopes = ['openid', 'email', 'profile'] - if self.config['provider'] in ['custos', 'cilogon']: - scopes.append('org.cilogon.userinfo') + base_authorize_url = self.config["authorization_endpoint"] + scopes = ["openid", "email", "profile"] + if self.config["provider"] in ["custos", "cilogon"]: + scopes.append("org.cilogon.userinfo") oauth2_session = self._create_oauth2_session(scope=scopes) nonce = generate_nonce() nonce_hash = self._hash_nonce(nonce) extra_params = {"nonce": nonce_hash} if idphint is not None: - extra_params['idphint'] = idphint + extra_params["idphint"] = idphint if "extra_params" in self.config: - extra_params.update(self.config['extra_params']) - authorization_url, state = oauth2_session.authorization_url( - base_authorize_url, **extra_params) + extra_params.update(self.config["extra_params"]) + authorization_url, state = oauth2_session.authorization_url(base_authorize_url, **extra_params) trans.set_cookie(value=state, name=STATE_COOKIE_NAME) trans.set_cookie(value=nonce, name=NONCE_COOKIE_NAME) return authorization_url @@ -71,29 +80,31 @@ class CustosAuthnz(IdentityProvider): state_cookie = trans.get_cookie(name=STATE_COOKIE_NAME) oauth2_session = self._create_oauth2_session(state=state_cookie) token = self._fetch_token(oauth2_session, trans) - access_token = token['access_token'] - id_token = token['id_token'] - refresh_token = token['refresh_token'] if 'refresh_token' in token else None - expiration_time = datetime.now() + timedelta(seconds=token.get('expires_in', 3600)) - refresh_expiration_time = (datetime.now() + timedelta(seconds=token['refresh_expires_in'])) if 'refresh_expires_in' in token else None + access_token = token["access_token"] + id_token = token["id_token"] + refresh_token = token["refresh_token"] if "refresh_token" in token else None + expiration_time = datetime.now() + timedelta(seconds=token.get("expires_in", 3600)) + refresh_expiration_time = ( + (datetime.now() + timedelta(seconds=token["refresh_expires_in"])) if "refresh_expires_in" in token else None + ) # Get nonce from token['id_token'] and validate. 'nonce' in the # id_token is a hash of the nonce stored in the NONCE_COOKIE_NAME # cookie. id_token_decoded = self._decode_token_no_signature(id_token) - nonce_hash = id_token_decoded['nonce'] + nonce_hash = id_token_decoded["nonce"] self._validate_nonce(trans, nonce_hash) # Get userinfo and lookup/create Galaxy user record - if id_token_decoded.get('email', None): + if id_token_decoded.get("email", None): userinfo = id_token_decoded else: userinfo = self._get_userinfo(oauth2_session) - email = userinfo['email'] - user_id = userinfo['sub'] + email = userinfo["email"] + user_id = userinfo["sub"] # Create or update custos_authnz_token record - custos_authnz_token = self._get_custos_authnz_token(trans.sa_session, user_id, self.config['provider']) + custos_authnz_token = self._get_custos_authnz_token(trans.sa_session, user_id, self.config["provider"]) if custos_authnz_token is None: user = trans.user if not user: @@ -105,15 +116,17 @@ class CustosAuthnz(IdentityProvider): # TODO: Future work will expand on this and provide an # interface for when there are multiple auth providers # allowing explicit authenticated association. - if (trans.app.config.enable_oidc - and len(trans.app.config.oidc) == 1 - and len(trans.app.auth_manager.authenticators) == 0): + if ( + trans.app.config.enable_oidc + and len(trans.app.config.oidc) == 1 + and len(trans.app.auth_manager.authenticators) == 0 + ): user = existing_user else: message = f"There already exists a user with email {email}. To associate this external login, you must first be logged in as that existing account." log.exception(message) raise exceptions.AuthenticationFailed(message) - elif self.config['provider'] == 'custos': + elif self.config["provider"] == "custos": login_redirect_url = f"{login_redirect_url}root/login?confirm=true&custos_token={json.dumps(token)}" return login_redirect_url, None else: @@ -122,14 +135,16 @@ class CustosAuthnz(IdentityProvider): if trans.app.config.user_activation_on: trans.app.user_manager.send_activation_email(trans, email, username) - custos_authnz_token = CustosAuthnzToken(user=user, - external_user_id=user_id, - provider=self.config['provider'], - access_token=access_token, - id_token=id_token, - refresh_token=refresh_token, - expiration_time=expiration_time, - refresh_expiration_time=refresh_expiration_time) + custos_authnz_token = CustosAuthnzToken( + user=user, + external_user_id=user_id, + provider=self.config["provider"], + access_token=access_token, + id_token=id_token, + refresh_token=refresh_token, + expiration_time=expiration_time, + refresh_expiration_time=refresh_expiration_time, + ) else: custos_authnz_token.access_token = access_token custos_authnz_token.id_token = id_token @@ -143,11 +158,17 @@ class CustosAuthnz(IdentityProvider): def create_user(self, token, trans, login_redirect_url): token_dict = json.loads(token) - access_token = token_dict['access_token'] - id_token = token_dict['id_token'] - refresh_token = token_dict['refresh_token'] if 'refresh_token' in token_dict else None - expiration_time = datetime.now() + timedelta(seconds=token_dict.get('expires_in', 3600)) # might be a problem cause times no long valid - refresh_expiration_time = (datetime.now() + timedelta(seconds=token_dict['refresh_expires_in'])) if 'refresh_expires_in' in token_dict else None + access_token = token_dict["access_token"] + id_token = token_dict["id_token"] + refresh_token = token_dict["refresh_token"] if "refresh_token" in token_dict else None + expiration_time = datetime.now() + timedelta( + seconds=token_dict.get("expires_in", 3600) + ) # might be a problem cause times no long valid + refresh_expiration_time = ( + (datetime.now() + timedelta(seconds=token_dict["refresh_expires_in"])) + if "refresh_expires_in" in token_dict + else None + ) # Get nonce from token['id_token'] and validate. 'nonce' in the # id_token is a hash of the nonce stored in the NONCE_COOKIE_NAME @@ -155,23 +176,25 @@ class CustosAuthnz(IdentityProvider): userinfo = self._decode_token_no_signature(id_token) # Get userinfo and create Galaxy user record - email = userinfo['email'] + email = userinfo["email"] # Check if username if already taken username = self._username_from_userinfo(trans, userinfo) - user_id = userinfo['sub'] + user_id = userinfo["sub"] user = trans.app.user_manager.create(email=email, username=username) if trans.app.config.user_activation_on: trans.app.user_manager.send_activation_email(trans, email, username) - custos_authnz_token = CustosAuthnzToken(user=user, - external_user_id=user_id, - provider=self.config['provider'], - access_token=access_token, - id_token=id_token, - refresh_token=refresh_token, - expiration_time=expiration_time, - refresh_expiration_time=refresh_expiration_time) + custos_authnz_token = CustosAuthnzToken( + user=user, + external_user_id=user_id, + provider=self.config["provider"], + access_token=access_token, + id_token=id_token, + refresh_token=refresh_token, + expiration_time=expiration_time, + refresh_expiration_time=refresh_expiration_time, + ) trans.sa_session.add(user) trans.sa_session.add(custos_authnz_token) @@ -189,7 +212,7 @@ class CustosAuthnz(IdentityProvider): if len(provider_tokens) > 1: for idx, token in enumerate(provider_tokens): id_token_decoded = self._decode_token_no_signature(token.id_token) - if (id_token_decoded['email'] == email): + if id_token_decoded["email"] == email: index = idx trans.sa_session.delete(provider_tokens[index]) trans.sa_session.flush() @@ -199,7 +222,7 @@ class CustosAuthnz(IdentityProvider): def logout(self, trans, post_logout_redirect_url=None): try: - redirect_url = self.config['end_session_endpoint'] + redirect_url = self.config["end_session_endpoint"] if post_logout_redirect_url is not None: redirect_url += f"?redirect_uri={quote(post_logout_redirect_url)}" return redirect_url @@ -208,43 +231,40 @@ class CustosAuthnz(IdentityProvider): return None def _create_oauth2_session(self, state=None, scope=None): - client_id = self.config['client_id'] - redirect_uri = self.config['redirect_uri'] - if (redirect_uri.startswith('http://localhost') - and os.environ.get("OAUTHLIB_INSECURE_TRANSPORT", None) != "1"): + client_id = self.config["client_id"] + redirect_uri = self.config["redirect_uri"] + if redirect_uri.startswith("http://localhost") and os.environ.get("OAUTHLIB_INSECURE_TRANSPORT", None) != "1": log.warning("Setting OAUTHLIB_INSECURE_TRANSPORT to '1' to allow plain HTTP (non-SSL) callback") - os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = "1" - session = OAuth2Session(client_id, - scope=scope, - redirect_uri=redirect_uri, - state=state) + os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1" + session = OAuth2Session(client_id, scope=scope, redirect_uri=redirect_uri, state=state) session.verify = self._get_verify_param() return session def _fetch_token(self, oauth2_session, trans): - if self.config.get('iam_client_secret'): + if self.config.get("iam_client_secret"): # Custos uses the Keycloak client secret to get the token - client_secret = self.config['iam_client_secret'] + client_secret = self.config["iam_client_secret"] else: - client_secret = self.config['client_secret'] - token_endpoint = self.config['token_endpoint'] + client_secret = self.config["client_secret"] + token_endpoint = self.config["token_endpoint"] clientIdAndSec = f"{self.config['client_id']}:{self.config['client_secret']}" # for custos return oauth2_session.fetch_token( token_endpoint, client_secret=client_secret, authorization_response=trans.request.url, - headers={"Authorization": f"Basic {util.unicodify(base64.b64encode(util.smart_str(clientIdAndSec)))}"}, # for custos - verify=self._get_verify_param()) + headers={ + "Authorization": f"Basic {util.unicodify(base64.b64encode(util.smart_str(clientIdAndSec)))}" + }, # for custos + verify=self._get_verify_param(), + ) def _get_userinfo(self, oauth2_session): - userinfo_endpoint = self.config['userinfo_endpoint'] - return oauth2_session.get(userinfo_endpoint, - verify=self._get_verify_param()).json() + userinfo_endpoint = self.config["userinfo_endpoint"] + return oauth2_session.get(userinfo_endpoint, verify=self._get_verify_param()).json() def _get_custos_authnz_token(self, sa_session, user_id, provider): - return sa_session.query(CustosAuthnzToken).filter_by( - external_user_id=user_id, provider=provider).one_or_none() + return sa_session.query(CustosAuthnzToken).filter_by(external_user_id=user_id, provider=provider).one_or_none() def _hash_nonce(self, nonce): return hashlib.sha256(util.smart_str(nonce)).hexdigest() @@ -252,48 +272,53 @@ class CustosAuthnz(IdentityProvider): def _validate_nonce(self, trans, nonce_hash): nonce_cookie = trans.get_cookie(name=NONCE_COOKIE_NAME) # Delete the nonce cookie - trans.set_cookie('', name=NONCE_COOKIE_NAME, age=-1) + trans.set_cookie("", name=NONCE_COOKIE_NAME, age=-1) nonce_cookie_hash = self._hash_nonce(nonce_cookie) if nonce_hash != nonce_cookie_hash: raise Exception("Nonce mismatch!") def _load_config_for_cilogon(self): # Set cilogon endpoints - self.config['authorization_endpoint'] = "https://cilogon.org/authorize" - self.config['token_endpoint'] = "https://cilogon.org/oauth2/token" - self.config['userinfo_endpoint'] = "https://cilogon.org/oauth2/userinfo" + self.config["authorization_endpoint"] = "https://cilogon.org/authorize" + self.config["token_endpoint"] = "https://cilogon.org/oauth2/token" + self.config["userinfo_endpoint"] = "https://cilogon.org/oauth2/userinfo" def _load_config_for_custos(self): - self.config['well_known_oidc_config_uri'] = self._get_well_known_uri_from_url(self.config['provider']) - self.config['credential_url'] = f"{self.config['url'].rstrip('/')}/credentials" + self.config["well_known_oidc_config_uri"] = self._get_well_known_uri_from_url(self.config["provider"]) + self.config["credential_url"] = f"{self.config['url'].rstrip('/')}/credentials" self._get_custos_credentials() # Set custos endpoints clientIdAndSec = f"{self.config['client_id']}:{self.config['client_secret']}" - eps = requests.get(self.config['well_known_oidc_config_uri'], - headers={"Authorization": f"Basic {util.unicodify(base64.b64encode(util.smart_str(clientIdAndSec)))}"}, - verify=False, - params={'client_id': self.config['client_id']}, - timeout=util.DEFAULT_SOCKET_TIMEOUT) + eps = requests.get( + self.config["well_known_oidc_config_uri"], + headers={"Authorization": f"Basic {util.unicodify(base64.b64encode(util.smart_str(clientIdAndSec)))}"}, + verify=False, + params={"client_id": self.config["client_id"]}, + timeout=util.DEFAULT_SOCKET_TIMEOUT, + ) well_known_oidc_config = eps.json() self._load_well_known_oidc_config(well_known_oidc_config) def _load_config_for_keycloak(self): - self.config['well_known_oidc_config_uri'] = self._get_well_known_uri_from_url(self.config['provider']) - well_known_oidc_config = self._fetch_well_known_oidc_config(self.config['well_known_oidc_config_uri']) + self.config["well_known_oidc_config_uri"] = self._get_well_known_uri_from_url(self.config["provider"]) + well_known_oidc_config = self._fetch_well_known_oidc_config(self.config["well_known_oidc_config_uri"]) self._load_well_known_oidc_config(well_known_oidc_config) def _get_custos_credentials(self): clientIdAndSec = f"{self.config['client_id']}:{self.config['client_secret']}" - creds = requests.get(self.config['credential_url'], - headers={"Authorization": f"Basic {util.unicodify(base64.b64encode(util.smart_str(clientIdAndSec)))}"}, - verify=False, params={'client_id': self.config['client_id']}, - timeout=util.DEFAULT_SOCKET_TIMEOUT) + creds = requests.get( + self.config["credential_url"], + headers={"Authorization": f"Basic {util.unicodify(base64.b64encode(util.smart_str(clientIdAndSec)))}"}, + verify=False, + params={"client_id": self.config["client_id"]}, + timeout=util.DEFAULT_SOCKET_TIMEOUT, + ) credentials = creds.json() - self.config['iam_client_secret'] = credentials['iam_client_secret'] + self.config["iam_client_secret"] = credentials["iam_client_secret"] def _get_well_known_uri_from_url(self, provider): # TODO: Look up this URL from a Python library - if provider in ['custos', 'keycloak']: + if provider in ["custos", "keycloak"]: base_url = self.config["url"] # Remove potential trailing slash to avoid "//realms" base_url = base_url if base_url[-1] != "/" else base_url[:-1] @@ -303,35 +328,35 @@ class CustosAuthnz(IdentityProvider): def _fetch_well_known_oidc_config(self, well_known_uri): try: - return requests.get(well_known_uri, - verify=self._get_verify_param(), - timeout=util.DEFAULT_SOCKET_TIMEOUT).json() + return requests.get( + well_known_uri, verify=self._get_verify_param(), timeout=util.DEFAULT_SOCKET_TIMEOUT + ).json() except Exception: log.error(f"Failed to load well-known OIDC config URI: {well_known_uri}") raise def _load_well_known_oidc_config(self, well_known_oidc_config): - self.config['authorization_endpoint'] = well_known_oidc_config['authorization_endpoint'] - self.config['token_endpoint'] = well_known_oidc_config['token_endpoint'] - self.config['userinfo_endpoint'] = well_known_oidc_config['userinfo_endpoint'] - self.config['end_session_endpoint'] = well_known_oidc_config.get('end_session_endpoint') + self.config["authorization_endpoint"] = well_known_oidc_config["authorization_endpoint"] + self.config["token_endpoint"] = well_known_oidc_config["token_endpoint"] + self.config["userinfo_endpoint"] = well_known_oidc_config["userinfo_endpoint"] + self.config["end_session_endpoint"] = well_known_oidc_config.get("end_session_endpoint") def _get_verify_param(self): """Return 'ca_bundle' if 'verify_ssl' is true and 'ca_bundle' is configured.""" # in requests_oauthlib, the verify param can either be a boolean or a CA bundle path - if self.config['ca_bundle'] is not None and self.config['verify_ssl']: - return self.config['ca_bundle'] + if self.config["ca_bundle"] is not None and self.config["verify_ssl"]: + return self.config["ca_bundle"] else: - return self.config['verify_ssl'] + return self.config["verify_ssl"] def _username_from_userinfo(self, trans, userinfo): - username = userinfo.get('preferred_username', userinfo['email']) + username = userinfo.get("preferred_username", userinfo["email"]) if "@" in username: - username = username.split('@')[0] # username created from username portion of email - if (trans.sa_session.query(trans.app.model.User).filter_by(username=username).first()): + username = username.split("@")[0] # username created from username portion of email + if trans.sa_session.query(trans.app.model.User).filter_by(username=username).first(): # if username already exists in database, append integer and iterate until unique username found count = 0 - while (trans.sa_session.query(trans.app.model.User).filter_by(username=(f"{username}{count}")).first()): + while trans.sa_session.query(trans.app.model.User).filter_by(username=(f"{username}{count}")).first(): count += 1 return f"{username}{count}" else: diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index 479924f74fb..7049448c39f 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -8,12 +8,12 @@ import string import requests from cloudauthz import CloudAuthz -from cloudauthz.exceptions import ( - CloudAuthzBaseException -) +from cloudauthz.exceptions import CloudAuthzBaseException -from galaxy import exceptions -from galaxy import model +from galaxy import ( + exceptions, + model, +) from galaxy.util import ( asbool, etree, @@ -31,22 +31,20 @@ from .psa_authnz import ( on_the_fly_config, PSAAuthnz, Storage, - Strategy + Strategy, ) - log = logging.getLogger(__name__) # Note: This if for backward compatibility. Icons can be specified in oidc_backends_config.xml. DEFAULT_OIDC_IDP_ICONS = { - 'google': 'https://developers.google.com/identity/images/btn_google_signin_light_normal_web.png', - 'elixir': 'https://elixir-europe.org/sites/default/files/images/login-button-orange.png', - 'okta': 'https://www.okta.com/sites/all/themes/Okta/images/blog/Logos/Okta_Logo_BrightBlue_Medium.png' + "google": "https://developers.google.com/identity/images/btn_google_signin_light_normal_web.png", + "elixir": "https://elixir-europe.org/sites/default/files/images/login-button-orange.png", + "okta": "https://www.okta.com/sites/all/themes/Okta/images/blog/Logos/Okta_Logo_BrightBlue_Medium.png", } class AuthnzManager: - def __init__(self, app, oidc_config_file, oidc_backends_config_file): """ :type app: galaxy.app.UniverseApplication @@ -66,35 +64,42 @@ class AuthnzManager: try: tree = parse_xml(config_file) root = tree.getroot() - if root.tag != 'OIDC': - raise etree.ParseError("The root element in OIDC_Config xml file is expected to be `OIDC`, " - "found `{}` instead -- unable to continue.".format(root.tag)) + if root.tag != "OIDC": + raise etree.ParseError( + "The root element in OIDC_Config xml file is expected to be `OIDC`, " + "found `{}` instead -- unable to continue.".format(root.tag) + ) for child in root: - if child.tag != 'Setter': - log.error("Expect a node with `Setter` tag, found a node with `{}` tag instead; " - "skipping this node.".format(child.tag)) + if child.tag != "Setter": + log.error( + "Expect a node with `Setter` tag, found a node with `{}` tag instead; " + "skipping this node.".format(child.tag) + ) continue - if 'Property' not in child.attrib or 'Value' not in child.attrib or 'Type' not in child.attrib: - log.error("Could not find the node attributes `Property` and/or `Value` and/or `Type`;" - " found these attributes: `{}`; skipping this node.".format(child.attrib)) + if "Property" not in child.attrib or "Value" not in child.attrib or "Type" not in child.attrib: + log.error( + "Could not find the node attributes `Property` and/or `Value` and/or `Type`;" + " found these attributes: `{}`; skipping this node.".format(child.attrib) + ) continue try: - if child.get('Type') == "bool": + if child.get("Type") == "bool": func = string_as_bool else: - func = getattr(builtins, child.get('Type')) + func = getattr(builtins, child.get("Type")) except AttributeError: - log.error("The value of attribute `Type`, `{}`, is not a valid built-in type;" - " skipping this node").format(child.get('Type')) + log.error( + "The value of attribute `Type`, `{}`, is not a valid built-in type;" " skipping this node" + ).format(child.get("Type")) continue - self.oidc_config[child.get('Property')] = func(child.get('Value')) + self.oidc_config[child.get("Property")] = func(child.get("Value")) except ImportError: raise except etree.ParseError as e: raise etree.ParseError(f"Invalid configuration at `{config_file}`: {e} -- unable to continue.") def _get_idp_icon(self, idp): - return self.oidc_backends_config[idp].get('icon') or DEFAULT_OIDC_IDP_ICONS.get(idp) + return self.oidc_backends_config[idp].get("icon") or DEFAULT_OIDC_IDP_ICONS.get(idp) def _parse_oidc_backends_config(self, config_file): self.oidc_backends_config = {} @@ -102,26 +107,30 @@ class AuthnzManager: try: tree = parse_xml(config_file) root = tree.getroot() - if root.tag != 'OIDC': - raise etree.ParseError("The root element in OIDC config xml file is expected to be `OIDC`, " - "found `{}` instead -- unable to continue.".format(root.tag)) + if root.tag != "OIDC": + raise etree.ParseError( + "The root element in OIDC config xml file is expected to be `OIDC`, " + "found `{}` instead -- unable to continue.".format(root.tag) + ) for child in root: - if child.tag != 'provider': - log.error("Expect a node with `provider` tag, found a node with `{}` tag instead; " - "skipping the node.".format(child.tag)) + if child.tag != "provider": + log.error( + "Expect a node with `provider` tag, found a node with `{}` tag instead; " + "skipping the node.".format(child.tag) + ) continue - if 'name' not in child.attrib: + if "name" not in child.attrib: log.error(f"Could not find a node attribute 'name'; skipping the node '{child.tag}'.") continue - idp = child.get('name').lower() + idp = child.get("name").lower() if idp in BACKENDS_NAME: self.oidc_backends_config[idp] = self._parse_idp_config(child) - self.oidc_backends_implementation[idp] = 'psa' - self.app.config.oidc[idp] = {'icon': self._get_idp_icon(idp)} + self.oidc_backends_implementation[idp] = "psa" + self.app.config.oidc[idp] = {"icon": self._get_idp_icon(idp)} elif idp in KEYCLOAK_BACKENDS: self.oidc_backends_config[idp] = self._parse_custos_config(child) - self.oidc_backends_implementation[idp] = 'custos' - self.app.config.oidc[idp] = {'icon': self._get_idp_icon(idp)} + self.oidc_backends_implementation[idp] = "custos" + self.app.config.oidc[idp] = {"icon": self._get_idp_icon(idp)} else: raise etree.ParseError("Unknown provider specified") if len(self.oidc_backends_config) == 0: @@ -133,40 +142,42 @@ class AuthnzManager: def _parse_idp_config(self, config_xml): rtv = { - 'client_id': config_xml.find('client_id').text, - 'client_secret': config_xml.find('client_secret').text, - 'redirect_uri': config_xml.find('redirect_uri').text, - 'enable_idp_logout': asbool(config_xml.findtext('enable_idp_logout', 'false'))} - if config_xml.find('prompt') is not None: - rtv['prompt'] = config_xml.find('prompt').text - if config_xml.find('api_url') is not None: - rtv['api_url'] = config_xml.find('api_url').text - if config_xml.find('url') is not None: - rtv['url'] = config_xml.find('url').text - if config_xml.find('icon') is not None: - rtv['icon'] = config_xml.find('icon').text - if config_xml.find('extra_scopes') is not None: - rtv['extra_scopes'] = listify(config_xml.find('extra_scopes').text) + "client_id": config_xml.find("client_id").text, + "client_secret": config_xml.find("client_secret").text, + "redirect_uri": config_xml.find("redirect_uri").text, + "enable_idp_logout": asbool(config_xml.findtext("enable_idp_logout", "false")), + } + if config_xml.find("prompt") is not None: + rtv["prompt"] = config_xml.find("prompt").text + if config_xml.find("api_url") is not None: + rtv["api_url"] = config_xml.find("api_url").text + if config_xml.find("url") is not None: + rtv["url"] = config_xml.find("url").text + if config_xml.find("icon") is not None: + rtv["icon"] = config_xml.find("icon").text + if config_xml.find("extra_scopes") is not None: + rtv["extra_scopes"] = listify(config_xml.find("extra_scopes").text) return rtv def _parse_custos_config(self, config_xml): rtv = { - 'url': config_xml.find('url').text, - 'client_id': config_xml.find('client_id').text, - 'client_secret': config_xml.find('client_secret').text, - 'redirect_uri': config_xml.find('redirect_uri').text, - 'enable_idp_logout': asbool(config_xml.findtext('enable_idp_logout', 'false'))} - if config_xml.find('credential_url') is not None: - rtv['credential_url'] = config_xml.find('credential_url').text - if config_xml.find('well_known_oidc_config_uri') is not None: - rtv['well_known_oidc_config_uri'] = config_xml.find('well_known_oidc_config_uri').text - if config_xml.findall('allowed_idp') is not None: - self.allowed_idps = list(map(lambda idp: idp.text, config_xml.findall('allowed_idp'))) - if config_xml.find('ca_bundle') is not None: - rtv['ca_bundle'] = config_xml.find('ca_bundle').text - if config_xml.find('icon') is not None: - rtv['icon'] = config_xml.find('icon').text + "url": config_xml.find("url").text, + "client_id": config_xml.find("client_id").text, + "client_secret": config_xml.find("client_secret").text, + "redirect_uri": config_xml.find("redirect_uri").text, + "enable_idp_logout": asbool(config_xml.findtext("enable_idp_logout", "false")), + } + if config_xml.find("credential_url") is not None: + rtv["credential_url"] = config_xml.find("credential_url").text + if config_xml.find("well_known_oidc_config_uri") is not None: + rtv["well_known_oidc_config_uri"] = config_xml.find("well_known_oidc_config_uri").text + if config_xml.findall("allowed_idp") is not None: + self.allowed_idps = list(map(lambda idp: idp.text, config_xml.findall("allowed_idp"))) + if config_xml.find("ca_bundle") is not None: + rtv["ca_bundle"] = config_xml.find("ca_bundle").text + if config_xml.find("icon") is not None: + rtv["icon"] = config_xml.find("icon").text return rtv def get_allowed_idps(self): @@ -188,22 +199,37 @@ class AuthnzManager: identity_provider_class = self._get_identity_provider_class(self.oidc_backends_implementation[provider]) try: if provider in KEYCLOAK_BACKENDS: - return True, "", identity_provider_class(unified_provider_name, self.oidc_config, self.oidc_backends_config[unified_provider_name], idphint=idphint) + return ( + True, + "", + identity_provider_class( + unified_provider_name, + self.oidc_config, + self.oidc_backends_config[unified_provider_name], + idphint=idphint, + ), + ) else: - return True, "", identity_provider_class(unified_provider_name, self.oidc_config, self.oidc_backends_config[unified_provider_name]) + return ( + True, + "", + identity_provider_class( + unified_provider_name, self.oidc_config, self.oidc_backends_config[unified_provider_name] + ), + ) except Exception as e: - log.exception(f'An error occurred when loading {identity_provider_class.__name__}') + log.exception(f"An error occurred when loading {identity_provider_class.__name__}") return False, unicodify(e), None else: - msg = f'The requested identity provider, `{provider}`, is not a recognized/expected provider.' + msg = f"The requested identity provider, `{provider}`, is not a recognized/expected provider." log.debug(msg) return False, msg, None @staticmethod def _get_identity_provider_class(implementation): - if implementation == 'psa': + if implementation == "psa": return PSAAuthnz - elif implementation == 'custos': + elif implementation == "custos": return CustosAuthnz else: return None @@ -215,17 +241,22 @@ class AuthnzManager: strategy = Strategy(request, None, Storage, backend.config) on_the_fly_config(sa_session) try: - config['id_token'] = cloudauthz.authn.get_id_token(strategy) + config["id_token"] = cloudauthz.authn.get_id_token(strategy) except requests.exceptions.HTTPError as e: - msg = "Sign-out from Galaxy and remove its access from `{}`, then log back in using `{}` " \ - "account.".format(self._unify_provider_name(cloudauthz.authn.provider), cloudauthz.authn.uid) - log.debug("Failed to get/refresh ID token for user with ID `{}` for assuming authz_id `{}`. " - "User may not have a refresh token. If the problem persists, set the `prompt` key to " - "`consent` in `oidc_backends_config.xml`, then restart Galaxy and ask user to: {}" - "Error Message: `{}`".format(user_id, cloudauthz.id, msg, e.response.text)) + msg = ( + "Sign-out from Galaxy and remove its access from `{}`, then log back in using `{}` " + "account.".format(self._unify_provider_name(cloudauthz.authn.provider), cloudauthz.authn.uid) + ) + log.debug( + "Failed to get/refresh ID token for user with ID `{}` for assuming authz_id `{}`. " + "User may not have a refresh token. If the problem persists, set the `prompt` key to " + "`consent` in `oidc_backends_config.xml`, then restart Galaxy and ask user to: {}" + "Error Message: `{}`".format(user_id, cloudauthz.id, msg, e.response.text) + ) raise exceptions.AuthenticationFailed( err_msg="An error occurred getting your ID token. {}. If the problem persists, please " - "contact Galaxy admin.".format(msg)) + "contact Galaxy admin.".format(msg) + ) return config @staticmethod @@ -233,12 +264,14 @@ class AuthnzManager: qres = trans.sa_session.query(model.UserAuthnzToken).get(authn_id) if qres is None: msg = "Authentication record with the given `authn_id` (`{}`) not found.".format( - trans.security.encode_id(authn_id)) + trans.security.encode_id(authn_id) + ) log.debug(msg) raise exceptions.ObjectNotFound(msg) if qres.user_id != trans.user.id: - msg = "The request authentication with ID `{}` is not accessible to user with ID " \ - "`{}`.".format(trans.security.encode_id(authn_id), trans.security.encode_id(trans.user.id)) + msg = "The request authentication with ID `{}` is not accessible to user with ID " "`{}`.".format( + trans.security.encode_id(authn_id), trans.security.encode_id(trans.user.id) + ) log.warning(msg) raise exceptions.ItemAccessibilityException(msg) @@ -264,8 +297,10 @@ class AuthnzManager: if qres is None: raise exceptions.ObjectNotFound("An authorization configuration with given ID not found.") if user_id != qres.user_id: - msg = "The request authorization configuration (with ID:`{}`) is not accessible for user with " \ - "ID:`{}`.".format(qres.id, user_id) + msg = ( + "The request authorization configuration (with ID:`{}`) is not accessible for user with " + "ID:`{}`.".format(qres.id, user_id) + ) log.warning(msg) raise exceptions.ItemAccessibilityException(msg) return qres @@ -284,14 +319,22 @@ class AuthnzManager: if success is False: return False, message, None elif provider in KEYCLOAK_BACKENDS: - if (self.allowed_idps and (idphint not in self.allowed_idps)): - msg = f'An error occurred when authenticating a user. Invalid EntityID: `{idphint}`' + if self.allowed_idps and (idphint not in self.allowed_idps): + msg = f"An error occurred when authenticating a user. Invalid EntityID: `{idphint}`" log.exception(msg) return False, msg, None - return True, f"Redirecting to the `{provider}` identity provider for authentication", backend.authenticate(trans, idphint) - return True, f"Redirecting to the `{provider}` identity provider for authentication", backend.authenticate(trans) + return ( + True, + f"Redirecting to the `{provider}` identity provider for authentication", + backend.authenticate(trans, idphint), + ) + return ( + True, + f"Redirecting to the `{provider}` identity provider for authentication", + backend.authenticate(trans), + ) except Exception: - msg = f'An error occurred when authenticating a user on `{provider}` identity provider' + msg = f"An error occurred when authenticating a user on `{provider}` identity provider" log.exception(msg) return False, msg, None @@ -304,7 +347,7 @@ class AuthnzManager: except exceptions.AuthenticationFailed: raise except Exception: - msg = f'An error occurred when handling callback from `{provider}` identity provider. Please contact an administrator for assistance.' + msg = f"An error occurred when handling callback from `{provider}` identity provider. Please contact an administrator for assistance." log.exception(msg) return False, msg, (None, None) @@ -318,7 +361,7 @@ class AuthnzManager: log.exception("Error creating user") raise except Exception: - msg = f'An error occurred when creating a user with `{provider}` identity provider. Please contact an administrator for assistance.' + msg = f"An error occurred when creating a user with `{provider}` identity provider. Please contact an administrator for assistance." log.exception(msg) return False, msg, (None, None) @@ -338,7 +381,7 @@ class AuthnzManager: try: # check if logout is enabled for this idp and return false if not unified_provider_name = self._unify_provider_name(provider) - if self.oidc_backends_config[unified_provider_name]['enable_idp_logout'] is False: + if self.oidc_backends_config[unified_provider_name]["enable_idp_logout"] is False: return False, f"IDP logout is not enabled for {provider}", None success, message, backend = self._get_authnz_backend(provider) @@ -346,7 +389,7 @@ class AuthnzManager: return False, message, None return True, message, backend.logout(trans, post_logout_redirect_url) except Exception: - msg = f'An error occurred when logging out from `{provider}` identity provider. Please contact an administrator for assistance.' + msg = f"An error occurred when logging out from `{provider}` identity provider. Please contact an administrator for assistance." log.exception(msg) return False, msg, None @@ -359,8 +402,11 @@ class AuthnzManager: return backend.disconnect(provider, trans, email, disconnect_redirect_url) return backend.disconnect(provider, trans, disconnect_redirect_url) except Exception: - msg = 'An error occurred when disconnecting authentication with `{}` identity provider for user `{}`' \ - .format(provider, trans.user.username) + msg = ( + "An error occurred when disconnecting authentication with `{}` identity provider for user `{}`".format( + provider, trans.user.username + ) + ) log.exception(msg) return False, msg, None @@ -399,8 +445,11 @@ class AuthnzManager: config = self._extend_cloudauthz_config(cloudauthz, request, sa_session, user_id) try: ca = CloudAuthz() - log.info("Requesting credentials using CloudAuthz with config id `{}` on be half of user `{}`.".format( - cloudauthz.id, user_id)) + log.info( + "Requesting credentials using CloudAuthz with config id `{}` on be half of user `{}`.".format( + cloudauthz.id, user_id + ) + ) credentials = ca.authorize(cloudauthz.provider, config) return credentials except CloudAuthzBaseException as e: @@ -439,12 +488,18 @@ class AuthnzManager: :rtype: str :return: The filename to which credentials are written. """ - filename = os.path.abspath(os.path.join(new_file_path, - "cd_" + ''.join(random.SystemRandom().choice( - string.ascii_uppercase + string.digits) for _ in range(11)))) + filename = os.path.abspath( + os.path.join( + new_file_path, + "cd_" + + "".join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(11)), + ) + ) credentials = self.get_cloud_access_credentials(cloudauthz, sa_session, user_id, request) - log.info("Writting credentials generated using CloudAuthz with config id `{}` to the following file: `{}`" - "".format(cloudauthz.id, filename)) + log.info( + "Writting credentials generated using CloudAuthz with config id `{}` to the following file: `{}`" + "".format(cloudauthz.id, filename) + ) with open(filename, "w") as f: f.write(json.dumps(credentials)) return filename diff --git a/lib/galaxy/authnz/psa_authnz.py b/lib/galaxy/authnz/psa_authnz.py index 2139ab698a8..de6a6b4d576 100644 --- a/lib/galaxy/authnz/psa_authnz.py +++ b/lib/galaxy/authnz/psa_authnz.py @@ -1,39 +1,48 @@ import json import requests -from social_core.actions import do_auth, do_complete, do_disconnect +from social_core.actions import ( + do_auth, + do_complete, + do_disconnect, +) from social_core.backends.utils import get_backend from social_core.strategy import BaseStrategy -from social_core.utils import module_member, setting_name +from social_core.utils import ( + module_member, + setting_name, +) from sqlalchemy.exc import IntegrityError from galaxy.exceptions import MalformedContents from galaxy.util import DEFAULT_SOCKET_TIMEOUT from ..authnz import IdentityProvider -from ..model import PSAAssociation, PSACode, PSANonce, PSAPartial, UserAuthnzToken - +from ..model import ( + PSAAssociation, + PSACode, + PSANonce, + PSAPartial, + UserAuthnzToken, +) # key: a component name which PSA requests. # value: is the name of a class associated with that key. -DEFAULTS = { - 'STRATEGY': 'Strategy', - 'STORAGE': 'Storage' -} +DEFAULTS = {"STRATEGY": "Strategy", "STORAGE": "Storage"} BACKENDS = { - 'google': 'social_core.backends.google_openidconnect.GoogleOpenIdConnect', - 'globus': 'social_core.backends.globus.GlobusOpenIdConnect', - 'elixir': 'social_core.backends.elixir.ElixirOpenIdConnect', - 'okta': 'social_core.backends.okta_openidconnect.OktaOpenIdConnect', - 'azure': 'social_core.backends.azuread_tenant.AzureADTenantOAuth2' + "google": "social_core.backends.google_openidconnect.GoogleOpenIdConnect", + "globus": "social_core.backends.globus.GlobusOpenIdConnect", + "elixir": "social_core.backends.elixir.ElixirOpenIdConnect", + "okta": "social_core.backends.okta_openidconnect.OktaOpenIdConnect", + "azure": "social_core.backends.azuread_tenant.AzureADTenantOAuth2", } BACKENDS_NAME = { - 'google': 'google-openidconnect', - 'globus': 'globus', - 'elixir': 'elixir', - 'okta': 'okta-openidconnect', - 'azure': 'azuread-tenant-oauth2' + "google": "google-openidconnect", + "globus": "globus", + "elixir": "elixir", + "okta": "okta-openidconnect", + "azure": "azuread-tenant-oauth2", } AUTH_PIPELINE = ( @@ -41,67 +50,52 @@ AUTH_PIPELINE = ( # format to create the user instance later. On some cases the details are # already part of the auth response from the provider, but sometimes this # could hit a provider API. - 'social_core.pipeline.social_auth.social_details', - + "social_core.pipeline.social_auth.social_details", # Get the social uid from whichever service we're authing thru. The uid is # the unique identifier of the given user in the provider. - 'social_core.pipeline.social_auth.social_uid', - + "social_core.pipeline.social_auth.social_uid", # Verifies that the current auth process is valid within the current # project, this is where emails and domains allowlists are applied (if # defined). - 'social_core.pipeline.social_auth.auth_allowed', - + "social_core.pipeline.social_auth.auth_allowed", # Checks if the decoded response contains all the required fields such # as an ID token or a refresh token. - 'galaxy.authnz.psa_authnz.contains_required_data', - - 'galaxy.authnz.psa_authnz.verify', - + "galaxy.authnz.psa_authnz.contains_required_data", + "galaxy.authnz.psa_authnz.verify", # Checks if the current social-account is already associated in the site. - 'social_core.pipeline.social_auth.social_user', - + "social_core.pipeline.social_auth.social_user", # Make up a username for this person, appends a random string at the end if # there's any collision. - 'social_core.pipeline.user.get_username', - + "social_core.pipeline.user.get_username", # Send a validation email to the user to verify its email address. # 'social_core.pipeline.mail.mail_validation', - # Associates the current social details with another user account with # a similar email address. - 'social_core.pipeline.social_auth.associate_by_email', - + "social_core.pipeline.social_auth.associate_by_email", # Create a user account if we haven't found one yet. - 'social_core.pipeline.user.create_user', - + "social_core.pipeline.user.create_user", # Create the record that associated the social account with this user. - 'social_core.pipeline.social_auth.associate_user', - + "social_core.pipeline.social_auth.associate_user", # Populate the extra_data field in the social record with the values # specified by settings (and the default ones like access_token, etc). - 'social_core.pipeline.social_auth.load_extra_data', - + "social_core.pipeline.social_auth.load_extra_data", # Update the user record with any changed info from the auth service. - 'social_core.pipeline.user.user_details' + "social_core.pipeline.user.user_details", ) -DISCONNECT_PIPELINE = ( - 'galaxy.authnz.psa_authnz.allowed_to_disconnect', - 'galaxy.authnz.psa_authnz.disconnect' -) +DISCONNECT_PIPELINE = ("galaxy.authnz.psa_authnz.allowed_to_disconnect", "galaxy.authnz.psa_authnz.disconnect") class PSAAuthnz(IdentityProvider): def __init__(self, provider, oidc_config, oidc_backend_config): - self.config = {'provider': provider.lower()} + self.config = {"provider": provider.lower()} for key, value in oidc_config.items(): self.config[setting_name(key)] = value - self.config[setting_name('USER_MODEL')] = 'models.User' - self.config['SOCIAL_AUTH_PIPELINE'] = AUTH_PIPELINE - self.config['DISCONNECT_PIPELINE'] = DISCONNECT_PIPELINE - self.config[setting_name('AUTHENTICATION_BACKENDS')] = (BACKENDS[provider],) + self.config[setting_name("USER_MODEL")] = "models.User" + self.config["SOCIAL_AUTH_PIPELINE"] = AUTH_PIPELINE + self.config["DISCONNECT_PIPELINE"] = DISCONNECT_PIPELINE + self.config[setting_name("AUTHENTICATION_BACKENDS")] = (BACKENDS[provider],) self.config["VERIFY_SSL"] = oidc_config.get("VERIFY_SSL") self.config["REQUESTS_TIMEOUT"] = oidc_config.get("REQUESTS_TIMEOUT") @@ -111,90 +105,95 @@ class PSAAuthnz(IdentityProvider): # logging in a user. If this setting is set to false, the `_login_user` # would not be called, and as a result Galaxy would not know who is # the just logged-in user. - self.config[setting_name('INACTIVE_USER_LOGIN')] = True + self.config[setting_name("INACTIVE_USER_LOGIN")] = True if provider in BACKENDS_NAME: self._setup_idp(oidc_backend_config) # Secondary AuthZ with Google identities is currently supported if provider != "google": - if 'SOCIAL_AUTH_SECONDARY_AUTH_PROVIDER' in self.config: + if "SOCIAL_AUTH_SECONDARY_AUTH_PROVIDER" in self.config: del self.config["SOCIAL_AUTH_SECONDARY_AUTH_PROVIDER"] - if 'SOCIAL_AUTH_SECONDARY_AUTH_ENDPOINT' in self.config: + if "SOCIAL_AUTH_SECONDARY_AUTH_ENDPOINT" in self.config: del self.config["SOCIAL_AUTH_SECONDARY_AUTH_ENDPOINT"] def _setup_idp(self, oidc_backend_config): - self.config[setting_name('AUTH_EXTRA_ARGUMENTS')] = {'access_type': 'offline'} - self.config['KEY'] = oidc_backend_config.get('client_id') - self.config['SECRET'] = oidc_backend_config.get('client_secret') - self.config['redirect_uri'] = oidc_backend_config.get('redirect_uri') - self.config['EXTRA_SCOPES'] = oidc_backend_config.get('extra_scopes') - if oidc_backend_config.get('prompt') is not None: - self.config[setting_name('AUTH_EXTRA_ARGUMENTS')]['prompt'] = oidc_backend_config.get('prompt') - if oidc_backend_config.get('api_url') is not None: - self.config[setting_name('API_URL')] = oidc_backend_config.get('api_url') - if oidc_backend_config.get('url') is not None: - self.config[setting_name('URL')] = oidc_backend_config.get('url') + self.config[setting_name("AUTH_EXTRA_ARGUMENTS")] = {"access_type": "offline"} + self.config["KEY"] = oidc_backend_config.get("client_id") + self.config["SECRET"] = oidc_backend_config.get("client_secret") + self.config["redirect_uri"] = oidc_backend_config.get("redirect_uri") + self.config["EXTRA_SCOPES"] = oidc_backend_config.get("extra_scopes") + if oidc_backend_config.get("prompt") is not None: + self.config[setting_name("AUTH_EXTRA_ARGUMENTS")]["prompt"] = oidc_backend_config.get("prompt") + if oidc_backend_config.get("api_url") is not None: + self.config[setting_name("API_URL")] = oidc_backend_config.get("api_url") + if oidc_backend_config.get("url") is not None: + self.config[setting_name("URL")] = oidc_backend_config.get("url") def _get_helper(self, name, do_import=False): this_config = self.config.get(setting_name(name), DEFAULTS.get(name, None)) return do_import and module_member(this_config) or this_config def _load_backend(self, strategy, redirect_uri): - backends = self._get_helper('AUTHENTICATION_BACKENDS') - backend = get_backend(backends, BACKENDS_NAME[self.config['provider']]) + backends = self._get_helper("AUTHENTICATION_BACKENDS") + backend = get_backend(backends, BACKENDS_NAME[self.config["provider"]]) return backend(strategy, redirect_uri) def _login_user(self, backend, user, social_user): - self.config['user'] = user + self.config["user"] = user def authenticate(self, trans): on_the_fly_config(trans.sa_session) strategy = Strategy(trans.request, trans.session, Storage, self.config) - backend = self._load_backend(strategy, self.config['redirect_uri']) - if backend.name is BACKENDS_NAME["google"] and \ - "SOCIAL_AUTH_SECONDARY_AUTH_PROVIDER" in self.config and \ - "SOCIAL_AUTH_SECONDARY_AUTH_ENDPOINT" in self.config: + backend = self._load_backend(strategy, self.config["redirect_uri"]) + if ( + backend.name is BACKENDS_NAME["google"] + and "SOCIAL_AUTH_SECONDARY_AUTH_PROVIDER" in self.config + and "SOCIAL_AUTH_SECONDARY_AUTH_ENDPOINT" in self.config + ): backend.DEFAULT_SCOPE.append("https://www.googleapis.com/auth/cloud-platform") - if self.config['EXTRA_SCOPES'] is not None: - backend.DEFAULT_SCOPE.extend(self.config['EXTRA_SCOPES']) + if self.config["EXTRA_SCOPES"] is not None: + backend.DEFAULT_SCOPE.extend(self.config["EXTRA_SCOPES"]) return do_auth(backend) def callback(self, state_token, authz_code, trans, login_redirect_url): on_the_fly_config(trans.sa_session) - self.config[setting_name('LOGIN_REDIRECT_URL')] = login_redirect_url + self.config[setting_name("LOGIN_REDIRECT_URL")] = login_redirect_url strategy = Strategy(trans.request, trans.session, Storage, self.config) strategy.session_set(f"{BACKENDS_NAME[self.config['provider']]}_state", state_token) - backend = self._load_backend(strategy, self.config['redirect_uri']) + backend = self._load_backend(strategy, self.config["redirect_uri"]) redirect_url = do_complete( backend, login=lambda backend, user, social_user: self._login_user(backend, user, social_user), user=trans.user, - state=state_token) - return redirect_url, self.config.get('user', None) + state=state_token, + ) + return redirect_url, self.config.get("user", None) def disconnect(self, provider, trans, disconnect_redirect_url=None, association_id=None): on_the_fly_config(trans.sa_session) - self.config[setting_name('DISCONNECT_REDIRECT_URL')] =\ + self.config[setting_name("DISCONNECT_REDIRECT_URL")] = ( disconnect_redirect_url if disconnect_redirect_url is not None else () + ) strategy = Strategy(trans.request, trans.session, Storage, self.config) - backend = self._load_backend(strategy, self.config['redirect_uri']) + backend = self._load_backend(strategy, self.config["redirect_uri"]) response = do_disconnect(backend, trans.user, association_id) if isinstance(response, str): return True, "", response - return response.get('success', False), response.get('message', ""), "" + return response.get("success", False), response.get("message", ""), "" class Strategy(BaseStrategy): - def __init__(self, request, session, storage, config, tpl=None): self.request = request self.session = session if session else {} self.config = config - self.config['SOCIAL_AUTH_REDIRECT_IS_HTTPS'] = True if self.request and self.request.host.startswith('https:') else False - self.config['SOCIAL_AUTH_GOOGLE_OPENIDCONNECT_EXTRA_DATA'] = ['id_token'] + self.config["SOCIAL_AUTH_REDIRECT_IS_HTTPS"] = ( + True if self.request and self.request.host.startswith("https:") else False + ) + self.config["SOCIAL_AUTH_GOOGLE_OPENIDCONNECT_EXTRA_DATA"] = ["id_token"] super().__init__(storage, tpl) def get_setting(self, name): @@ -207,7 +206,7 @@ class Strategy(BaseStrategy): self.session[name] = value def session_pop(self, name): - raise NotImplementedError('Not implemented.') + raise NotImplementedError("Not implemented.") def request_data(self, merge=True): if not self.request: @@ -215,7 +214,7 @@ class Strategy(BaseStrategy): if merge: data = self.request.GET.copy() data.update(self.request.POST) - elif self.request.method == 'POST': + elif self.request.method == "POST": data = self.request.POST else: data = self.request.GET @@ -226,23 +225,25 @@ class Strategy(BaseStrategy): return self.request.host def build_absolute_uri(self, path=None): - path = path or '' - if path.startswith('http://') or path.startswith('https://'): + path = path or "" + if path.startswith("http://") or path.startswith("https://"): return path if self.request: - return \ - self.request.host +\ - '/authnz' + ('/' + self.config.get('provider')) if self.config.get('provider', None) is not None else '' + return ( + self.request.host + "/authnz" + ("/" + self.config.get("provider")) + if self.config.get("provider", None) is not None + else "" + ) return path def redirect(self, url): return url def html(self, content): - raise NotImplementedError('Not implemented.') + raise NotImplementedError("Not implemented.") def render_html(self, tpl=None, html=None, context=None): - raise NotImplementedError('Not implemented.') + raise NotImplementedError("Not implemented.") def start(self): self.clean_partial_pipeline() @@ -316,11 +317,13 @@ def contains_required_data(response=None, is_new=False, **kwargs): :rtype: void :return: Raises an exception if any of the required arguments is missing, and pass if all are given. """ - hint_msg = "Visit the identity provider's permitted applications page " \ - "(e.g., visit `https://myaccount.google.com/u/0/permissions` " \ - "for Google), then revoke the access of this Galaxy instance, " \ - "and then retry to login. If the problem persists, contact " \ - "the Admin of this Galaxy instance." + hint_msg = ( + "Visit the identity provider's permitted applications page " + "(e.g., visit `https://myaccount.google.com/u/0/permissions` " + "for Google), then revoke the access of this Galaxy instance, " + "and then retry to login. If the problem persists, contact " + "the Admin of this Galaxy instance." + ) if response is None or not isinstance(response, dict): # This can happen only if PSA is not able to decode the `authnz code` # sent back from the identity provider. PSA internally handles such @@ -356,10 +359,10 @@ def verify(strategy=None, response=None, details=None, **kwargs): result = requests.post( f"https://iam.googleapis.com/v1/projects/-/serviceAccounts/{endpoint}:getIamPolicy", headers={ - 'Authorization': f"Bearer {response.get('access_token')}", - 'Accept': 'application/json', + "Authorization": f"Bearer {response.get('access_token')}", + "Accept": "application/json", }, - timeout=DEFAULT_SOCKET_TIMEOUT + timeout=DEFAULT_SOCKET_TIMEOUT, ) res = json.loads(result.content) if result.status_code == requests.codes.ok: @@ -380,8 +383,9 @@ def verify(strategy=None, response=None, details=None, **kwargs): raise Exception(f"`{provider}` is an unsupported secondary authorization provider, contact admin.") -def allowed_to_disconnect(name=None, user=None, user_storage=None, strategy=None, - backend=None, request=None, details=None, **kwargs): +def allowed_to_disconnect( + name=None, user=None, user_storage=None, strategy=None, backend=None, request=None, details=None, **kwargs +): """ Disconnect is the process of disassociating a Galaxy user and a third-party authnz. In other words, it is the process of removing any access and/or ID tokens of a user. @@ -402,8 +406,9 @@ def allowed_to_disconnect(name=None, user=None, user_storage=None, strategy=None """ -def disconnect(name=None, user=None, user_storage=None, strategy=None, - backend=None, request=None, details=None, **kwargs): +def disconnect( + name=None, user=None, user_storage=None, strategy=None, backend=None, request=None, details=None, **kwargs +): """ Disconnect is the process of disassociating a Galaxy user and a third-party authnz. In other words, it is the process of removing any access and/or ID tokens of a user. @@ -422,10 +427,13 @@ def disconnect(name=None, user=None, user_storage=None, strategy=None, """ sa_session = user_storage.sa_session - user_authnz = sa_session.query(user_storage).filter(user_storage.table.c.user_id == user.id, - user_storage.table.c.provider == name).first() + user_authnz = ( + sa_session.query(user_storage) + .filter(user_storage.table.c.user_id == user.id, user_storage.table.c.provider == name) + .first() + ) if user_authnz is None: - return {'success': False, 'message': 'Not authenticated by any identity providers.'} + return {"success": False, "message": "Not authenticated by any identity providers."} # option A sa_session.delete(user_authnz) # option B diff --git a/lib/galaxy/celery/__init__.py b/lib/galaxy/celery/__init__.py index 5bc16fb0d96..6aa5a4c8535 100644 --- a/lib/galaxy/celery/__init__.py +++ b/lib/galaxy/celery/__init__.py @@ -15,7 +15,10 @@ from galaxy.config import Configuration from galaxy.main_config import find_config from galaxy.util.custom_logging import get_logger from galaxy.util.properties import load_app_properties -from ._serialization import schema_dumps, schema_loads +from ._serialization import ( + schema_dumps, + schema_loads, +) log = get_logger(__name__) @@ -23,11 +26,12 @@ log = get_logger(__name__) @lru_cache(maxsize=1) def get_galaxy_app(): import galaxy.app + if galaxy.app.app: return galaxy.app.app kwargs = get_app_properties() if kwargs: - kwargs['check_migrate_databases'] = False + kwargs["check_migrate_databases"] = False galaxy_app = galaxy.app.GalaxyManagerApplication(configure_logging=False, **kwargs) return galaxy_app @@ -35,16 +39,16 @@ def get_galaxy_app(): @lru_cache(maxsize=1) def get_app_properties(): config_file = os.environ.get("GALAXY_CONFIG_FILE") - galaxy_root_dir = os.environ.get('GALAXY_ROOT_DIR') + galaxy_root_dir = os.environ.get("GALAXY_ROOT_DIR") if not config_file and galaxy_root_dir: config_file = find_config(config_file, galaxy_root_dir) if config_file: properties = load_app_properties( config_file=os.path.abspath(config_file), - config_section='galaxy', + config_section="galaxy", ) if galaxy_root_dir: - properties['root_dir'] = galaxy_root_dir + properties["root_dir"] = galaxy_root_dir return properties @@ -52,7 +56,7 @@ def get_app_properties(): def get_config(): kwargs = get_app_properties() if kwargs: - kwargs['override_tempdir'] = False + kwargs["override_tempdir"] = False return Configuration(**kwargs) @@ -71,33 +75,30 @@ def get_history_audit_table_prune_interval(): broker = get_broker() -celery_app = Celery('galaxy', broker=broker, include=['galaxy.celery.tasks']) +celery_app = Celery("galaxy", broker=broker, include=["galaxy.celery.tasks"]) prune_interval = get_history_audit_table_prune_interval() if prune_interval > 0: celery_app.conf.beat_schedule = { - 'prune-history-audit-table': { - 'task': 'galaxy.celery.tasks.prune_history_audit_table', - 'schedule': prune_interval, + "prune-history-audit-table": { + "task": "galaxy.celery.tasks.prune_history_audit_table", + "schedule": prune_interval, }, } -celery_app.conf.timezone = 'UTC' +celery_app.conf.timezone = "UTC" CELERY_TASKS = [] -PYDANTIC_AWARE_SERIALIER_NAME = 'pydantic-aware-json' +PYDANTIC_AWARE_SERIALIER_NAME = "pydantic-aware-json" serialization.register( - PYDANTIC_AWARE_SERIALIER_NAME, - encoder=schema_dumps, - decoder=schema_loads, - content_type='application/json' + PYDANTIC_AWARE_SERIALIER_NAME, encoder=schema_dumps, decoder=schema_loads, content_type="application/json" ) def galaxy_task(*args, **celery_task_kwd): - if 'serializer' not in celery_task_kwd: - celery_task_kwd['serializer'] = PYDANTIC_AWARE_SERIALIER_NAME + if "serializer" not in celery_task_kwd: + celery_task_kwd["serializer"] = PYDANTIC_AWARE_SERIALIER_NAME def decorate(func): CELERY_TASKS.append(func.__name__) @@ -117,5 +118,5 @@ def galaxy_task(*args, **celery_task_kwd): return decorate -if __name__ == '__main__': +if __name__ == "__main__": celery_app.start() diff --git a/lib/galaxy/celery/_serialization.py b/lib/galaxy/celery/_serialization.py index a7a6b3a4670..8accdb632d3 100644 --- a/lib/galaxy/celery/_serialization.py +++ b/lib/galaxy/celery/_serialization.py @@ -10,32 +10,34 @@ SCHEMA_MODELS_PACKAGE_BASE = "galaxy.schema." def fullname(o): klass = o.__class__ module = klass.__module__ - if module == 'builtins': + if module == "builtins": return klass.__qualname__ # avoid outputs like 'builtins.str' - return module + '.' + klass.__qualname__ + return module + "." + klass.__qualname__ class SchemaEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, BaseModel): return { - '__type__': '__pydantic_object__', - '__class__': fullname(obj), - '__object__': obj.dict(), + "__type__": "__pydantic_object__", + "__class__": fullname(obj), + "__object__": obj.dict(), } else: return json.JSONEncoder.default(self, obj) def schema_decoder(obj): - if '__type__' in obj: - if obj['__type__'] == '__pydantic_object__': - clazz_str = obj['__class__'] - assert clazz_str.startswith(SCHEMA_MODELS_PACKAGE_BASE) and ".." not in clazz_str, f"Invalid class str {clazz_str}" - module_name, class_name = clazz_str.rsplit('.', 1) + if "__type__" in obj: + if obj["__type__"] == "__pydantic_object__": + clazz_str = obj["__class__"] + assert ( + clazz_str.startswith(SCHEMA_MODELS_PACKAGE_BASE) and ".." not in clazz_str + ), f"Invalid class str {clazz_str}" + module_name, class_name = clazz_str.rsplit(".", 1) module = import_module(module_name) clazz = getattr(module, class_name, None) - obj = clazz(**obj['__object__']) + obj = clazz(**obj["__object__"]) return obj return obj diff --git a/lib/galaxy/celery/tasks.py b/lib/galaxy/celery/tasks.py index 8b1d4a97faa..8914cfaf193 100644 --- a/lib/galaxy/celery/tasks.py +++ b/lib/galaxy/celery/tasks.py @@ -31,24 +31,27 @@ def purge_hda(hda_manager: HDAManager, hda_id): @galaxy_task -def set_metadata(hda_manager: HDAManager, ldda_manager: LDDAManager, dataset_id, model_class='HistoryDatasetAssociation'): - if model_class == 'HistoryDatasetAssociation': +def set_metadata( + hda_manager: HDAManager, ldda_manager: LDDAManager, dataset_id, model_class="HistoryDatasetAssociation" +): + if model_class == "HistoryDatasetAssociation": dataset = hda_manager.by_id(dataset_id) - elif model_class == 'LibraryDatasetDatasetAssociation': + elif model_class == "LibraryDatasetDatasetAssociation": dataset = ldda_manager.by_id(dataset_id) dataset.datatype.set_meta(dataset) @galaxy_task(ignore_result=True) def export_history( - app: MinimalManagerApp, - sa_session: galaxy_scoped_session, - job_manager: JobManager, - store_directory: str, - history_id: int, - job_id: int, - include_hidden=False, - include_deleted=False): + app: MinimalManagerApp, + sa_session: galaxy_scoped_session, + job_manager: JobManager, + store_directory: str, + history_id: int, + job_id: int, + include_hidden=False, + include_deleted=False, +): history = sa_session.query(model.History).get(history_id) with model.store.DirectoryModelExportStore(store_directory, app=app, export_files="symlink") as export_store: export_store.export_history(history, include_hidden=include_hidden, include_deleted=include_deleted) diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index 0d76a4e39f6..15ac7dd8d16 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -61,7 +61,10 @@ from galaxy.util.properties import ( ) from galaxy.web.formatting import expand_pretty_datetime_format from galaxy.web_stack import get_stack_facts -from ..version import VERSION_MAJOR, VERSION_MINOR +from ..version import ( + VERSION_MAJOR, + VERSION_MINOR, +) try: from importlib.resources import files # type: ignore[attr-defined] @@ -78,60 +81,60 @@ if TYPE_CHECKING: log = logging.getLogger(__name__) -GALAXY_APP_NAME = 'galaxy' -GALAXY_SCHEMAS_PATH = files('galaxy.config') / 'schemas' -GALAXY_CONFIG_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / 'config_schema.yml' -UWSGI_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / 'uwsgi_schema.yml' +GALAXY_APP_NAME = "galaxy" +GALAXY_SCHEMAS_PATH = files("galaxy.config") / "schemas" +GALAXY_CONFIG_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / "config_schema.yml" +UWSGI_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / "uwsgi_schema.yml" LOGGING_CONFIG_DEFAULT: Dict[str, Any] = { - 'disable_existing_loggers': False, - 'version': 1, - 'root': { - 'handlers': ['console'], - 'level': 'DEBUG', + "disable_existing_loggers": False, + "version": 1, + "root": { + "handlers": ["console"], + "level": "DEBUG", }, - 'loggers': { - 'paste.httpserver.ThreadPool': { - 'level': 'WARN', - 'qualname': 'paste.httpserver.ThreadPool', + "loggers": { + "paste.httpserver.ThreadPool": { + "level": "WARN", + "qualname": "paste.httpserver.ThreadPool", }, - 'sqlalchemy_json.track': { - 'level': 'WARN', - 'qualname': 'sqlalchemy_json.track', + "sqlalchemy_json.track": { + "level": "WARN", + "qualname": "sqlalchemy_json.track", }, - 'urllib3.connectionpool': { - 'level': 'WARN', - 'qualname': 'urllib3.connectionpool', + "urllib3.connectionpool": { + "level": "WARN", + "qualname": "urllib3.connectionpool", }, - 'routes.middleware': { - 'level': 'WARN', - 'qualname': 'routes.middleware', + "routes.middleware": { + "level": "WARN", + "qualname": "routes.middleware", }, - 'amqp': { - 'level': 'INFO', - 'qualname': 'amqp', + "amqp": { + "level": "INFO", + "qualname": "amqp", }, - 'botocore': { - 'level': 'INFO', - 'qualname': 'botocore', + "botocore": { + "level": "INFO", + "qualname": "botocore", }, }, - 'filters': { - 'stack': { - '()': 'galaxy.web_stack.application_stack_log_filter', + "filters": { + "stack": { + "()": "galaxy.web_stack.application_stack_log_filter", }, }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - 'formatter': 'stack', - 'level': 'DEBUG', - 'stream': 'ext://sys.stderr', - 'filters': ['stack'], + "handlers": { + "console": { + "class": "logging.StreamHandler", + "formatter": "stack", + "level": "DEBUG", + "stream": "ext://sys.stderr", + "filters": ["stack"], }, }, - 'formatters': { - 'stack': { - '()': 'galaxy.web_stack.application_stack_log_formatter', + "formatters": { + "stack": { + "()": "galaxy.web_stack.application_stack_log_formatter", }, }, } @@ -139,7 +142,7 @@ LOGGING_CONFIG_DEFAULT: Dict[str, Any] = { def find_root(kwargs): - return os.path.abspath(kwargs.get('root_dir', '.')) + return os.path.abspath(kwargs.get("root_dir", ".")) OptStr = TypeVar("OptStr", None, str) @@ -150,7 +153,9 @@ class BaseAppConfiguration(HasDynamicProperties): # If VALUE == first directory in a user-supplied path that resolves to KEY, it will be stripped from that path renamed_options: Optional[Dict[str, str]] = None deprecated_dirs: Dict[str, str] = {} - paths_to_check_against_root: Set[str] = set() # backward compatibility: if resolved path doesn't exist, try resolving w.r.t root + paths_to_check_against_root: Set[ + str + ] = set() # backward compatibility: if resolved path doesn't exist, try resolving w.r.t root add_sample_file_to_defaults: Set[str] = set() # for these options, add sample config files to their defaults listify_options: Set[str] = set() # values for these options are processed as lists of values object_store_store_by: str @@ -188,19 +193,21 @@ class BaseAppConfiguration(HasDynamicProperties): Fix deprecated database URLs (postgres... >> postgresql...) https://docs.sqlalchemy.org/en/14/changelog/changelog_14.html#change-3687655465c25a39b968b4f5f6e9170b """ - old_dialect, new_dialect = 'postgres', 'postgresql' - old_prefixes = (f'{old_dialect}:', f'{old_dialect}+') # check for postgres://foo and postgres+driver//foo + old_dialect, new_dialect = "postgres", "postgresql" + old_prefixes = (f"{old_dialect}:", f"{old_dialect}+") # check for postgres://foo and postgres+driver//foo offset = len(old_dialect) - keys = ('database_connection', 'install_database_connection') + keys = ("database_connection", "install_database_connection") for key in keys: if key in kwargs: value = kwargs[key] for prefix in old_prefixes: if value.startswith(prefix): - value = f'{new_dialect}{value[offset:]}' + value = f"{new_dialect}{value[offset:]}" kwargs[key] = value - log.warning('PostgreSQL database URLs of the form "postgres://" have been ' - 'deprecated. Please use "postgresql://".') + log.warning( + 'PostgreSQL database URLs of the form "postgres://" have been ' + 'deprecated. Please use "postgresql://".' + ) def is_set(self, key): """Check if a configuration option has been explicitly set.""" @@ -215,13 +222,12 @@ class BaseAppConfiguration(HasDynamicProperties): return self._in_root_dir(path) def _set_config_base(self, config_kwargs): - def _set_global_conf(): - self.config_file = find_config_file('galaxy') - self.global_conf = config_kwargs.get('global_conf') + self.config_file = find_config_file("galaxy") + self.global_conf = config_kwargs.get("global_conf") self.global_conf_parser = configparser.ConfigParser() if not self.config_file and self.global_conf and "__file__" in self.global_conf: - self.config_file = os.path.join(self.root, self.global_conf['__file__']) + self.config_file = os.path.join(self.root, self.global_conf["__file__"]) if self.config_file is None: log.warning("No Galaxy config file found, running from current working directory: %s", os.getcwd()) @@ -236,40 +242,40 @@ class BaseAppConfiguration(HasDynamicProperties): def _set_config_directories(): # Set config_dir to value from kwargs OR dirname of config_file OR None _config_dir = os.path.dirname(self.config_file) if self.config_file else None - self.config_dir = config_kwargs.get('config_dir', _config_dir) + self.config_dir = config_kwargs.get("config_dir", _config_dir) # Make path absolute before using it as base for other paths if self.config_dir: self.config_dir = os.path.abspath(self.config_dir) - self.data_dir = config_kwargs.get('data_dir') + self.data_dir = config_kwargs.get("data_dir") if self.data_dir: self.data_dir = os.path.abspath(self.data_dir) - self.sample_config_dir = os.path.join(os.path.dirname(__file__), 'sample') + self.sample_config_dir = os.path.join(os.path.dirname(__file__), "sample") if self.sample_config_dir: self.sample_config_dir = os.path.abspath(self.sample_config_dir) - self.managed_config_dir = config_kwargs.get('managed_config_dir') + self.managed_config_dir = config_kwargs.get("managed_config_dir") if self.managed_config_dir: self.managed_config_dir = os.path.abspath(self.managed_config_dir) if running_from_source: if not self.config_dir: - self.config_dir = os.path.join(self.root, 'config') + self.config_dir = os.path.join(self.root, "config") if not self.data_dir: - self.data_dir = os.path.join(self.root, 'database') + self.data_dir = os.path.join(self.root, "database") if not self.managed_config_dir: self.managed_config_dir = self.config_dir else: if not self.config_dir: self.config_dir = os.getcwd() if not self.data_dir: - self.data_dir = self._in_config_dir('data') + self.data_dir = self._in_config_dir("data") if not self.managed_config_dir: - self.managed_config_dir = self._in_data_dir('config') + self.managed_config_dir = self._in_data_dir("config") # TODO: do we still need to support ../shed_tools when running_from_source? - self.shed_tools_dir = self._in_data_dir('shed_tools') + self.shed_tools_dir = self._in_data_dir("shed_tools") log.debug("Configuration directory is %s", self.config_dir) log.debug("Data directory is %s", self.data_dir) @@ -280,7 +286,7 @@ class BaseAppConfiguration(HasDynamicProperties): def _load_schema(self): # Override in subclasses - raise Exception('Not implemented') + raise Exception("Not implemented") def _preprocess_paths_to_resolve(self): # For these options, if option is not set, listify its defaults and add a sample config file. @@ -288,17 +294,20 @@ class BaseAppConfiguration(HasDynamicProperties): for key in self.add_sample_file_to_defaults: if not self.is_set(key): defaults = listify(getattr(self, key), do_strip=True) - sample = f'{defaults[-1]}.sample' # if there are multiple defaults, use last as template + sample = f"{defaults[-1]}.sample" # if there are multiple defaults, use last as template sample = self._in_sample_dir(sample) # resolve w.r.t sample_dir defaults.append(sample) setattr(self, key, defaults) def _postprocess_paths_to_resolve(self): - def select_one_path_from_list(): # To consider: options with a sample file added to defaults except options that can have multiple values. # If value is not set, check each path in list; set to first path that exists; if none exist, set to last path in list. - keys = self.add_sample_file_to_defaults - self.listify_options if self.listify_options else self.add_sample_file_to_defaults + keys = ( + self.add_sample_file_to_defaults - self.listify_options + if self.listify_options + else self.add_sample_file_to_defaults + ) for key in keys: if not self.is_set(key): paths = getattr(self, key) @@ -307,7 +316,9 @@ class BaseAppConfiguration(HasDynamicProperties): setattr(self, key, path) break else: - setattr(self, key, paths[-1]) # TODO: we assume it exists; but we've already checked in the loop! Raise error instead? + setattr( + self, key, paths[-1] + ) # TODO: we assume it exists; but we've already checked in the loop! Raise error instead? def select_one_or_all_paths_from_list(): # Values for these options are lists of paths. If value is not set, use defaults if all paths in list exist; @@ -320,7 +331,9 @@ class BaseAppConfiguration(HasDynamicProperties): setattr(self, key, [paths[-1]]) # value is a list break - if self.add_sample_file_to_defaults: # Currently, this is the ONLY case when we need to pick one file from a list + if ( + self.add_sample_file_to_defaults + ): # Currently, this is the ONLY case when we need to pick one file from a list select_one_path_from_list() if self.listify_options: select_one_or_all_paths_from_list() @@ -346,7 +359,7 @@ class BaseAppConfiguration(HasDynamicProperties): } def convert_datatype(key, value): - datatype = self.schema.app_schema[key].get('type') + datatype = self.schema.app_schema[key].get("type") # check for `not None` explicitly (value can be falsy) if value is not None and datatype in type_converters: # convert value or each item in value to type `datatype` @@ -367,14 +380,18 @@ class BaseAppConfiguration(HasDynamicProperties): ignore = first_dir + os.sep log.warning( "Paths for the '%s' option are now relative to '%s', remove the leading '%s' " - "to suppress this warning: %s", key, resolves_to, ignore, path + "to suppress this warning: %s", + key, + resolves_to, + ignore, + path, ) - paths[i] = path[len(ignore):] + paths[i] = path[len(ignore) :] # return list or string, depending on type of `value` if isinstance(value, list): return paths - return ','.join(paths) + return ",".join(paths) return value for key, value in kwargs.items(): @@ -387,7 +404,7 @@ class BaseAppConfiguration(HasDynamicProperties): def _create_attributes_from_raw_config(self): # `base_configs` are a special case: these attributes have been created and will be ignored # by the code below. Trying to overwrite any other existing attributes will raise an error. - base_configs = {'config_dir', 'data_dir', 'managed_config_dir'} + base_configs = {"config_dir", "data_dir", "managed_config_dir"} for key, value in self._raw_config.items(): if not hasattr(self, key): setattr(self, key, value) @@ -395,7 +412,6 @@ class BaseAppConfiguration(HasDynamicProperties): raise ConfigurationError(f"Attempting to override existing attribute '{key}'") def _resolve_paths(self): - def resolve(key): if key in _cache: # resolve each path only once return _cache[key] @@ -420,7 +436,7 @@ class BaseAppConfiguration(HasDynamicProperties): # Check if value is a list or should be listified; if so, listify and resolve each item separately. if type(value) is list or (self.listify_options and key in self.listify_options): saved_values = listify(getattr(self, key), do_strip=True) # listify and save original value - setattr(self, key, '_') # replace value with temporary placeholder + setattr(self, key, "_") # replace value with temporary placeholder resolve(key) # resolve temporary value (`_` becomes `parent-path/_`) resolved_base = getattr(self, key)[:-1] # get rid of placeholder in resolved path # apply resolved base to saved values @@ -433,7 +449,6 @@ class BaseAppConfiguration(HasDynamicProperties): self._check_against_root(key) def _check_against_root(self, key): - def get_path(current_path, initial_path): # if path does not exist and was set as relative: if not self._path_exists(current_path) and not os.path.isabs(initial_path): @@ -531,64 +546,71 @@ class CommonConfigurationMixin: class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): - deprecated_options = ('database_file', 'track_jobs_in_database', 'blacklist_file', 'whitelist_file', - 'sanitize_whitelist_file', 'user_library_import_symlink_whitelist', 'fetch_url_whitelist', - 'containers_resolvers_config_file') + deprecated_options = ( + "database_file", + "track_jobs_in_database", + "blacklist_file", + "whitelist_file", + "sanitize_whitelist_file", + "user_library_import_symlink_whitelist", + "fetch_url_whitelist", + "containers_resolvers_config_file", + ) renamed_options = { - 'blacklist_file': 'email_domain_blocklist_file', - 'whitelist_file': 'email_domain_allowlist_file', - 'sanitize_whitelist_file': 'sanitize_allowlist_file', - 'user_library_import_symlink_whitelist': 'user_library_import_symlink_allowlist', - 'fetch_url_whitelist': 'fetch_url_allowlist', - 'containers_resolvers_config_file': 'container_resolvers_config_file', + "blacklist_file": "email_domain_blocklist_file", + "whitelist_file": "email_domain_allowlist_file", + "sanitize_whitelist_file": "sanitize_allowlist_file", + "user_library_import_symlink_whitelist": "user_library_import_symlink_allowlist", + "fetch_url_whitelist": "fetch_url_allowlist", + "containers_resolvers_config_file": "container_resolvers_config_file", } - default_config_file_name = 'galaxy.yml' - deprecated_dirs = {'config_dir': 'config', 'data_dir': 'database'} + default_config_file_name = "galaxy.yml" + deprecated_dirs = {"config_dir": "config", "data_dir": "database"} paths_to_check_against_root = { - 'auth_config_file', - 'build_sites_config_file', - 'containers_config_file', - 'data_manager_config_file', - 'datatypes_config_file', - 'dependency_resolvers_config_file', - 'error_report_file', - 'job_config_file', - 'job_metrics_config_file', - 'job_resource_params_file', - 'local_conda_mapping_file', - 'migrated_tools_config', - 'modules_mapping_files', - 'object_store_config_file', - 'oidc_backends_config_file', - 'oidc_config_file', - 'shed_data_manager_config_file', - 'shed_tool_config_file', - 'shed_tool_data_table_config', - 'tool_destinations_config_file', - 'tool_sheds_config_file', - 'user_preferences_extra_conf_path', - 'workflow_resource_params_file', - 'workflow_schedulers_config_file', - 'markdown_export_css', - 'markdown_export_css_pages', - 'markdown_export_css_invocation_reports', - 'file_path', - 'tool_data_table_config_path', - 'tool_config_file', + "auth_config_file", + "build_sites_config_file", + "containers_config_file", + "data_manager_config_file", + "datatypes_config_file", + "dependency_resolvers_config_file", + "error_report_file", + "job_config_file", + "job_metrics_config_file", + "job_resource_params_file", + "local_conda_mapping_file", + "migrated_tools_config", + "modules_mapping_files", + "object_store_config_file", + "oidc_backends_config_file", + "oidc_config_file", + "shed_data_manager_config_file", + "shed_tool_config_file", + "shed_tool_data_table_config", + "tool_destinations_config_file", + "tool_sheds_config_file", + "user_preferences_extra_conf_path", + "workflow_resource_params_file", + "workflow_schedulers_config_file", + "markdown_export_css", + "markdown_export_css_pages", + "markdown_export_css_invocation_reports", + "file_path", + "tool_data_table_config_path", + "tool_config_file", } add_sample_file_to_defaults = { - 'build_sites_config_file', - 'datatypes_config_file', - 'job_metrics_config_file', - 'tool_data_table_config_path', - 'tool_config_file', + "build_sites_config_file", + "datatypes_config_file", + "job_metrics_config_file", + "tool_data_table_config_path", + "tool_config_file", } listify_options = { - 'tool_data_table_config_path', - 'tool_config_file', + "tool_data_table_config_path", + "tool_config_file", } database_connection: str tool_path: str @@ -637,17 +659,20 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): This method should be deleted after migration to SQLAlchemy 2.0 is complete. To enable warnings, set `GALAXY_CONFIG_SQLALCHEMY_WARN_20=1`, """ - warn = string_as_bool(kwargs.get('sqlalchemy_warn_20', False)) + warn = string_as_bool(kwargs.get("sqlalchemy_warn_20", False)) if warn: import sqlalchemy + sqlalchemy.util.deprecations.SQLALCHEMY_WARN_20 = True self._setup_sqlalchemy20_warnings_filters() def _setup_sqlalchemy20_warnings_filters(self): import warnings + from sqlalchemy.exc import RemovedIn20Warning + # Always display RemovedIn20Warning warnings. - warnings.filterwarnings('always', category=RemovedIn20Warning) + warnings.filterwarnings("always", category=RemovedIn20Warning) # Optionally, enable filters for specific warnings (raise error, or log, etc.) # messages = [ # r"replace with warning text to match", @@ -693,13 +718,13 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): self.version_minor = VERSION_MINOR # Database related configuration - self.check_migrate_databases = kwargs.get('check_migrate_databases', True) + self.check_migrate_databases = kwargs.get("check_migrate_databases", True) if not self.database_connection: # Provide default if not supplied by user - db_path = self._in_data_dir('universe.sqlite') - self.database_connection = f'sqlite:///{db_path}?isolation_level=IMMEDIATE' + db_path = self._in_data_dir("universe.sqlite") + self.database_connection = f"sqlite:///{db_path}?isolation_level=IMMEDIATE" self.database_engine_options = get_database_engine_options(kwargs) - self.database_create_tables = string_as_bool(kwargs.get('database_create_tables', 'True')) - self.database_encoding = kwargs.get('database_encoding') # Create new databases with this encoding + self.database_create_tables = string_as_bool(kwargs.get("database_create_tables", "True")) + self.database_encoding = kwargs.get("database_encoding") # Create new databases with this encoding self.thread_local_log = None if self.enable_per_request_sql_debugging: self.thread_local_log = threading.local() @@ -710,12 +735,12 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): self.tool_path = self._in_root_dir(self.tool_path) self.tool_data_path = self._in_root_dir(self.tool_data_path) if not running_from_source and kwargs.get("tool_data_path") is None: - self.tool_data_path = self._in_data_dir(self.schema.defaults['tool_data_path']) + self.tool_data_path = self._in_data_dir(self.schema.defaults["tool_data_path"]) self.builds_file_path = os.path.join(self.tool_data_path, self.builds_file_path) self.len_file_path = os.path.join(self.tool_data_path, self.len_file_path) self.oidc = {} self.integrated_tool_panel_config = self._in_managed_config_dir(self.integrated_tool_panel_config) - integrated_tool_panel_tracking_directory = kwargs.get('integrated_tool_panel_tracking_directory') + integrated_tool_panel_tracking_directory = kwargs.get("integrated_tool_panel_tracking_directory") if integrated_tool_panel_tracking_directory: self.integrated_tool_panel_tracking_directory = self._in_root_dir(integrated_tool_panel_tracking_directory) else: @@ -728,18 +753,18 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): self.user_tool_filters = listify(self.user_tool_filters, do_strip=True) self.user_tool_label_filters = listify(self.user_tool_label_filters, do_strip=True) self.user_tool_section_filters = listify(self.user_tool_section_filters, do_strip=True) - self.has_user_tool_filters = bool(self.user_tool_filters or self.user_tool_label_filters or self.user_tool_section_filters) - - self.password_expiration_period = timedelta( - days=int(cast(SupportsInt, self.password_expiration_period)) + self.has_user_tool_filters = bool( + self.user_tool_filters or self.user_tool_label_filters or self.user_tool_section_filters ) + self.password_expiration_period = timedelta(days=int(cast(SupportsInt, self.password_expiration_period))) + if self.shed_tool_data_path: self.shed_tool_data_path = self._in_root_dir(self.shed_tool_data_path) else: self.shed_tool_data_path = self.tool_data_path - self.running_functional_tests = string_as_bool(kwargs.get('running_functional_tests', False)) + self.running_functional_tests = string_as_bool(kwargs.get("running_functional_tests", False)) if isinstance(self.hours_between_check, str): self.hours_between_check = float(self.hours_between_check) try: @@ -765,9 +790,9 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): self.use_remote_user = self.use_remote_user or self.single_user self.fetch_url_allowlist_ips = [ ipaddress.ip_network(unicodify(ip.strip())) # If it has a slash, assume 127.0.0.1/24 notation - if '/' in ip else - ipaddress.ip_address(unicodify(ip.strip())) # Otherwise interpret it as an ip address. - for ip in kwargs.get("fetch_url_allowlist", "").split(',') + if "/" in ip + else ipaddress.ip_address(unicodify(ip.strip())) # Otherwise interpret it as an ip address. + for ip in kwargs.get("fetch_url_allowlist", "").split(",") if len(ip.strip()) > 0 ] self.job_queue_cleanup_interval = int(kwargs.get("job_queue_cleanup_interval", "5")) @@ -780,36 +805,47 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): self.preserve_python_environment = "legacy_only" self.nodejs_path = kwargs.get("nodejs_path") self.container_image_cache_path = self._in_data_dir(kwargs.get("container_image_cache_path", "container_cache")) - self.output_size_limit = int(kwargs.get('output_size_limit', 0)) + self.output_size_limit = int(kwargs.get("output_size_limit", 0)) # activation_email was used until release_15.03 - activation_email = kwargs.get('activation_email') + activation_email = kwargs.get("activation_email") self.email_from = self.email_from or activation_email - self.email_domain_blocklist_content = self._load_list_from_file(self._in_config_dir(self.email_domain_blocklist_file)) if self.email_domain_blocklist_file else None - self.email_domain_allowlist_content = self._load_list_from_file(self._in_config_dir(self.email_domain_allowlist_file)) if self.email_domain_allowlist_file else None + self.email_domain_blocklist_content = ( + self._load_list_from_file(self._in_config_dir(self.email_domain_blocklist_file)) + if self.email_domain_blocklist_file + else None + ) + self.email_domain_allowlist_content = ( + self._load_list_from_file(self._in_config_dir(self.email_domain_allowlist_file)) + if self.email_domain_allowlist_file + else None + ) # These are not even beta - just experiments - don't use them unless # you want yours tools to be broken in the future. - self.enable_beta_tool_formats = string_as_bool(kwargs.get('enable_beta_tool_formats', 'False')) + self.enable_beta_tool_formats = string_as_bool(kwargs.get("enable_beta_tool_formats", "False")) - if self.workflow_resource_params_mapper and ':' not in self.workflow_resource_params_mapper: + if self.workflow_resource_params_mapper and ":" not in self.workflow_resource_params_mapper: # Assume it is not a Python function, so a file; else: a Python function self.workflow_resource_params_mapper = self._in_root_dir(self.workflow_resource_params_mapper) - self.pbs_application_server = kwargs.get('pbs_application_server', "") - self.pbs_dataset_server = kwargs.get('pbs_dataset_server', "") - self.pbs_dataset_path = kwargs.get('pbs_dataset_path', "") - self.pbs_stage_path = kwargs.get('pbs_stage_path', "") + self.pbs_application_server = kwargs.get("pbs_application_server", "") + self.pbs_dataset_server = kwargs.get("pbs_dataset_server", "") + self.pbs_dataset_path = kwargs.get("pbs_dataset_path", "") + self.pbs_stage_path = kwargs.get("pbs_stage_path", "") _sanitize_allowlist_path = self._in_managed_config_dir(self.sanitize_allowlist_file) if not os.path.isfile(_sanitize_allowlist_path): # then check old default location for deprecated in ( - self._in_managed_config_dir('sanitize_whitelist.txt'), - self._in_root_dir('config/sanitize_whitelist.txt')): + self._in_managed_config_dir("sanitize_whitelist.txt"), + self._in_root_dir("config/sanitize_whitelist.txt"), + ): if os.path.isfile(deprecated): - log.warning("The path '%s' for the 'sanitize_allowlist_file' config option is " + log.warning( + "The path '%s' for the 'sanitize_allowlist_file' config option is " "deprecated and will be no longer checked in a future release. Please consult " - "the latest version of the sample configuration file." % deprecated) + "the latest version of the sample configuration file." % deprecated + ) _sanitize_allowlist_path = deprecated break self.sanitize_allowlist_file = _sanitize_allowlist_path @@ -818,18 +854,24 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): if "trust_jupyter_notebook_conversion" not in kwargs: # if option not set, check IPython-named alternative, falling back to schema default if not set either _default = self.trust_jupyter_notebook_conversion - self.trust_jupyter_notebook_conversion = string_as_bool(kwargs.get('trust_ipython_notebook_conversion', _default)) + self.trust_jupyter_notebook_conversion = string_as_bool( + kwargs.get("trust_ipython_notebook_conversion", _default) + ) # Configuration for the message box directly below the masthead. - self.blog_url = kwargs.get('blog_url') + self.blog_url = kwargs.get("blog_url") self.user_library_import_symlink_allowlist = listify(self.user_library_import_symlink_allowlist, do_strip=True) - self.user_library_import_dir_auto_creation = self.user_library_import_dir_auto_creation if self.user_library_import_dir else False + self.user_library_import_dir_auto_creation = ( + self.user_library_import_dir_auto_creation if self.user_library_import_dir else False + ) # Searching data libraries - self.ftp_upload_dir_template = kwargs.get('ftp_upload_dir_template', '${ftp_upload_dir}%s${ftp_upload_dir_identifier}' % os.path.sep) + self.ftp_upload_dir_template = kwargs.get( + "ftp_upload_dir_template", "${ftp_upload_dir}%s${ftp_upload_dir_identifier}" % os.path.sep + ) # Support older library-specific path paste option but just default to the new # allow_path_paste value. - self.allow_library_path_paste = string_as_bool(kwargs.get('allow_library_path_paste', self.allow_path_paste)) - self.disable_library_comptypes = kwargs.get('disable_library_comptypes', '').lower().split(',') - self.check_upload_content = string_as_bool(kwargs.get('check_upload_content', True)) + self.allow_library_path_paste = string_as_bool(kwargs.get("allow_library_path_paste", self.allow_path_paste)) + self.disable_library_comptypes = kwargs.get("disable_library_comptypes", "").lower().split(",") + self.check_upload_content = string_as_bool(kwargs.get("check_upload_content", True)) # On can mildly speed up Galaxy startup time by disabling index of help, # not needed on production systems but useful if running many functional tests. self.index_tool_help = string_as_bool(kwargs.get("index_tool_help", True)) @@ -839,7 +881,9 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): # Deployers may either specify a complete list of mapping files or get the default for free and just # specify a local mapping file to adapt and extend the default one. if "conda_mapping_files" not in kwargs: - _default_mapping = self._in_root_dir(os.path.join("lib", "galaxy", "tool_util", "deps", "resolvers", "default_conda_mapping.yml")) + _default_mapping = self._in_root_dir( + os.path.join("lib", "galaxy", "tool_util", "deps", "resolvers", "default_conda_mapping.yml") + ) # dependency resolution options are consumed via config_dict - so don't populate # self, populate config_dict self.config_dict["conda_mapping_files"] = [self.local_conda_mapping_file, _default_mapping] @@ -848,16 +892,16 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): self.container_resolvers_config_file = self._in_config_dir(self.container_resolvers_config_file) # tool_dependency_dir can be "none" (in old configs). If so, set it to None - if self.tool_dependency_dir and self.tool_dependency_dir.lower() == 'none': + if self.tool_dependency_dir and self.tool_dependency_dir.lower() == "none": self.tool_dependency_dir = None if self.involucro_path is None: - target_dir = self.tool_dependency_dir or self.schema.defaults['tool_dependency_dir'] + target_dir = self.tool_dependency_dir or self.schema.defaults["tool_dependency_dir"] self.involucro_path = self._in_data_dir(os.path.join(target_dir, "involucro")) self.involucro_path = self._in_root_dir(self.involucro_path) if self.mulled_channels: self.mulled_channels = [c.strip() for c in self.mulled_channels.split(",")] # type: ignore[attr-defined] - default_job_resubmission_condition = kwargs.get('default_job_resubmission_condition', '') + default_job_resubmission_condition = kwargs.get("default_job_resubmission_condition", "") if not default_job_resubmission_condition.strip(): default_job_resubmission_condition = None self.default_job_resubmission_condition = default_job_resubmission_condition @@ -866,92 +910,100 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): if self.nginx_upload_store: self.nginx_upload_store = os.path.abspath(self.nginx_upload_store) - self.object_store = kwargs.get('object_store', 'disk') - self.object_store_check_old_style = string_as_bool(kwargs.get('object_store_check_old_style', False)) - self.object_store_cache_path = self._in_root_dir(kwargs.get("object_store_cache_path", self._in_data_dir("object_store_cache"))) + self.object_store = kwargs.get("object_store", "disk") + self.object_store_check_old_style = string_as_bool(kwargs.get("object_store_check_old_style", False)) + self.object_store_cache_path = self._in_root_dir( + kwargs.get("object_store_cache_path", self._in_data_dir("object_store_cache")) + ) self._configure_dataset_storage() # Handle AWS-specific config options for backward compatibility - if kwargs.get('aws_access_key') is not None: - self.os_access_key = kwargs.get('aws_access_key') - self.os_secret_key = kwargs.get('aws_secret_key') - self.os_bucket_name = kwargs.get('s3_bucket') - self.os_use_reduced_redundancy = kwargs.get('use_reduced_redundancy', False) + if kwargs.get("aws_access_key") is not None: + self.os_access_key = kwargs.get("aws_access_key") + self.os_secret_key = kwargs.get("aws_secret_key") + self.os_bucket_name = kwargs.get("s3_bucket") + self.os_use_reduced_redundancy = kwargs.get("use_reduced_redundancy", False) else: - self.os_access_key = kwargs.get('os_access_key') - self.os_secret_key = kwargs.get('os_secret_key') - self.os_bucket_name = kwargs.get('os_bucket_name') - self.os_use_reduced_redundancy = kwargs.get('os_use_reduced_redundancy', False) - self.os_host = kwargs.get('os_host') - self.os_port = kwargs.get('os_port') - self.os_is_secure = string_as_bool(kwargs.get('os_is_secure', True)) - self.os_conn_path = kwargs.get('os_conn_path', '/') - self.object_store_cache_size = float(kwargs.get('object_store_cache_size', -1)) - self.distributed_object_store_config_file = kwargs.get('distributed_object_store_config_file') + self.os_access_key = kwargs.get("os_access_key") + self.os_secret_key = kwargs.get("os_secret_key") + self.os_bucket_name = kwargs.get("os_bucket_name") + self.os_use_reduced_redundancy = kwargs.get("os_use_reduced_redundancy", False) + self.os_host = kwargs.get("os_host") + self.os_port = kwargs.get("os_port") + self.os_is_secure = string_as_bool(kwargs.get("os_is_secure", True)) + self.os_conn_path = kwargs.get("os_conn_path", "/") + self.object_store_cache_size = float(kwargs.get("object_store_cache_size", -1)) + self.distributed_object_store_config_file = kwargs.get("distributed_object_store_config_file") if self.distributed_object_store_config_file is not None: self.distributed_object_store_config_file = self._in_root_dir(self.distributed_object_store_config_file) - self.irods_root_collection_path = kwargs.get('irods_root_collection_path') - self.irods_default_resource = kwargs.get('irods_default_resource') + self.irods_root_collection_path = kwargs.get("irods_root_collection_path") + self.irods_default_resource = kwargs.get("irods_default_resource") # Heartbeat log file name override - if self.global_conf is not None and 'heartbeat_log' in self.global_conf: - self.heartbeat_log = self.global_conf['heartbeat_log'] + if self.global_conf is not None and "heartbeat_log" in self.global_conf: + self.heartbeat_log = self.global_conf["heartbeat_log"] # Determine which 'server:' this is - self.server_name = 'main' + self.server_name = "main" for arg in sys.argv: # Crummy, but PasteScript does not give you a way to determine this - if arg.lower().startswith('--server-name='): - self.server_name = arg.split('=', 1)[-1] + if arg.lower().startswith("--server-name="): + self.server_name = arg.split("=", 1)[-1] # Allow explicit override of server name in config params if "server_name" in kwargs: self.server_name = kwargs.get("server_name") # The application stack code may manipulate the server name. It also needs to be accessible via the get() method # for galaxy.util.facts() - self.config_dict['base_server_name'] = self.base_server_name = self.server_name + self.config_dict["base_server_name"] = self.base_server_name = self.server_name # Store all configured server names for the message queue routing self.server_names = [] for section in self.global_conf_parser.sections(): - if section.startswith('server:'): - self.server_names.append(section.replace('server:', '', 1)) + if section.startswith("server:"): + self.server_names.append(section.replace("server:", "", 1)) self._set_galaxy_infrastructure_url(kwargs) # Asynchronous execution process pools - limited functionality for now, attach_to_pools is designed to allow # webless Galaxy server processes to attach to arbitrary message queues (e.g. as job handlers) so they do not # have to be explicitly defined as such in the job configuration. - self.attach_to_pools = kwargs.get('attach_to_pools', []) or [] + self.attach_to_pools = kwargs.get("attach_to_pools", []) or [] # Store advanced job management config - self.job_handlers = [x.strip() for x in kwargs.get('job_handlers', self.server_name).split(',')] - self.default_job_handlers = [x.strip() for x in kwargs.get('default_job_handlers', ','.join(self.job_handlers)).split(',')] + self.job_handlers = [x.strip() for x in kwargs.get("job_handlers", self.server_name).split(",")] + self.default_job_handlers = [ + x.strip() for x in kwargs.get("default_job_handlers", ",".join(self.job_handlers)).split(",") + ] # Galaxy internal control queue configuration. # If specified in universe, use it, otherwise we use whatever 'real' # database is specified. Lastly, we create and use new sqlite database # (to minimize locking) as a final option. - if 'amqp_internal_connection' in kwargs: - self.amqp_internal_connection = kwargs.get('amqp_internal_connection') + if "amqp_internal_connection" in kwargs: + self.amqp_internal_connection = kwargs.get("amqp_internal_connection") # TODO Get extra amqp args as necessary for ssl - elif 'database_connection' in kwargs: + elif "database_connection" in kwargs: self.amqp_internal_connection = f"sqlalchemy+{self.database_connection}" else: - self.amqp_internal_connection = f"sqlalchemy+sqlite:///{self._in_data_dir('control.sqlite')}?isolation_level=IMMEDIATE" + self.amqp_internal_connection = ( + f"sqlalchemy+sqlite:///{self._in_data_dir('control.sqlite')}?isolation_level=IMMEDIATE" + ) self.pretty_datetime_format = expand_pretty_datetime_format(self.pretty_datetime_format) try: with open(self.user_preferences_extra_conf_path) as stream: self.user_preferences_extra = yaml.safe_load(stream) except Exception: - if self.is_set('user_preferences_extra_conf_path'): - log.warning(f'Config file ({self.user_preferences_extra_conf_path}) could not be found or is malformed.') - self.user_preferences_extra = {'preferences': {}} + if self.is_set("user_preferences_extra_conf_path"): + log.warning( + f"Config file ({self.user_preferences_extra_conf_path}) could not be found or is malformed." + ) + self.user_preferences_extra = {"preferences": {}} # Experimental: This will not be enabled by default and will hide # nonproduction code. # The api_folders refers to whether the API exposes the /folders section. - self.api_folders = string_as_bool(kwargs.get('api_folders', False)) + self.api_folders = string_as_bool(kwargs.get("api_folders", False)) # This is for testing new library browsing capabilities. - self.new_lib_browse = string_as_bool(kwargs.get('new_lib_browse', False)) + self.new_lib_browse = string_as_bool(kwargs.get("new_lib_browse", False)) # Logging configuration with logging.config.configDict: # Statistics and profiling with statsd - self.statsd_host = kwargs.get('statsd_host', '') + self.statsd_host = kwargs.get("statsd_host", "") ie_dirs = self.interactive_environment_plugins_directory self.gie_dirs = [d.strip() for d in (ie_dirs.split(",") if ie_dirs else [])] @@ -962,7 +1014,9 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): self.manage_dynamic_proxy = self.dynamic_proxy_manage # Set to false if being launched externally # InteractiveTools propagator mapping file - self.interactivetools_map = self._in_root_dir(kwargs.get("interactivetools_map", self._in_data_dir("interactivetools_map.sqlite"))) + self.interactivetools_map = self._in_root_dir( + kwargs.get("interactivetools_map", self._in_data_dir("interactivetools_map.sqlite")) + ) self.containers_conf = parse_containers_config(self.containers_config_file) @@ -989,58 +1043,56 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): self.redact_user_address_during_deletion = True self.allow_user_deletion = True - LOGGING_CONFIG_DEFAULT['formatters']['brief'] = { - 'format': '%(asctime)s %(levelname)-8s %(name)-15s %(message)s' + LOGGING_CONFIG_DEFAULT["formatters"]["brief"] = { + "format": "%(asctime)s %(levelname)-8s %(name)-15s %(message)s" } - LOGGING_CONFIG_DEFAULT['handlers']['compliance_log'] = { - 'class': 'logging.handlers.RotatingFileHandler', - 'formatter': 'brief', - 'filename': 'compliance.log', - 'backupCount': 0, + LOGGING_CONFIG_DEFAULT["handlers"]["compliance_log"] = { + "class": "logging.handlers.RotatingFileHandler", + "formatter": "brief", + "filename": "compliance.log", + "backupCount": 0, } - LOGGING_CONFIG_DEFAULT['loggers']['COMPLIANCE'] = { - 'handlers': ['compliance_log'], - 'level': 'DEBUG', - 'qualname': 'COMPLIANCE' + LOGGING_CONFIG_DEFAULT["loggers"]["COMPLIANCE"] = { + "handlers": ["compliance_log"], + "level": "DEBUG", + "qualname": "COMPLIANCE", } log_destination = kwargs.get("log_destination") - galaxy_daemon_log_destination = os.environ.get('GALAXY_DAEMON_LOG') + galaxy_daemon_log_destination = os.environ.get("GALAXY_DAEMON_LOG") if log_destination == "stdout": - LOGGING_CONFIG_DEFAULT['handlers']['console'] = { - 'class': 'logging.StreamHandler', - 'formatter': 'stack', - 'level': 'DEBUG', - 'stream': 'ext://sys.stdout', - 'filters': ['stack'] + LOGGING_CONFIG_DEFAULT["handlers"]["console"] = { + "class": "logging.StreamHandler", + "formatter": "stack", + "level": "DEBUG", + "stream": "ext://sys.stdout", + "filters": ["stack"], } elif log_destination: - LOGGING_CONFIG_DEFAULT['handlers']['console'] = { - 'class': 'logging.FileHandler', - 'formatter': 'stack', - 'level': 'DEBUG', - 'filename': log_destination, - 'filters': ['stack'] + LOGGING_CONFIG_DEFAULT["handlers"]["console"] = { + "class": "logging.FileHandler", + "formatter": "stack", + "level": "DEBUG", + "filename": log_destination, + "filters": ["stack"], } if galaxy_daemon_log_destination: - LOGGING_CONFIG_DEFAULT['handlers']['files'] = { - 'class': 'logging.FileHandler', - 'formatter': 'stack', - 'level': 'DEBUG', - 'filename': galaxy_daemon_log_destination, - 'filters': ['stack'] + LOGGING_CONFIG_DEFAULT["handlers"]["files"] = { + "class": "logging.FileHandler", + "formatter": "stack", + "level": "DEBUG", + "filename": galaxy_daemon_log_destination, + "filters": ["stack"], } - LOGGING_CONFIG_DEFAULT['root']['handlers'].append('files') + LOGGING_CONFIG_DEFAULT["root"]["handlers"].append("files") def _configure_dataset_storage(self): # The default for `file_path` has changed in 20.05; we may need to fall back to the old default - self._set_alt_paths('file_path', self._in_data_dir('files')) # this is called BEFORE guessing id/uuid - ID, UUID = 'id', 'uuid' - if self.is_set('object_store_store_by'): + self._set_alt_paths("file_path", self._in_data_dir("files")) # this is called BEFORE guessing id/uuid + ID, UUID = "id", "uuid" + if self.is_set("object_store_store_by"): if self.object_store_store_by not in [ID, UUID]: - raise Exception( - f"Invalid value for object_store_store_by [{self.object_store_store_by}]" - ) + raise Exception(f"Invalid value for object_store_store_by [{self.object_store_store_by}]") elif os.path.basename(self.file_path) == "objects": self.object_store_store_by = UUID else: @@ -1053,35 +1105,41 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): def _set_galaxy_infrastructure_url(self, kwargs): # indicate if this was not set explicitly, so dependending on the context a better default # can be used (request url in a web thread, Docker parent in IE stuff, etc.) - self.galaxy_infrastructure_url_set = kwargs.get('galaxy_infrastructure_url') is not None + self.galaxy_infrastructure_url_set = kwargs.get("galaxy_infrastructure_url") is not None if "HOST_IP" in self.galaxy_infrastructure_url: - self.galaxy_infrastructure_url = string.Template(self.galaxy_infrastructure_url).safe_substitute({ - 'HOST_IP': socket.gethostbyname(socket.gethostname()) - }) + self.galaxy_infrastructure_url = string.Template(self.galaxy_infrastructure_url).safe_substitute( + {"HOST_IP": socket.gethostbyname(socket.gethostname())} + ) if "GALAXY_WEB_PORT" in self.galaxy_infrastructure_url: - port = os.environ.get('GALAXY_WEB_PORT') + port = os.environ.get("GALAXY_WEB_PORT") if not port: - raise Exception('$GALAXY_WEB_PORT set in galaxy_infrastructure_url, but environment variable not set') - self.galaxy_infrastructure_url = string.Template(self.galaxy_infrastructure_url).safe_substitute({ - 'GALAXY_WEB_PORT': port - }) + raise Exception("$GALAXY_WEB_PORT set in galaxy_infrastructure_url, but environment variable not set") + self.galaxy_infrastructure_url = string.Template(self.galaxy_infrastructure_url).safe_substitute( + {"GALAXY_WEB_PORT": port} + ) if "UWSGI_PORT" in self.galaxy_infrastructure_url: import uwsgi - http = unicodify(uwsgi.opt['http']) + + http = unicodify(uwsgi.opt["http"]) host, port = http.split(":", 1) assert port, "galaxy_infrastructure_url depends on dynamic PORT determination but port unknown" - self.galaxy_infrastructure_url = string.Template(self.galaxy_infrastructure_url).safe_substitute({ - 'UWSGI_PORT': port - }) + self.galaxy_infrastructure_url = string.Template(self.galaxy_infrastructure_url).safe_substitute( + {"UWSGI_PORT": port} + ) def reload_sanitize_allowlist(self, explicit=True): self.sanitize_allowlist = [] if not os.path.exists(self.sanitize_allowlist_file): if explicit: - log.warning("Sanitize log file explicitly specified as '%s' but does not exist, continuing with no tools allowlisted.", self.sanitize_allowlist_file) + log.warning( + "Sanitize log file explicitly specified as '%s' but does not exist, continuing with no tools allowlisted.", + self.sanitize_allowlist_file, + ) else: with open(self.sanitize_allowlist_file) as f: - self.sanitize_allowlist = sorted(line.strip() for line in f.readlines() if line.strip() and not line.startswith('#')) + self.sanitize_allowlist = sorted( + line.strip() for line in f.readlines() if line.strip() and not line.startswith("#") + ) def ensure_tempdir(self): self._ensure_directory(self.new_file_path) @@ -1113,7 +1171,9 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): # Check for deprecated options. for key in self.config_dict.keys(): if key in self.deprecated_options: - log.warning(f"Config option '{key}' is deprecated and will be removed in a future release. Please consult the latest version of the sample configuration file.") + log.warning( + f"Config option '{key}' is deprecated and will be removed in a future release. Please consult the latest version of the sample configuration file." + ) @staticmethod def _parse_allowed_origin_hostnames(allowed_origin_hostnames): @@ -1127,7 +1187,7 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): def parse(string): # a string enclosed in fwd slashes will be parsed as a regexp: e.g. // - if string[0] == '/' and string[-1] == '/': + if string[0] == "/" and string[-1] == "/": string = string[1:-1] return re.compile(string, flags=(re.UNICODE)) return string @@ -1148,24 +1208,24 @@ def reload_config_options(current_config): if current_config._raw_config[option] != modified_config[option]: current_config._raw_config[option] = modified_config[option] setattr(current_config, option, modified_config[option]) - log.info(f'Reloaded {option}') + log.info(f"Reloaded {option}") -def get_database_engine_options(kwargs, model_prefix=''): +def get_database_engine_options(kwargs, model_prefix=""): """ Allow options for the SQLAlchemy database engine to be passed by using the prefix "database_engine_option". """ conversions: Dict[str, Callable[[Any], Union[bool, int]]] = { - 'convert_unicode': string_as_bool, - 'pool_timeout': int, - 'echo': string_as_bool, - 'echo_pool': string_as_bool, - 'pool_recycle': int, - 'pool_size': int, - 'max_overflow': int, - 'pool_threadlocal': string_as_bool, - 'server_side_cursors': string_as_bool + "convert_unicode": string_as_bool, + "pool_timeout": int, + "echo": string_as_bool, + "echo_pool": string_as_bool, + "pool_recycle": int, + "pool_size": int, + "max_overflow": int, + "pool_threadlocal": string_as_bool, + "server_side_cursors": string_as_bool, } prefix = f"{model_prefix}database_engine_option_" prefix_len = len(prefix) @@ -1194,7 +1254,7 @@ def init_models_from_config(config, map_install_models=False, object_store=None, database_query_profiling_proxy=config.database_query_profiling_proxy, object_store=object_store, trace_logger=trace_logger, - use_pbkdf2=config.get_bool('use_pbkdf2', True), + use_pbkdf2=config.get_bool("use_pbkdf2", True), slow_query_log_threshold=config.slow_query_log_threshold, thread_local_log=config.thread_local_log, log_query_counts=config.database_log_query_counts, @@ -1218,19 +1278,25 @@ def configure_logging(config): paste_configures_logging = config.global_conf_parser.has_section("loggers") else: paste_configures_logging = False - auto_configure_logging = not paste_configures_logging and string_as_bool(config.get("auto_configure_logging", "True")) + auto_configure_logging = not paste_configures_logging and string_as_bool( + config.get("auto_configure_logging", "True") + ) if auto_configure_logging: - logging_conf = config.get('logging', None) + logging_conf = config.get("logging", None) if logging_conf is None: # if using the default logging config, honor the log_level setting logging_conf = LOGGING_CONFIG_DEFAULT - if config.get('log_level', 'DEBUG') != 'DEBUG': - logging_conf['handlers']['console']['level'] = config.get('log_level', 'DEBUG') + if config.get("log_level", "DEBUG") != "DEBUG": + logging_conf["handlers"]["console"]["level"] = config.get("log_level", "DEBUG") # configure logging with logging dict in config, template *FileHandler handler filenames with the `filename_template` option - for name, conf in logging_conf.get('handlers', {}).items(): - if conf['class'].startswith('logging.') and conf['class'].endswith('FileHandler') and 'filename_template' in conf: - conf['filename'] = conf.pop('filename_template').format(**get_stack_facts(config=config)) - logging_conf['handlers'][name] = conf + for name, conf in logging_conf.get("handlers", {}).items(): + if ( + conf["class"].startswith("logging.") + and conf["class"].endswith("FileHandler") + and "filename_template" in conf + ): + conf["filename"] = conf.pop("filename_template").format(**get_stack_facts(config=config)) + logging_conf["handlers"][name] = conf logging.config.dictConfig(logging_conf) @@ -1249,15 +1315,15 @@ class ConfiguresGalaxyMixin: def wait_for_toolbox_reload(self, old_toolbox): timer = ExecutionTimer() - log.debug('Waiting for toolbox reload') + log.debug("Waiting for toolbox reload") # Wait till toolbox reload has been triggered (or more than 60 seconds have passed) while timer.elapsed < 60: if self.toolbox.has_reloaded(old_toolbox): - log.debug('Finished waiting for toolbox reload %s', timer) + log.debug("Finished waiting for toolbox reload %s", timer) break time.sleep(0.1) else: - log.warning('Waiting for toolbox reload timed out after 60 seconds') + log.warning("Waiting for toolbox reload timed out after 60 seconds") def _configure_tool_config_files(self): if self.config.shed_tool_config_file not in self.config.tool_configs: @@ -1265,17 +1331,19 @@ class ConfiguresGalaxyMixin: # The value of migrated_tools_config is the file reserved for containing only those tools that have been # eliminated from the distribution and moved to the tool shed. If migration checking is disabled, only add it if # it exists (since this may be an existing deployment where migrations were previously run). - if (os.path.exists(self.config.migrated_tools_config) - and self.config.migrated_tools_config not in self.config.tool_configs): + if ( + os.path.exists(self.config.migrated_tools_config) + and self.config.migrated_tools_config not in self.config.tool_configs + ): self.config.tool_configs.append(self.config.migrated_tools_config) def _configure_toolbox(self): + import galaxy.tools.search from galaxy import tools - from galaxy.tools.biotools import get_galaxy_biotools_metadata_source from galaxy.managers.citations import CitationsManager from galaxy.tool_util.deps import containers from galaxy.tool_util.deps.dependencies import AppInfo - import galaxy.tools.search + from galaxy.tools.biotools import get_galaxy_biotools_metadata_source if not isinstance(self, BasicSharedApp): raise Exception("Must inherit from BasicSharedApp") @@ -1308,15 +1376,19 @@ class ConfiguresGalaxyMixin: mulled_resolution_cache = None if self.config.mulled_resolution_cache_type: cache_opts = { - 'cache.type': self.config.mulled_resolution_cache_type, - 'cache.data_dir': self.config.mulled_resolution_cache_data_dir, - 'cache.lock_dir': self.config.mulled_resolution_cache_lock_dir, + "cache.type": self.config.mulled_resolution_cache_type, + "cache.data_dir": self.config.mulled_resolution_cache_data_dir, + "cache.lock_dir": self.config.mulled_resolution_cache_lock_dir, } - mulled_resolution_cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache('mulled_resolution') + mulled_resolution_cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache( + "mulled_resolution" + ) self.container_finder = containers.ContainerFinder(app_info, mulled_resolution_cache=mulled_resolution_cache) self._set_enabled_container_types() index_help = getattr(self.config, "index_tool_help", True) - self.toolbox_search = galaxy.tools.search.ToolBoxSearch(self.toolbox, index_dir=self.config.tool_search_index_dir, index_help=index_help) + self.toolbox_search = galaxy.tools.search.ToolBoxSearch( + self.toolbox, index_dir=self.config.tool_search_index_dir, index_help=index_help + ) def reindex_tool_search(self): # Call this when tools are added or removed. @@ -1330,28 +1402,37 @@ class ConfiguresGalaxyMixin: for enabled_container_type in self.container_finder._enabled_container_types(destination.params): container_types_to_destinations[enabled_container_type].append(destination) self.toolbox.dependency_manager.set_enabled_container_types(container_types_to_destinations) - self.toolbox.dependency_manager.resolver_classes.update(self.container_finder.default_container_registry.resolver_classes) - self.toolbox.dependency_manager.dependency_resolvers.extend(self.container_finder.default_container_registry.container_resolvers) + self.toolbox.dependency_manager.resolver_classes.update( + self.container_finder.default_container_registry.resolver_classes + ) + self.toolbox.dependency_manager.dependency_resolvers.extend( + self.container_finder.default_container_registry.container_resolvers + ) def _configure_tool_data_tables(self, from_shed_config): from galaxy.tools.data import ToolDataTableManager # Initialize tool data tables using the config defined by self.config.tool_data_table_config_path. - self.tool_data_tables = ToolDataTableManager(tool_data_path=self.config.tool_data_path, - config_filename=self.config.tool_data_table_config_path, - other_config_dict=self.config) + self.tool_data_tables = ToolDataTableManager( + tool_data_path=self.config.tool_data_path, + config_filename=self.config.tool_data_table_config_path, + other_config_dict=self.config, + ) # Load additional entries defined by self.config.shed_tool_data_table_config into tool data tables. try: - self.tool_data_tables.load_from_config_file(config_filename=self.config.shed_tool_data_table_config, - tool_data_path=self.tool_data_tables.tool_data_path, - from_shed_config=from_shed_config) + self.tool_data_tables.load_from_config_file( + config_filename=self.config.shed_tool_data_table_config, + tool_data_path=self.tool_data_tables.tool_data_path, + from_shed_config=from_shed_config, + ) except OSError as exc: # Missing shed_tool_data_table_config is okay if it's the default - if exc.errno != errno.ENOENT or self.config.is_set('shed_tool_data_table_config'): + if exc.errno != errno.ENOENT or self.config.is_set("shed_tool_data_table_config"): raise def _configure_datatypes_registry(self, installed_repository_manager=None): from galaxy.datatypes import registry + # Create an empty datatypes registry. self.datatypes_registry = registry.Registry(self.config) if installed_repository_manager and self.config.load_tool_shed_datatypes: @@ -1372,10 +1453,12 @@ class ConfiguresGalaxyMixin: def _configure_object_store(self, **kwds): from galaxy.objectstore import build_object_store_from_config + self.object_store = build_object_store_from_config(self.config, **kwds) def _configure_security(self): from galaxy.security import idencoding + self.security = idencoding.IdEncodingHelper(id_secret=self.config.id_secret) BaseDatabaseIdField.security = self.security @@ -1394,22 +1477,34 @@ class ConfiguresGalaxyMixin: install_db_url = self.config.install_database_connection # TODO: Consider more aggressive check here that this is not the same # database file under the hood. - combined_install_database = not(install_db_url and install_db_url != db_url) + combined_install_database = not (install_db_url and install_db_url != db_url) install_db_url = install_db_url or db_url - install_database_options = self.config.database_engine_options if combined_install_database else self.config.install_database_engine_options + install_database_options = ( + self.config.database_engine_options + if combined_install_database + else self.config.install_database_engine_options + ) if self.config.database_wait: self._wait_for_database(db_url) if getattr(self.config, "max_metadata_value_size", None): from galaxy.model import custom_types + custom_types.MAX_METADATA_VALUE_SIZE = self.config.max_metadata_value_size if check_migrate_databases: # Initialize database / check for appropriate schema version. # If this # is a new installation, we'll restrict the tool migration messaging. from galaxy.model.migrate.check import create_or_verify_database - create_or_verify_database(db_url, config_file, self.config.database_engine_options, app=self, map_install_models=combined_install_database) + + create_or_verify_database( + db_url, + config_file, + self.config.database_engine_options, + app=self, + map_install_models=combined_install_database, + ) if not combined_install_database: tsi_create_or_verify_database(install_db_url, install_database_options, app=self) @@ -1417,17 +1512,17 @@ class ConfiguresGalaxyMixin: self.config, map_install_models=combined_install_database, object_store=self.object_store, - trace_logger=getattr(self, "trace_logger", None) + trace_logger=getattr(self, "trace_logger", None), ) if combined_install_database: log.info("Install database targetting Galaxy's database configuration.") self.install_model = self.model else: from galaxy.model.tool_shed_install import mapping as install_mapping + install_db_url = self.config.install_database_connection log.info(f"Install database using its own connection {install_db_url}") - self.install_model = install_mapping.init(install_db_url, - install_database_options) + self.install_model = install_mapping.init(install_db_url, install_database_options) def _configure_signal_handlers(self, handlers): for sig, handler in handlers.items(): diff --git a/lib/galaxy/config/config_manage.py b/lib/galaxy/config/config_manage.py index 26638a11ab4..8f508a1688d 100644 --- a/lib/galaxy/config/config_manage.py +++ b/lib/galaxy/config/config_manage.py @@ -7,7 +7,11 @@ import sys import tempfile from io import StringIO from textwrap import TextWrapper -from typing import Any, List, NamedTuple +from typing import ( + Any, + List, + NamedTuple, +) import requests import yaml @@ -18,7 +22,7 @@ try: except ImportError: Core = None -if __name__ == '__main__': +if __name__ == "__main__": sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))) @@ -38,7 +42,6 @@ from galaxy.util.yaml_util import ( ordered_load, ) - DESCRIPTION = "Convert configuration files." APP_DESCRIPTION = """Application to target for operation (i.e. galaxy, tool_shed, or reports))""" @@ -49,151 +52,224 @@ EXTRA_SERVER_MESSAGE = "Additional server section after [%s] encountered [%s], w MISSING_FILTER_TYPE_MESSAGE = "Missing filter type for section [%s], it will be ignored." UNHANDLED_FILTER_TYPE_MESSAGE = "Unhandled filter type encountered [%s] for section [%s]." NO_APP_MAIN_MESSAGE = "No app:main section found, using application defaults throughout." -YAML_COMMENT_WRAPPER = TextWrapper(initial_indent="# ", subsequent_indent="# ", break_long_words=False, break_on_hyphens=False) -RST_DESCRIPTION_WRAPPER = TextWrapper(initial_indent=" ", subsequent_indent=" ", break_long_words=False, break_on_hyphens=False) +YAML_COMMENT_WRAPPER = TextWrapper( + initial_indent="# ", subsequent_indent="# ", break_long_words=False, break_on_hyphens=False +) +RST_DESCRIPTION_WRAPPER = TextWrapper( + initial_indent=" ", subsequent_indent=" ", break_long_words=False, break_on_hyphens=False +) -UWSGI_OPTIONS = dict([ - ('http', { - 'desc': """The address and port on which to listen. By default, only listen to localhost ($app_name will not be accessible over the network). Use ':$default_port' to listen on all available network interfaces.""", - 'default': '127.0.0.1:$default_port', - 'type': 'str', - }), - ('buffer-size', { - 'desc': """By default uWSGI allocates a very small buffer (4096 bytes) for the headers of each request. If you start receiving "invalid request block size" in your logs, it could mean you need a bigger buffer. We recommend at least 16384.""", - 'default': 16384, - 'type': 'int', - }), - ('processes', { - 'desc': """Number of web server (worker) processes to fork after the application has loaded. If this is set to greater than 1, thunder-lock likely should be enabled below.""", - 'default': 1, - 'type': 'int', - }), - ('threads', { - 'desc': """Number of threads for each web server process.""", - 'default': 4, - 'type': 'int', - }), - ('offload-threads', { - 'desc': """Number of threads for serving static content and handling internal routing requests.""", - 'default': 2, - 'type': 'int', - }), - ('static-map.1', { - 'key': 'static-map', - 'desc': """Mapping to serve static content.""", - 'default': '/static=static', - 'type': 'str', - }), - ('static-map.2', { - 'key': 'static-map', - 'desc': """Mapping to serve the favicon.""", - 'default': '/favicon.ico=static/favicon.ico', - 'type': 'str', - }), - ('static-safe', { - 'key': 'static-safe', - 'desc': """Allow serving certain assets out of `client`. Most modern Galaxy interfaces bundle all of this, but some older pages still serve these via symlink, requiring this rule.""", - 'default': 'client/src/assets', - 'type': 'str', - }), - ('master', { - 'desc': """Enable the master process manager. Disabled by default for maximum compatibility with CTRL+C, but should be enabled for use with --daemon and/or production deployments.""", - 'default': False, - 'type': 'bool', - }), - ('virtualenv', { - 'desc': """Path to the application's Python virtual environment. If using Conda for Galaxy's framework dependencies (not tools!), do not set this.""", - 'default': '.venv', - 'type': 'str', - }), - ('pythonpath', { - 'desc': """Path to the application's Python library.""", - 'default': 'lib', - 'type': 'str', - }), - ('module', { - 'desc': """The entry point which returns the web application (e.g. Galaxy, Reports, etc.) that you are loading.""", - 'default': '$uwsgi_module', - 'type': 'str', - }), - ('#mount', { - 'desc': """Mount the web application (e.g. Galaxy, Reports, etc.) at the given URL prefix. Cannot be used together with 'module:' above.""", - 'default': '/galaxy=$uwsgi_module', - 'type': 'str', - }), - ('manage-script-name', { - 'desc': """Make uWSGI rewrite PATH_INFO and SCRIPT_NAME according to mount-points. Set this to true if a URL prefix is used.""", - 'default': False, - 'type': 'bool', - }), - ('thunder-lock', { - 'desc': """It is usually a good idea to set this to ``true`` if processes is greater than 1.""", - 'default': False, - 'type': 'bool', - }), - ('die-on-term', { - 'desc': """Cause uWSGI to respect the traditional behavior of dying on SIGTERM (its default is to brutally reload workers)""", - 'default': True, - 'type': 'bool', - }), - ('hook-master-start.1', { - 'key': 'hook-master-start', - 'desc': """Cause uWSGI to gracefully reload workers and mules upon receipt of SIGINT (its default is to brutally kill workers)""", - 'default': 'unix_signal:2 gracefully_kill_them_all', - 'type': 'str', - }), - ('hook-master-start.2', { - 'key': 'hook-master-start', - 'desc': """Cause uWSGI to gracefully reload workers and mules upon receipt of SIGTERM (its default is to brutally kill workers)""", - 'default': 'unix_signal:15 gracefully_kill_them_all', - 'type': 'str', - }), - ('py-call-osafterfork', { - 'desc': """Feature necessary for proper mule signal handling on Python versions below 3.7.2. The default is set to false to prevent a runtime error under Python 3.7.2 and newer (see https://github.com/unbit/uwsgi/issues/1978).""", - 'default': False, - 'type': 'bool', - }), - ('enable-threads', { - 'desc': """Ensure application threads will run if `threads` is unset.""", - 'default': True, - 'type': 'bool', - }), - ('umask', { - 'desc': """uWSGI default umask. On some systems uWSGI has a default umask of 000, for Galaxy a somewhat safer default is chosen. If Galaxy submits jobs as real user then all users needs to be able to read the files, i.e. the umask needs to be '022' or the Galaxy users need to be in the same group as the Galaxy system user""", - 'default': '027', - 'type': 'str', - }), - # ('route-uri', { - # 'default': '^/proxy/ goto:proxy' - # }), - # ('route', { - # 'default': '.* last:' - # }), - # ('route-label', { - # 'default': 'proxy' - # }), - # ('route-run', { - # 'default': 'rpcvar:TARGET_HOST galaxy_dynamic_proxy_mapper ${HTTP_HOST} ${cookie[galaxysession]}' - # }), - # ('route-run', { - # 'default': "['log:Proxy ${HTTP_HOST} to ${TARGET_HOST}', 'httpdumb:${TARGET_HOST}']", - # }), - # ('http-raw-body', { - # 'default': True - # }), -]) +UWSGI_OPTIONS = dict( + [ + ( + "http", + { + "desc": """The address and port on which to listen. By default, only listen to localhost ($app_name will not be accessible over the network). Use ':$default_port' to listen on all available network interfaces.""", + "default": "127.0.0.1:$default_port", + "type": "str", + }, + ), + ( + "buffer-size", + { + "desc": """By default uWSGI allocates a very small buffer (4096 bytes) for the headers of each request. If you start receiving "invalid request block size" in your logs, it could mean you need a bigger buffer. We recommend at least 16384.""", + "default": 16384, + "type": "int", + }, + ), + ( + "processes", + { + "desc": """Number of web server (worker) processes to fork after the application has loaded. If this is set to greater than 1, thunder-lock likely should be enabled below.""", + "default": 1, + "type": "int", + }, + ), + ( + "threads", + { + "desc": """Number of threads for each web server process.""", + "default": 4, + "type": "int", + }, + ), + ( + "offload-threads", + { + "desc": """Number of threads for serving static content and handling internal routing requests.""", + "default": 2, + "type": "int", + }, + ), + ( + "static-map.1", + { + "key": "static-map", + "desc": """Mapping to serve static content.""", + "default": "/static=static", + "type": "str", + }, + ), + ( + "static-map.2", + { + "key": "static-map", + "desc": """Mapping to serve the favicon.""", + "default": "/favicon.ico=static/favicon.ico", + "type": "str", + }, + ), + ( + "static-safe", + { + "key": "static-safe", + "desc": """Allow serving certain assets out of `client`. Most modern Galaxy interfaces bundle all of this, but some older pages still serve these via symlink, requiring this rule.""", + "default": "client/src/assets", + "type": "str", + }, + ), + ( + "master", + { + "desc": """Enable the master process manager. Disabled by default for maximum compatibility with CTRL+C, but should be enabled for use with --daemon and/or production deployments.""", + "default": False, + "type": "bool", + }, + ), + ( + "virtualenv", + { + "desc": """Path to the application's Python virtual environment. If using Conda for Galaxy's framework dependencies (not tools!), do not set this.""", + "default": ".venv", + "type": "str", + }, + ), + ( + "pythonpath", + { + "desc": """Path to the application's Python library.""", + "default": "lib", + "type": "str", + }, + ), + ( + "module", + { + "desc": """The entry point which returns the web application (e.g. Galaxy, Reports, etc.) that you are loading.""", + "default": "$uwsgi_module", + "type": "str", + }, + ), + ( + "#mount", + { + "desc": """Mount the web application (e.g. Galaxy, Reports, etc.) at the given URL prefix. Cannot be used together with 'module:' above.""", + "default": "/galaxy=$uwsgi_module", + "type": "str", + }, + ), + ( + "manage-script-name", + { + "desc": """Make uWSGI rewrite PATH_INFO and SCRIPT_NAME according to mount-points. Set this to true if a URL prefix is used.""", + "default": False, + "type": "bool", + }, + ), + ( + "thunder-lock", + { + "desc": """It is usually a good idea to set this to ``true`` if processes is greater than 1.""", + "default": False, + "type": "bool", + }, + ), + ( + "die-on-term", + { + "desc": """Cause uWSGI to respect the traditional behavior of dying on SIGTERM (its default is to brutally reload workers)""", + "default": True, + "type": "bool", + }, + ), + ( + "hook-master-start.1", + { + "key": "hook-master-start", + "desc": """Cause uWSGI to gracefully reload workers and mules upon receipt of SIGINT (its default is to brutally kill workers)""", + "default": "unix_signal:2 gracefully_kill_them_all", + "type": "str", + }, + ), + ( + "hook-master-start.2", + { + "key": "hook-master-start", + "desc": """Cause uWSGI to gracefully reload workers and mules upon receipt of SIGTERM (its default is to brutally kill workers)""", + "default": "unix_signal:15 gracefully_kill_them_all", + "type": "str", + }, + ), + ( + "py-call-osafterfork", + { + "desc": """Feature necessary for proper mule signal handling on Python versions below 3.7.2. The default is set to false to prevent a runtime error under Python 3.7.2 and newer (see https://github.com/unbit/uwsgi/issues/1978).""", + "default": False, + "type": "bool", + }, + ), + ( + "enable-threads", + { + "desc": """Ensure application threads will run if `threads` is unset.""", + "default": True, + "type": "bool", + }, + ), + ( + "umask", + { + "desc": """uWSGI default umask. On some systems uWSGI has a default umask of 000, for Galaxy a somewhat safer default is chosen. If Galaxy submits jobs as real user then all users needs to be able to read the files, i.e. the umask needs to be '022' or the Galaxy users need to be in the same group as the Galaxy system user""", + "default": "027", + "type": "str", + }, + ), + # ('route-uri', { + # 'default': '^/proxy/ goto:proxy' + # }), + # ('route', { + # 'default': '.* last:' + # }), + # ('route-label', { + # 'default': 'proxy' + # }), + # ('route-run', { + # 'default': 'rpcvar:TARGET_HOST galaxy_dynamic_proxy_mapper ${HTTP_HOST} ${cookie[galaxysession]}' + # }), + # ('route-run', { + # 'default': "['log:Proxy ${HTTP_HOST} to ${TARGET_HOST}', 'httpdumb:${TARGET_HOST}']", + # }), + # ('http-raw-body', { + # 'default': True + # }), + ] +) -SHED_ONLY_UWSGI_OPTIONS = [('cron', { - 'desc': """Task for rebuilding Toolshed search indexes using the uWSGI cron-like interface.""", - 'default': "0 -1 -1 -1 -1 python scripts/tool_shed/build_ts_whoosh_index.py -c config/tool_shed.yml --config-section tool_shed", - 'type': 'str', -})] +SHED_ONLY_UWSGI_OPTIONS = [ + ( + "cron", + { + "desc": """Task for rebuilding Toolshed search indexes using the uWSGI cron-like interface.""", + "default": "0 -1 -1 -1 -1 python scripts/tool_shed/build_ts_whoosh_index.py -c config/tool_shed.yml --config-section tool_shed", + "type": "str", + }, + ) +] DROP_OPTION_VALUE = object() class _OptionAction: - def converted(self, args, app_desc, key, value): pass @@ -202,13 +278,11 @@ class _OptionAction: class _DeprecatedAction(_OptionAction): - def lint(self, args, app_desc, key, value): print(f"Option [{key}] has been deprecated, this will likely be dropped in future releases of Galaxy.") class _DeprecatedAndDroppedAction(_OptionAction): - def converted(self, args, app_desc, key, value): print(f"Option [{key}] has been deprecated and dropped. It is not included in converted configuration.") return DROP_OPTION_VALUE @@ -218,7 +292,6 @@ class _DeprecatedAndDroppedAction(_OptionAction): class _PasteAppFactoryAction(_OptionAction): - def converted(self, args, app_desc, key, value): if value not in app_desc.expected_app_factories: raise Exception(f"Ending convert process - unknown paste factory encountered [{value}]") @@ -230,7 +303,6 @@ class _PasteAppFactoryAction(_OptionAction): class _ProductionUnsafe(_OptionAction): - def __init__(self, unsafe_value): self.unsafe_value = unsafe_value @@ -242,7 +314,6 @@ class _ProductionUnsafe(_OptionAction): class _ProductionPerformance(_OptionAction): - def lint(self, args, app_desc, key, value): template = "Problem - option [%s] should not be set to [%s] in production environments - it may cause performance issues or instability." message = template % (key, value) @@ -250,14 +321,12 @@ class _ProductionPerformance(_OptionAction): class _HandleFilterWithAction(_OptionAction): - def converted(self, args, app_desc, key, value): print("filter-with converted to prefixed module load of uwsgi module, dropping from converted configuration") return DROP_OPTION_VALUE class _RenameAction(_OptionAction): - def __init__(self, new_name): self.new_name = new_name @@ -271,50 +340,50 @@ class _RenameAction(_OptionAction): OPTION_ACTIONS = { - 'use_beaker_session': _DeprecatedAndDroppedAction(), - 'use_interactive': _DeprecatedAndDroppedAction(), - 'session_type': _DeprecatedAndDroppedAction(), - 'session_data_dir': _DeprecatedAndDroppedAction(), - 'session_key': _DeprecatedAndDroppedAction(), - 'session_secret': _DeprecatedAndDroppedAction(), - 'paste.app_factory': _PasteAppFactoryAction(), - 'filter-with': _HandleFilterWithAction(), - 'debug': _ProductionUnsafe(True), - 'serve_xss_vulnerable_mimetypes': _ProductionUnsafe(True), - 'use_printdebug': _ProductionUnsafe(True), - 'id_secret': _ProductionUnsafe('USING THE DEFAULT IS NOT SECURE!'), - 'master_api_key': _ProductionUnsafe('changethis'), - 'external_service_type_config_file': _DeprecatedAndDroppedAction(), - 'external_service_type_path': _DeprecatedAndDroppedAction(), - 'enable_sequencer_communication': _DeprecatedAndDroppedAction(), - 'run_workflow_toolform_upgrade': _DeprecatedAndDroppedAction(), + "use_beaker_session": _DeprecatedAndDroppedAction(), + "use_interactive": _DeprecatedAndDroppedAction(), + "session_type": _DeprecatedAndDroppedAction(), + "session_data_dir": _DeprecatedAndDroppedAction(), + "session_key": _DeprecatedAndDroppedAction(), + "session_secret": _DeprecatedAndDroppedAction(), + "paste.app_factory": _PasteAppFactoryAction(), + "filter-with": _HandleFilterWithAction(), + "debug": _ProductionUnsafe(True), + "serve_xss_vulnerable_mimetypes": _ProductionUnsafe(True), + "use_printdebug": _ProductionUnsafe(True), + "id_secret": _ProductionUnsafe("USING THE DEFAULT IS NOT SECURE!"), + "master_api_key": _ProductionUnsafe("changethis"), + "external_service_type_config_file": _DeprecatedAndDroppedAction(), + "external_service_type_path": _DeprecatedAndDroppedAction(), + "enable_sequencer_communication": _DeprecatedAndDroppedAction(), + "run_workflow_toolform_upgrade": _DeprecatedAndDroppedAction(), # Next 4 were from library search which is no longer available. - 'enable_lucene_library_search': _DeprecatedAndDroppedAction(), - 'fulltext_max_size': _DeprecatedAndDroppedAction(), - 'fulltext_noindex_filetypes': _DeprecatedAndDroppedAction(), - 'fulltext_url': _DeprecatedAndDroppedAction(), - 'enable_beta_job_managers': _DeprecatedAndDroppedAction(), - 'enable_legacy_sample_tracking_api': _DeprecatedAction(), - 'enable_new_user_preferences': _DeprecatedAndDroppedAction(), - 'force_beta_workflow_scheduled_for_collections': _DeprecatedAndDroppedAction(), - 'force_beta_workflow_scheduled_min_steps': _DeprecatedAndDroppedAction(), - 'history_local_serial_workflow_scheduling': _ProductionPerformance(), - 'allow_library_path_paste': _RenameAction("allow_path_paste"), - 'trust_ipython_notebook_conversion': _RenameAction("trust_jupyter_notebook_conversion"), - 'enable_beta_tool_command_isolation': _DeprecatedAndDroppedAction(), - 'enable_beta_ts_api_install': _DeprecatedAndDroppedAction(), - 'single_user': _ProductionUnsafe(True), - 'tool_submission_burst_threads': _DeprecatedAndDroppedAction(), - 'tool_submission_burst_at': _DeprecatedAndDroppedAction(), - 'toolform_upgrade': _DeprecatedAndDroppedAction(), - 'enable_beta_mulled_containers': _DeprecatedAndDroppedAction(), - 'enable_communication_server': _DeprecatedAndDroppedAction(), - 'communication_server_host': _DeprecatedAndDroppedAction(), - 'communication_server_port': _DeprecatedAndDroppedAction(), - 'persistent_communication_rooms': _DeprecatedAndDroppedAction(), - 'legacy_eager_objectstore_initialization': _DeprecatedAndDroppedAction(), - 'enable_openid': _DeprecatedAndDroppedAction(), - 'openid_consumer_cache_path': _DeprecatedAndDroppedAction(), + "enable_lucene_library_search": _DeprecatedAndDroppedAction(), + "fulltext_max_size": _DeprecatedAndDroppedAction(), + "fulltext_noindex_filetypes": _DeprecatedAndDroppedAction(), + "fulltext_url": _DeprecatedAndDroppedAction(), + "enable_beta_job_managers": _DeprecatedAndDroppedAction(), + "enable_legacy_sample_tracking_api": _DeprecatedAction(), + "enable_new_user_preferences": _DeprecatedAndDroppedAction(), + "force_beta_workflow_scheduled_for_collections": _DeprecatedAndDroppedAction(), + "force_beta_workflow_scheduled_min_steps": _DeprecatedAndDroppedAction(), + "history_local_serial_workflow_scheduling": _ProductionPerformance(), + "allow_library_path_paste": _RenameAction("allow_path_paste"), + "trust_ipython_notebook_conversion": _RenameAction("trust_jupyter_notebook_conversion"), + "enable_beta_tool_command_isolation": _DeprecatedAndDroppedAction(), + "enable_beta_ts_api_install": _DeprecatedAndDroppedAction(), + "single_user": _ProductionUnsafe(True), + "tool_submission_burst_threads": _DeprecatedAndDroppedAction(), + "tool_submission_burst_at": _DeprecatedAndDroppedAction(), + "toolform_upgrade": _DeprecatedAndDroppedAction(), + "enable_beta_mulled_containers": _DeprecatedAndDroppedAction(), + "enable_communication_server": _DeprecatedAndDroppedAction(), + "communication_server_host": _DeprecatedAndDroppedAction(), + "communication_server_port": _DeprecatedAndDroppedAction(), + "persistent_communication_rooms": _DeprecatedAndDroppedAction(), + "legacy_eager_objectstore_initialization": _DeprecatedAndDroppedAction(), + "enable_openid": _DeprecatedAndDroppedAction(), + "openid_consumer_cache_path": _DeprecatedAndDroppedAction(), } @@ -348,10 +417,12 @@ class OptionValue(NamedTuple): GALAXY_APP = App( ["universe_wsgi.ini", "config/galaxy.ini"], "8080", - ["galaxy.web.buildapp:app_factory"], # TODO: Galaxy could call factory a few different things and they'd all be fine. + [ + "galaxy.web.buildapp:app_factory" + ], # TODO: Galaxy could call factory a few different things and they'd all be fine. "config/galaxy.yml", str(GALAXY_CONFIG_SCHEMA_PATH), - 'galaxy.webapps.galaxy.buildapp:uwsgi_app()', + "galaxy.webapps.galaxy.buildapp:uwsgi_app()", ) SHED_APP = App( ["tool_shed_wsgi.ini", "config/tool_shed.ini"], @@ -359,7 +430,7 @@ SHED_APP = App( ["tool_shed.webapp.buildapp:app_factory"], "config/tool_shed.yml", "lib/tool_shed/webapp/config_schema.yml", - 'tool_shed.webapp.buildapp:uwsgi_app()', + "tool_shed.webapp.buildapp:uwsgi_app()", ) REPORTS_APP = App( ["reports_wsgi.ini", "config/reports.ini"], @@ -367,7 +438,7 @@ REPORTS_APP = App( ["galaxy.webapps.reports.buildapp:app_factory"], "config/reports.yml", "lib/galaxy/webapps/reports/config_schema.yml", - 'galaxy.webapps.reports.buildapp:uwsgi_app()', + "galaxy.webapps.reports.buildapp:uwsgi_app()", ) APPS = {"galaxy": GALAXY_APP, "tool_shed": SHED_APP, "reports": REPORTS_APP} @@ -386,15 +457,11 @@ def main(argv=None): def _arg_parser(): parser = argparse.ArgumentParser(description=DESCRIPTION) - parser.add_argument('action', metavar='ACTION', type=str, - choices=list(ACTIONS.keys()), - help='action to perform') - parser.add_argument('app', metavar='APP', type=str, nargs="?", - help=APP_DESCRIPTION) - parser.add_argument('--add-comments', default=False, action="store_true") - parser.add_argument('--dry-run', default=False, action="store_true", - help=DRY_RUN_DESCRIPTION) - parser.add_argument('--galaxy_root', default=".", type=str) + parser.add_argument("action", metavar="ACTION", type=str, choices=list(ACTIONS.keys()), help="action to perform") + parser.add_argument("app", metavar="APP", type=str, nargs="?", help=APP_DESCRIPTION) + parser.add_argument("--add-comments", default=False, action="store_true") + parser.add_argument("--dry-run", default=False, action="store_true", help=DRY_RUN_DESCRIPTION) + parser.add_argument("--galaxy_root", default=".", type=str) return parser @@ -438,7 +505,7 @@ def _write_option_rst(args, rst, key, heading_level, option_value): def _build_uwsgi_schema(args, app_desc): - req = requests.get('https://raw.githubusercontent.com/unbit/uwsgi-docs/master/Options.rst') + req = requests.get("https://raw.githubusercontent.com/unbit/uwsgi-docs/master/Options.rst") rst_options = req.text last_line = None current_opt = None @@ -451,7 +518,7 @@ def _build_uwsgi_schema(args, app_desc): if line and (line == dots): current_opt = last_line option = { - 'type': 'any', + "type": "any", } options[current_opt] = option @@ -467,7 +534,7 @@ def _build_uwsgi_schema(args, app_desc): schema = { "type": "map", "desc": "uwsgi definition, see https://uwsgi-docs.readthedocs.io/en/latest/Options.html", - "mapping": options + "mapping": options, } contents = ordered_dump(schema) _write_to_file(args, contents, UWSGI_SCHEMA_PATH) @@ -534,14 +601,14 @@ def _validate(args, app_desc): if raw_config.get(app_desc.app_name) is None: raw_config[app_desc.app_name] = {} # Rewrite the file any way to merge any duplicate keys - with tempfile.NamedTemporaryFile('w', delete=False, suffix=".yml") as config_p: + with tempfile.NamedTemporaryFile("w", delete=False, suffix=".yml") as config_p: ordered_dump(raw_config, config_p) def _clean(p, k, v): - return k not in ['reloadable', 'path_resolves_to', 'per_host'] + return k not in ["reloadable", "path_resolves_to", "per_host"] clean_schema = remap(app_desc.schema.raw_schema, _clean) - with tempfile.NamedTemporaryFile('w', suffix=".yml") as fp: + with tempfile.NamedTemporaryFile("w", suffix=".yml") as fp: ordered_dump(clean_schema, fp) fp.flush() c = Core( @@ -553,14 +620,12 @@ def _validate(args, app_desc): class PrefixFilter: - def __init__(self, name, prefix): self.name = name self.prefix = prefix class GzipFilter: - def __init__(self, name): self.name = name @@ -585,7 +650,7 @@ def _run_conversion(args, app_desc): server_section = section if section.startswith("filter:"): - filter_name = section[len("filter:"):] + filter_name = section[len("filter:") :] filter_type = p.get(section, "use") if filter_type is None: MISSING_FILTER_TYPE_MESSAGE @@ -681,18 +746,20 @@ def _build_sample_yaml(args, app_desc): if not isinstance(field_value, str): continue - new_field_value = string.Template(field_value).safe_substitute(**{ - 'default_port': str(app_desc.default_port), - 'app_name': app_desc.app_name, - 'uwsgi_module': app_desc.uwsgi_module, - }) + new_field_value = string.Template(field_value).safe_substitute( + **{ + "default_port": str(app_desc.default_port), + "app_name": app_desc.app_name, + "uwsgi_module": app_desc.uwsgi_module, + } + ) value[field] = new_field_value description = getattr(schema, "description", None) if description: description = description.lstrip() as_comment = "\n".join(f"# {line}" for line in description.split("\n")) + "\n" f.write(as_comment) - _write_sample_section(args, f, 'uwsgi', Schema(UWSGI_OPTIONS), as_comment=False, uwsgi_hack=True) + _write_sample_section(args, f, "uwsgi", Schema(UWSGI_OPTIONS), as_comment=False, uwsgi_hack=True) _write_sample_section(args, f, app_desc.app_name, schema) destination = os.path.join(args.galaxy_root, app_desc.sample_destination) _write_to_file(args, f, destination) @@ -729,7 +796,7 @@ def _write_sample_section(args, f, section_header, schema, as_comment=True, uwsg option = schema.get_app_option(key) option_value = OptionValue(key, default, option) # support uWSGI "dumb YAML parser" (unbit/uwsgi#863) - key = option.get('key', key) + key = option.get("key", key) _write_option(args, f, key, option_value, as_comment=as_comment, uwsgi_hack=uwsgi_hack) @@ -835,5 +902,5 @@ ACTIONS = { } -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/lib/galaxy/config/schema.py b/lib/galaxy/config/schema.py index ebd88e05c74..b8f87c7bc85 100644 --- a/lib/galaxy/config/schema.py +++ b/lib/galaxy/config/schema.py @@ -16,12 +16,11 @@ UNKNOWN_OPTION = { "type": "str", "required": False, "unknown_option": True, - "desc": "Unknown option, may want to remove or report to Galaxy team." + "desc": "Unknown option, may want to remove or report to Galaxy team.", } class Schema: - def __init__(self, mapping): self.app_schema = mapping @@ -36,11 +35,10 @@ class Schema: class AppSchema(Schema): - def __init__(self, schema_path, app_name): self.raw_schema = self._read_schema(schema_path) self.description = self.raw_schema.get("desc", None) - app_schema = self.raw_schema['mapping'][app_name]['mapping'] + app_schema = self.raw_schema["mapping"][app_name]["mapping"] self._preprocess(app_schema) super().__init__(app_schema) @@ -55,13 +53,13 @@ class AppSchema(Schema): self._paths_to_resolve = {} # {config option: referenced config option} self._per_host_options = set() # config options that can be set using a per_host config parameter for key, data in app_schema.items(): - self._defaults[key] = data.get('default') - if data.get('reloadable'): + self._defaults[key] = data.get("default") + if data.get("reloadable"): self._reloadable_options.add(key) - if data.get('per_host'): + if data.get("per_host"): self._per_host_options.add(key) - if data.get('path_resolves_to'): - self._paths_to_resolve[key] = data.get('path_resolves_to') + if data.get("path_resolves_to"): + self._paths_to_resolve[key] = data.get("path_resolves_to") @property def defaults(self): @@ -81,16 +79,19 @@ class AppSchema(Schema): def validate_path_resolution_graph(self): """This method is for tests only: we SHOULD validate the schema's path resolution graph - as part of automated testing; but we should NOT validate it at runtime. + as part of automated testing; but we should NOT validate it at runtime. """ + def check_exists(option, key): if not option: - message = "Invalid schema: property '{}' listed as path resolution target " \ + message = ( + "Invalid schema: property '{}' listed as path resolution target " "for '{}' does not exist".format(resolves_to, key) + ) raise_error(message) def check_type_is_str_or_any(option, key): - if option.get('type') not in ('str', 'any'): + if option.get("type") not in ("str", "any"): message = f"Invalid schema: property '{key}' should have type 'str'" raise_error(message) @@ -100,9 +101,9 @@ class AppSchema(Schema): visited.clear() while key: visited.add(key) - key = self.app_schema[key].get('path_resolves_to') + key = self.app_schema[key].get("path_resolves_to") if key and key in visited: - raise_error('Invalid schema: cycle detected') + raise_error("Invalid schema: cycle detected") def raise_error(message): log.error(message) diff --git a/lib/galaxy/config/script.py b/lib/galaxy/config/script.py index 82cdad9018a..b6f724fad91 100644 --- a/lib/galaxy/config/script.py +++ b/lib/galaxy/config/script.py @@ -17,26 +17,26 @@ DESCRIPTION = "Initialize a directory with a minimal Galaxy config." HELP_CONFIG_DIR = "Directory containing the configuration files for Galaxy." HELP_DATA_DIR = "Directory containing Galaxy-created data." HELP_FORCE = "Overwrite existing files if they already exist." -HELP_WSGI_SERVER = ("Web server stack used to host Galaxy web application, and if uWSGI, which protocol to use.") +HELP_WSGI_SERVER = "Web server stack used to host Galaxy web application, and if uWSGI, which protocol to use." HELP_LIBDRMAA = ( "Configure Galaxy to submit jobs to a cluster via DRMAA by supplying the path to a libdrmaa.so file using this " "argument." ) -HELP_INSTALL = ("Install optional dependencies required by specified configuration (e.g. drmaa, etc...).") +HELP_INSTALL = "Install optional dependencies required by specified configuration (e.g. drmaa, etc...)." HELP_HOST = ( 'Host to bind Galaxy to - defaults to localhost. Specify an IP address or "all" to listen on all interfaces.' ) -HELP_PORT = ("Port to bind Galaxy to.") -HELP_DB_CONN = ("Galaxy database connection URI.") +HELP_PORT = "Port to bind Galaxy to." +HELP_DB_CONN = "Galaxy database connection URI." DEFAULT_HOST = "localhost" DEFAULT_YML = "galaxy.yml" -DEFAULT_DB_CONN = 'sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE' +DEFAULT_DB_CONN = "sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE" -SAMPLES_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), 'sample')) -GALAXY_CONFIG_TEMPLATE_FILE = os.path.join(SAMPLES_PATH, 'galaxy.yml.sample') -STATIC_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'web', 'framework', 'static')) -CLIENT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, 'client')) +SAMPLES_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "sample")) +GALAXY_CONFIG_TEMPLATE_FILE = os.path.join(SAMPLES_PATH, "galaxy.yml.sample") +STATIC_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "web", "framework", "static")) +CLIENT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, "client")) MSG_CONFIG_SUMMARY = """ For help on configuring Galaxy, consult the documentation at: \n {} @@ -51,15 +51,15 @@ Start Galaxy by running the command from directory [{}]: # whole thing into galaxy.config for templating, so for now just substitute some lines. In the future we will build # configs differently. GALAXY_CONFIG_SUBSTITUTIONS = { - ' http: 127.0.0.1:8080': ' ${uwsgi_transport}: ${host}:${port}', - ' static-map: /static=static': ' static-map: /static=${static_path}', - ' static-map: /favicon.ico=static/favicon.ico': ' static-map: /static=${static_path}/favicon.ico', - ' static-safe: client/src/assets': ' ${client_path}/src/assets', - ' virtualenv: .venv': ' #venv: .venv # not used when running installed', - ' pythonpath: lib': ' #pythonpath: lib # not used when running installed', - ' #config_dir: false': ' config_dir: ${config_dir}', - ' #data_dir: false': ' data_dir: ${data_dir}', - ' #database_connection: sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE': ' database_connection: ${database_connection}', + " http: 127.0.0.1:8080": " ${uwsgi_transport}: ${host}:${port}", + " static-map: /static=static": " static-map: /static=${static_path}", + " static-map: /favicon.ico=static/favicon.ico": " static-map: /static=${static_path}/favicon.ico", + " static-safe: client/src/assets": " ${client_path}/src/assets", + " virtualenv: .venv": " #venv: .venv # not used when running installed", + " pythonpath: lib": " #pythonpath: lib # not used when running installed", + " #config_dir: false": " config_dir: ${config_dir}", + " #data_dir: false": " data_dir: ${data_dir}", + " #database_connection: sqlite:///./database/universe.sqlite?isolation_level=IMMEDIATE": " database_connection: ${database_connection}", } @@ -68,8 +68,9 @@ def main(argv=None): arg_parser = ArgumentParser(description=DESCRIPTION) arg_parser.add_argument("--config-dir", default=".", help=HELP_CONFIG_DIR) arg_parser.add_argument("--data-dir", default="./data", help=HELP_DATA_DIR) - arg_parser.add_argument("--wsgi-server", choices=["uwsgi-http", "uwsgi-native"], default="uwsgi-http", - help=HELP_WSGI_SERVER) + arg_parser.add_argument( + "--wsgi-server", choices=["uwsgi-http", "uwsgi-native"], default="uwsgi-http", help=HELP_WSGI_SERVER + ) arg_parser.add_argument("--host", default=DEFAULT_HOST, help=HELP_HOST) arg_parser.add_argument("--port", default="8080", help=HELP_PORT) arg_parser.add_argument("--db-conn", default=DEFAULT_DB_CONN, help=HELP_DB_CONN) @@ -129,7 +130,7 @@ def _determine_mode(args): def _determine_host(args): - return '0.0.0.0' if args.host == 'all' else args.host + return "0.0.0.0" if args.host == "all" else args.host def _determine_yml_file(config_dir): @@ -140,7 +141,7 @@ def _handle_galaxy_yml(args, config_dir, data_dir): force = args.force yml_file = _determine_yml_file(config_dir) _check_file(yml_file, force) - uwsgi_transport = 'socket' if args.wsgi_server == 'uwsgi-native' else 'http' + uwsgi_transport = "socket" if args.wsgi_server == "uwsgi-native" else "http" config_dict = dict( port=args.port, host=_determine_host(args), @@ -155,16 +156,14 @@ def _handle_galaxy_yml(args, config_dir, data_dir): galaxy_config_template = [] with open(GALAXY_CONFIG_TEMPLATE_FILE) as fh: for line in fh: - line = line.rstrip('\n') + line = line.rstrip("\n") for k, v in GALAXY_CONFIG_SUBSTITUTIONS.items(): if line == k: line = v galaxy_config_template.append(line) - galaxy_config_template = string.Template('\n'.join(galaxy_config_template)) + galaxy_config_template = string.Template("\n".join(galaxy_config_template)) - galaxy_config = galaxy_config_template.safe_substitute( - **config_dict - ) + galaxy_config = galaxy_config_template.safe_substitute(**config_dict) open(yml_file, "w").write(galaxy_config) @@ -182,5 +181,5 @@ def _check_file(path, force): sys.exit(1) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/lib/galaxy/config_watchers.py b/lib/galaxy/config_watchers.py index 4601319deba..0482aa9ac40 100644 --- a/lib/galaxy/config_watchers.py +++ b/lib/galaxy/config_watchers.py @@ -31,42 +31,42 @@ class ConfigWatchers: try: # Run and wait for toolbox reload on the process that watches the config files. # The toolbox reload will update the integrated_tool_panel_file - self.app.queue_worker.send_local_control_task('reload_toolbox', get_response=True), + self.app.queue_worker.send_local_control_task("reload_toolbox", get_response=True), except Exception: save_integrated_tool_panel = True log.exception("Exception occured while reloading toolbox") - self.app.queue_worker.send_control_task('reload_toolbox', noop_self=True, kwargs={'save_integrated_tool_panel': save_integrated_tool_panel}), + self.app.queue_worker.send_control_task( + "reload_toolbox", noop_self=True, kwargs={"save_integrated_tool_panel": save_integrated_tool_panel} + ), self.tool_config_watcher = get_tool_conf_watcher( reload_callback=reload_toolbox, tool_cache=self.app.tool_cache, ) self.data_manager_config_watcher = get_tool_conf_watcher( - reload_callback=lambda: self.app.queue_worker.send_control_task('reload_data_managers'), + reload_callback=lambda: self.app.queue_worker.send_control_task("reload_data_managers"), ) - self.tool_data_watcher = get_watcher(self.app.config, 'watch_tool_data_dir', monitor_what_str='data tables') + self.tool_data_watcher = get_watcher(self.app.config, "watch_tool_data_dir", monitor_what_str="data tables") self.tool_watcher = get_tool_watcher(self, app.config) - if getattr(self.app, 'is_job_handler', False): - self.job_rule_watcher = get_watcher(app.config, 'watch_job_rules', monitor_what_str='job rules') + if getattr(self.app, "is_job_handler", False): + self.job_rule_watcher = get_watcher(app.config, "watch_job_rules", monitor_what_str="job rules") else: - self.job_rule_watcher = get_watcher(app.config, '__invalid__') - self.core_config_watcher = get_watcher( - app.config, - 'watch_core_config', - monitor_what_str='core config file' - ) - self.tour_watcher = get_watcher(app.config, 'watch_tours', monitor_what_str='tours') + self.job_rule_watcher = get_watcher(app.config, "__invalid__") + self.core_config_watcher = get_watcher(app.config, "watch_core_config", monitor_what_str="core config file") + self.tour_watcher = get_watcher(app.config, "watch_tours", monitor_what_str="tours") @property def watchers(self): - return (self.tool_watcher, - self.tool_config_watcher, - self.data_manager_config_watcher, - self.tool_data_watcher, - self.tool_watcher, - self.job_rule_watcher, - self.core_config_watcher, - self.tour_watcher) + return ( + self.tool_watcher, + self.tool_config_watcher, + self.data_manager_config_watcher, + self.tool_data_watcher, + self.tool_watcher, + self.job_rule_watcher, + self.core_config_watcher, + self.tour_watcher, + ) def change_state(self, active): if active: @@ -82,24 +82,27 @@ class ConfigWatchers: for tool_data_path in self.tool_data_paths: self.tool_data_watcher.watch_directory( tool_data_path, - callback=lambda path: self.app.queue_worker.send_control_task('reload_tool_data_tables', kwargs={'path': path}), - require_extensions=('.loc',), + callback=lambda path: self.app.queue_worker.send_control_task( + "reload_tool_data_tables", kwargs={"path": path} + ), + require_extensions=(".loc",), recursive=True, ) for job_rules_directory in self.job_rules_paths: self.job_rule_watcher.watch_directory( job_rules_directory, - callback=lambda: self.app.queue_worker.send_control_task('reload_job_rules'), + callback=lambda: self.app.queue_worker.send_control_task("reload_job_rules"), recursive=True, - ignore_extensions=('.pyc', '.pyo', '.pyd')) + ignore_extensions=(".pyc", ".pyo", ".pyd"), + ) if self.app.config.config_file: self.core_config_watcher.watch_file( self.app.config.config_file, - callback=lambda path: self.app.queue_worker.send_control_task('reload_core_config') + callback=lambda path: self.app.queue_worker.send_control_task("reload_core_config"), ) self.tour_watcher.watch_directory( self.app.config.tour_config_dir, - callback=lambda path: self.app.queue_worker.send_control_task('reload_tour', kwargs={'path': path}) + callback=lambda path: self.app.queue_worker.send_control_task("reload_tour", kwargs={"path": path}), ) self.active = True @@ -109,7 +112,7 @@ class ConfigWatchers: self.active = False def update_watch_data_table_paths(self): - if hasattr(self.tool_data_watcher, 'monitored_dirs'): + if hasattr(self.tool_data_watcher, "monitored_dirs"): for tool_data_table_path in self.tool_data_paths: if tool_data_table_path not in self.tool_data_watcher.monitored_dirs: self.tool_data_watcher.watch_directory(tool_data_table_path) @@ -117,25 +120,25 @@ class ConfigWatchers: @property def data_manager_configs(self): data_manager_configs = [] - if hasattr(self.app.config, 'data_manager_config_file'): + if hasattr(self.app.config, "data_manager_config_file"): data_manager_configs.append(self.app.config.data_manager_config_file) - if hasattr(self.app.config, 'shed_data_manager_config_file'): + if hasattr(self.app.config, "shed_data_manager_config_file"): data_manager_configs.append(self.app.config.shed_data_manager_config_file) return data_manager_configs @property def tool_data_paths(self): tool_data_paths = [] - if hasattr(self.app.config, 'tool_data_path'): + if hasattr(self.app.config, "tool_data_path"): tool_data_paths.append(self.app.config.tool_data_path) - if hasattr(self.app.config, 'shed_tool_data_path'): + if hasattr(self.app.config, "shed_tool_data_path"): tool_data_paths.append(self.app.config.shed_tool_data_path) return tool_data_paths @property def tool_config_paths(self): tool_config_paths = [] - if hasattr(self.app.config, 'tool_configs'): + if hasattr(self.app.config, "tool_configs"): tool_config_paths = self.app.config.tool_configs return tool_config_paths diff --git a/lib/galaxy/containers/__init__.py b/lib/galaxy/containers/__init__.py index ffb9c9e25c8..35d9cbf9012 100644 --- a/lib/galaxy/containers/__init__.py +++ b/lib/galaxy/containers/__init__.py @@ -12,24 +12,30 @@ import uuid from abc import ( ABCMeta, abstractmethod, - abstractproperty + abstractproperty, +) +from typing import ( + Any, + Dict, + NamedTuple, + Optional, + Type, ) -from typing import Any, Dict, NamedTuple, Optional, Type import yaml from galaxy.exceptions import ContainerCLIError from galaxy.util.submodules import import_submodules - -DEFAULT_CONTAINER_TYPE = 'docker' -DEFAULT_CONF = {'_default_': {'type': DEFAULT_CONTAINER_TYPE}} +DEFAULT_CONTAINER_TYPE = "docker" +DEFAULT_CONF = {"_default_": {"type": DEFAULT_CONTAINER_TYPE}} log = logging.getLogger(__name__) class ContainerPort(NamedTuple): """Named tuple representing ports published by a container, with attributes""" + port: int # Port number (inside the container) protocol: str # Port protocol, either ``tcp`` or ``udp`` hostaddr: str # Address or hostname where the published port can be accessed @@ -57,13 +63,11 @@ class ContainerVolume(metaclass=ABCMeta): @abstractmethod def __str__(self): - """Return this container type's string representation of the volume. - """ + """Return this container type's string representation of the volume.""" @abstractmethod def to_native(self): - """Return this container type's native representation of the volume. - """ + """Return this container type's native representation of the volume.""" @property def mode_is_valid(self): @@ -71,7 +75,6 @@ class ContainerVolume(metaclass=ABCMeta): class Container(metaclass=ABCMeta): - def __init__(self, interface, id, name=None, **kwargs): """ @@ -149,9 +152,13 @@ class Container(metaclass=ABCMeta): if port == mapping.port: return mapping if port is None: - log.warning("Container %s (%s): Don't know how to map ports to containers with multiple exposed ports " - "when a specific port is not requested. Arbitrarily choosing first: %s", - self.name, self.id, mapping) + log.warning( + "Container %s (%s): Don't know how to map ports to containers with multiple exposed ports " + "when a specific port is not requested. Arbitrarily choosing first: %s", + self.name, + self.id, + mapping, + ) return mapping else: if port is None: @@ -167,7 +174,7 @@ class ContainerInterface(metaclass=ABCMeta): container_class: Optional[Type[Container]] = None volume_class = Optional[Type[ContainerVolume]] conf_defaults: Dict[str, Optional[Any]] = { - 'name_prefix': 'galaxy_', + "name_prefix": "galaxy_", } option_map: Dict[str, Dict] = {} publish_port_list_required = False @@ -190,18 +197,18 @@ class ContainerInterface(metaclass=ABCMeta): return command def _guess_kwopt_type(self, val): - opttype = 'string' + opttype = "string" if isinstance(val, bool): - opttype = 'boolean' + opttype = "boolean" elif isinstance(val, list): - opttype = 'list' + opttype = "list" try: if isinstance(val[0], tuple) and len(val[0]) == 3: - opttype = 'list_of_kovtrips' + opttype = "list_of_kovtrips" except IndexError: pass elif isinstance(val, dict): - opttype = 'list_of_kvpairs' + opttype = "list_of_kvpairs" return opttype def _guess_kwopt_flag(self, opt): @@ -214,34 +221,35 @@ class ContainerInterface(metaclass=ABCMeta): optdef = self.option_map[opt] except KeyError: optdef = { - 'flag': self._guess_kwopt_flag(opt), - 'type': self._guess_kwopt_type(val), + "flag": self._guess_kwopt_flag(opt), + "type": self._guess_kwopt_type(val), } - log.warning("option '%s' not in %s.option_map, guessing flag '%s' type '%s'", - opt, self.__class__.__name__, optdef['flag'], optdef['type']) - opts.append(getattr(self, f"_stringify_kwopt_{optdef['type']}")(optdef['flag'], val)) - return ' '.join(opts) + log.warning( + "option '%s' not in %s.option_map, guessing flag '%s' type '%s'", + opt, + self.__class__.__name__, + optdef["flag"], + optdef["type"], + ) + opts.append(getattr(self, f"_stringify_kwopt_{optdef['type']}")(optdef["flag"], val)) + return " ".join(opts) def _stringify_kwopt_boolean(self, flag, val): - """ - """ - return f'{flag}={str(val).lower()}' + """ """ + return f"{flag}={str(val).lower()}" def _stringify_kwopt_string(self, flag, val): - """ - """ - return f'{flag} {shlex.quote(str(val))}' + """ """ + return f"{flag} {shlex.quote(str(val))}" def _stringify_kwopt_list(self, flag, val): - """ - """ + """ """ if isinstance(val, str): return self._stringify_kwopt_string(flag, val) - return ' '.join(f'{flag} {shlex.quote(str(v))}' for v in val) + return " ".join(f"{flag} {shlex.quote(str(v))}" for v in val) def _stringify_kwopt_list_of_kvpairs(self, flag, val): - """ - """ + """ """ kwopt_list = [] if isinstance(val, list): # ['foo=bar', 'baz=quux'] @@ -249,22 +257,21 @@ class ContainerInterface(metaclass=ABCMeta): else: # {'foo': 'bar', 'baz': 'quux'} for k, v in dict(val).items(): - kwopt_list.append(f'{k}={v}') + kwopt_list.append(f"{k}={v}") return self._stringify_kwopt_list(flag, kwopt_list) def _stringify_kwopt_list_of_kovtrips(self, flag, val): - """ - """ + """ """ if isinstance(val, str): return self._stringify_kwopt_string(flag, val) kwopt_list = [] for k, o, v in val: - kwopt_list.append(f'{k}{o}{v}') + kwopt_list.append(f"{k}{o}{v}") return self._stringify_kwopt_list(flag, kwopt_list) def _run_command(self, command, verbose=False): if verbose: - log.debug('running command: [%s]', command) + log.debug("running command: [%s]", command) command_list = self._normalize_command(command) p = subprocess.Popen(command_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) stdout, stderr = p.communicate() @@ -279,7 +286,8 @@ class ContainerInterface(metaclass=ABCMeta): stderr=stderr.strip(), returncode=p.returncode, command=command, - subprocess_command=command_list) + subprocess_command=command_list, + ) @property def key(self): @@ -294,25 +302,19 @@ class ContainerInterface(metaclass=ABCMeta): def set_kwopts_name(self, kwopts): if self._name_prefix is not None: - name = '{prefix}{name}'.format( - prefix=self._name_prefix, - name=kwopts.get('name', uuid.uuid4().hex) - ) - kwopts['name'] = name + name = "{prefix}{name}".format(prefix=self._name_prefix, name=kwopts.get("name", uuid.uuid4().hex)) + kwopts["name"] = name def validate_config(self): - """ - """ + """ """ self._name_prefix = self._conf.name_prefix @abstractmethod def run_in_container(self, command, image=None, **kwopts): - """ - """ + """ """ class ContainerInterfaceConfig(dict): - def __setattr__(self, name, value): self[name] = value @@ -345,7 +347,7 @@ def build_container_interfaces(containers_config_file, containers_conf=None): interface_classes = _get_interface_modules() interfaces = {} for k, conf in containers_conf.items(): - container_type = conf.get('type', DEFAULT_CONTAINER_TYPE) + container_type = conf.get("type", DEFAULT_CONTAINER_TYPE) assert container_type in interface_classes, f"unknown container interface type: {container_type}" interfaces[k] = interface_classes[container_type](conf, k, containers_config_file) return interfaces @@ -363,7 +365,7 @@ def parse_containers_config(containers_config_file): try: with open(containers_config_file) as fh: c = yaml.safe_load(fh) - conf.update(c.get('containers', {})) + conf.update(c.get("containers", {})) except OSError as exc: if exc.errno == errno.ENOENT: log.debug("config file '%s' does not exist, running with default config", containers_config_file) @@ -377,7 +379,10 @@ def _get_interface_modules(): modules = import_submodules(sys.modules[__name__]) for module in modules: module_names = [getattr(module, _) for _ in dir(module)] - classes = [_ for _ in module_names if inspect.isclass(_) - and not _ == ContainerInterface and issubclass(_, ContainerInterface)] + classes = [ + _ + for _ in module_names + if inspect.isclass(_) and not _ == ContainerInterface and issubclass(_, ContainerInterface) + ] interfaces.extend(classes) return {x.container_type: x for x in interfaces} diff --git a/lib/galaxy/containers/docker.py b/lib/galaxy/containers/docker.py index 438483d70f7..3f08f84f053 100644 --- a/lib/galaxy/containers/docker.py +++ b/lib/galaxy/containers/docker.py @@ -6,9 +6,17 @@ import logging import os import shlex from functools import partial -from itertools import cycle, repeat +from itertools import ( + cycle, + repeat, +) from time import sleep -from typing import Any, Dict, Optional, Type +from typing import ( + Any, + Dict, + Optional, + Type, +) try: import docker @@ -16,24 +24,30 @@ except ImportError: docker = None # type: ignore[assignment] try: - from requests.exceptions import ConnectionError, ReadTimeout + from requests.exceptions import ( + ConnectionError, + ReadTimeout, + ) except ImportError: ConnectionError = None # type: ignore[assignment,misc] ReadTimeout = None # type: ignore[assignment,misc] -from galaxy.containers import Container, ContainerInterface +from galaxy.containers import ( + Container, + ContainerInterface, +) from galaxy.containers.docker_decorators import ( docker_columns, - docker_json + docker_json, ) from galaxy.containers.docker_model import ( DockerContainer, - DockerVolume + DockerVolume, ) from galaxy.exceptions import ( ContainerCLIError, ContainerImageNotFound, - ContainerNotFound + ContainerNotFound, ) from galaxy.util.json import safe_dumps_formatted @@ -45,18 +59,18 @@ class DockerInterface(ContainerInterface): container_class: Type[Container] = DockerContainer volume_class = DockerVolume conf_defaults: Dict[str, Optional[Any]] = { - 'host': None, - 'tls': False, - 'force_tlsverify': False, - 'auto_remove': True, - 'image': None, - 'cpus': None, - 'memory': None, + "host": None, + "tls": False, + "force_tlsverify": False, + "auto_remove": True, + "image": None, + "cpus": None, + "memory": None, } # These values are inserted into kwopts for run commands conf_run_kwopts = ( - 'cpus', - 'memory', + "cpus", + "memory", ) def validate_config(self): @@ -92,7 +106,7 @@ class DockerInterface(ContainerInterface): """ try: inspect = self.image_inspect(image) - return inspect['RepoDigests'][0] + return inspect["RepoDigests"][0] except ContainerImageNotFound: return image @@ -107,45 +121,45 @@ class DockerInterface(ContainerInterface): class DockerCLIInterface(DockerInterface): - container_type = 'docker_cli' + container_type = "docker_cli" conf_defaults: Dict[str, Optional[Any]] = { - 'command_template': '{executable} {global_kwopts} {subcommand} {args}', - 'executable': 'docker', + "command_template": "{executable} {global_kwopts} {subcommand} {args}", + "executable": "docker", } option_map = { # `run` options - 'environment': {'flag': '--env', 'type': 'list_of_kvpairs'}, - 'volumes': {'flag': '--volume', 'type': 'docker_volumes'}, - 'name': {'flag': '--name', 'type': 'string'}, - 'detach': {'flag': '--detach', 'type': 'boolean'}, - 'publish_all_ports': {'flag': '--publish-all', 'type': 'boolean'}, - 'publish_port_random': {'flag': '--publish', 'type': 'string'}, - 'auto_remove': {'flag': '--rm', 'type': 'boolean'}, - 'cpus': {'flag': '--cpus', 'type': 'string'}, - 'memory': {'flag': '--memory', 'type': 'string'}, + "environment": {"flag": "--env", "type": "list_of_kvpairs"}, + "volumes": {"flag": "--volume", "type": "docker_volumes"}, + "name": {"flag": "--name", "type": "string"}, + "detach": {"flag": "--detach", "type": "boolean"}, + "publish_all_ports": {"flag": "--publish-all", "type": "boolean"}, + "publish_port_random": {"flag": "--publish", "type": "string"}, + "auto_remove": {"flag": "--rm", "type": "boolean"}, + "cpus": {"flag": "--cpus", "type": "string"}, + "memory": {"flag": "--memory", "type": "string"}, } def validate_config(self): - log.warning('The `docker_cli` interface is deprecated and will be removed in Galaxy 18.09, please use `docker`') + log.warning("The `docker_cli` interface is deprecated and will be removed in Galaxy 18.09, please use `docker`") super().validate_config() global_kwopts = [] if self._conf.host: - global_kwopts.append('--host') + global_kwopts.append("--host") global_kwopts.append(shlex.quote(self._conf.host)) if self._conf.force_tlsverify: - global_kwopts.append('--tlsverify') - self._docker_command = self._conf['command_template'].format( - executable=self._conf['executable'], - global_kwopts=' '.join(global_kwopts), - subcommand='{subcommand}', - args='{args}' + global_kwopts.append("--tlsverify") + self._docker_command = self._conf["command_template"].format( + executable=self._conf["executable"], + global_kwopts=" ".join(global_kwopts), + subcommand="{subcommand}", + args="{args}", ) def _filter_by_id_or_name(self, id, name): if id: - return f'--filter id={id}' + return f"--filter id={id}" elif name: - return f'--filter name={name}' + return f"--filter name={name}" return None def _stringify_kwopt_docker_volumes(self, flag, val): @@ -160,20 +174,20 @@ class DockerCLIInterface(DockerInterface): for hostvol, guestopts in val.items(): if isinstance(guestopts, str): # {'/host/vol': '/container/vol'} - kwopt_list.append(f'{hostvol}:{guestopts}') + kwopt_list.append(f"{hostvol}:{guestopts}") else: # {'/host/vol': {'bind': '/container/vol'}} # {'/host/vol': {'bind': '/container/vol', 'mode': 'rw'}} - mode = guestopts.get('mode', '') - kwopt_list.append('{vol}:{bind}{mode}'.format( - vol=hostvol, - bind=guestopts['bind'], - mode=f":{mode}" if mode else '' - )) + mode = guestopts.get("mode", "") + kwopt_list.append( + "{vol}:{bind}{mode}".format( + vol=hostvol, bind=guestopts["bind"], mode=f":{mode}" if mode else "" + ) + ) return self._stringify_kwopt_list(flag, kwopt_list) def _run_docker(self, subcommand, args=None, verbose=False): - command = self._docker_command.format(subcommand=subcommand, args=args or '') + command = self._docker_command.format(subcommand=subcommand, args=args or "") return self._run_command(command, verbose=verbose) # @@ -182,24 +196,24 @@ class DockerCLIInterface(DockerInterface): @docker_columns def ps(self, id=None, name=None): - return self._run_docker(subcommand='ps', args=self._filter_by_id_or_name(id, name)) + return self._run_docker(subcommand="ps", args=self._filter_by_id_or_name(id, name)) def run(self, command, image=None, **kwopts): - args = '{kwopts} {image} {command}'.format( + args = "{kwopts} {image} {command}".format( kwopts=self._stringify_kwopts(kwopts), image=image or self._default_image, - command=command if command else '' + command=command if command else "", ).strip() - container_id = self._run_docker(subcommand='run', args=args, verbose=True) + container_id = self._run_docker(subcommand="run", args=args, verbose=True) return DockerContainer.from_id(self, container_id) @docker_json def inspect(self, container_id): try: - return self._run_docker(subcommand='inspect', args=container_id)[0] + return self._run_docker(subcommand="inspect", args=container_id)[0] except (IndexError, ContainerCLIError) as exc: msg = f"Invalid container id: {container_id}" - if exc.stdout == '[]' and exc.stderr == f'Error: no such object: {container_id}': + if exc.stdout == "[]" and exc.stderr == f"Error: no such object: {container_id}": log.warning(msg) return [] else: @@ -208,10 +222,10 @@ class DockerCLIInterface(DockerInterface): @docker_json def image_inspect(self, image): try: - return self._run_docker(subcommand='image inspect', args=image)[0] + return self._run_docker(subcommand="image inspect", args=image)[0] except (IndexError, ContainerCLIError) as exc: msg = f"{image} not pulled, cannot get digest" - if exc.stdout == '[]' and exc.stderr == f'Error: no such image: {image}': + if exc.stdout == "[]" and exc.stderr == f"Error: no such image: {image}": log.warning(msg, image) return [] else: @@ -219,8 +233,7 @@ class DockerCLIInterface(DockerInterface): class DockerAPIClient: - """Wraps a ``docker.APIClient`` to catch exceptions. - """ + """Wraps a ``docker.APIClient`` to catch exceptions.""" _exception_retry_time = 5 _default_max_tries = 10 @@ -234,7 +247,7 @@ class DockerAPIClient: if isinstance(f, partial): f = f.func try: - return getattr(f, '__qualname__', f"{f.im_class.__name__}.{f.__name__}") + return getattr(f, "__qualname__", f"{f.im_class.__name__}.{f.__name__}") except AttributeError: return f.__name__ @@ -253,15 +266,15 @@ class DockerAPIClient: @staticmethod def _init_client(): kwargs = DockerAPIClient._client_kwargs.copy() - if DockerAPIClient._host_iter is not None and 'base_url' not in kwargs: - kwargs['base_url'] = next(DockerAPIClient._host_iter) + if DockerAPIClient._host_iter is not None and "base_url" not in kwargs: + kwargs["base_url"] = next(DockerAPIClient._host_iter) DockerAPIClient._client = docker.APIClient(*DockerAPIClient._client_args, **kwargs) - log.info('Initialized Docker API client for server: %s', kwargs.get('base_url', 'localhost')) + log.info("Initialized Docker API client for server: %s", kwargs.get("base_url", "localhost")) @staticmethod def _default_client_handler(fname, *args, **kwargs): - success_test = kwargs.pop('success_test', None) - max_tries = kwargs.pop('max_tries', DockerAPIClient._default_max_tries) + success_test = kwargs.pop("success_test", None) + max_tries = kwargs.pop("max_tries", DockerAPIClient._default_max_tries) for tries in range(1, max_tries + 1): retry_time = DockerAPIClient._exception_retry_time reinit = False @@ -272,7 +285,7 @@ class DockerAPIClient: try: r = f(*args, **kwargs) if tries > 1: - log.info('%s() succeeded on attempt %s', qualname, tries) + log.info("%s() succeeded on attempt %s", qualname, tries) return r except (ConnectionError, docker.errors.APIError, ReadTimeout) as exc: if isinstance(exc, ConnectionError): @@ -283,8 +296,9 @@ class DockerAPIClient: else: # ReadTimeout reinit = True retry_time = 0 - log.warning("Caught exception on %s(): %s: %s", - DockerAPIClient._qualname(f), exc.__class__.__name__, exc) + log.warning( + "Caught exception on %s(): %s: %s", DockerAPIClient._qualname(f), exc.__class__.__name__, exc + ) if reinit: log.warning("Reinitializing Docker API client due to connection-oriented failure") DockerAPIClient._init_client() @@ -299,7 +313,7 @@ class DockerAPIClient: return r elif tries >= max_tries: log.error("Maximum number of attempts (%s) exceeded", max_tries) - if 'response' in exc and DockerAPIClient._nonfatal_error(exc.response.status_code): + if "response" in exc and DockerAPIClient._nonfatal_error(exc.response.status_code): return None else: raise @@ -309,15 +323,14 @@ class DockerAPIClient: def __init__(self, *args, **kwargs): # Only initialize the host iterator once - host_iter = kwargs.pop('host_iter', None) + host_iter = kwargs.pop("host_iter", None) DockerAPIClient._host_iter = DockerAPIClient._host_iter or host_iter DockerAPIClient._client_args = args DockerAPIClient._client_kwargs = kwargs DockerAPIClient._init_client() def __getattr__(self, attr): - """Allow the calling of methods on this class as if it were a docker.APIClient instance. - """ + """Allow the calling of methods on this class as if it were a docker.APIClient instance.""" cattr = DockerAPIClient._unwrapped_attr(attr) if callable(cattr): return partial(DockerAPIClient._default_client_handler, attr) @@ -327,16 +340,16 @@ class DockerAPIClient: class DockerAPIInterface(DockerInterface): - container_type = 'docker' + container_type = "docker" # 'publish_port_random' and 'volumes' are special cases handled in _create_host_config() host_config_option_map = { - 'auto_remove': {}, - 'publish_all_ports': {}, - 'cpus': {'param': 'nano_cpus', 'map': lambda x: int(x * 1000000000)}, - 'memory': {'param': 'mem_limit'}, - 'binds': {}, - 'port_bindings': {}, + "auto_remove": {}, + "publish_all_ports": {}, + "cpus": {"param": "nano_cpus", "map": lambda x: int(x * 1000000000)}, + "memory": {"param": "mem_limit"}, + "binds": {}, + "port_bindings": {}, } def validate_config(self): @@ -347,14 +360,13 @@ class DockerAPIInterface(DockerInterface): @property def _client(self): # TODO: add cert options to containers conf - cert_path = os.environ.get('DOCKER_CERT_PATH') or None + cert_path = os.environ.get("DOCKER_CERT_PATH") or None if not cert_path: - cert_path = os.path.join(os.path.expanduser('~'), '.docker') + cert_path = os.path.join(os.path.expanduser("~"), ".docker") if self._conf.force_tlsverify or self._conf.tls: tls_config = docker.tls.TLSConfig( - client_cert=(os.path.join(cert_path, 'cert.pem'), - os.path.join(cert_path, 'key.pem')), - ca_cert=os.path.join(cert_path, 'ca.pem'), + client_cert=(os.path.join(cert_path, "cert.pem"), os.path.join(cert_path, "key.pem")), + ca_cert=os.path.join(cert_path, "ca.pem"), verify=self._conf.force_tlsverify, ) else: @@ -376,9 +388,9 @@ class DockerAPIInterface(DockerInterface): @staticmethod def _filter_by_id_or_name(id, name): if id: - return {'id': id} + return {"id": id} elif name: - return {'name': name} + return {"name": name} return None @staticmethod @@ -388,11 +400,11 @@ class DockerAPIInterface(DockerInterface): See :meth:`_create_docker_api_spec`. """ params = [] - if 'param' not in map_spec and 'params' not in map_spec: + if "param" not in map_spec and "params" not in map_spec: params.append(key) - elif 'param' in map_spec: - params.append(map_spec['param']) - params.extend(map_spec.get('params', ())) + elif "param" in map_spec: + params.append(map_spec["param"]) + params.extend(map_spec.get("params", ())) return params @staticmethod @@ -403,8 +415,8 @@ class DockerAPIInterface(DockerInterface): See :meth:`_create_docker_api_spec`. """ params = {} - if 'map' in map_spec: - value = map_spec['map'](value) + if "map" in map_spec: + value = map_spec["map"](value) for param in DockerAPIInterface._kwopt_to_param_names(map_spec, key): params[param] = value return params @@ -475,15 +487,17 @@ class DockerAPIInterface(DockerInterface): :returns: Instantiated ``spec_class`` object :rtype: ``type(spec_class)`` """ + def _kwopt_to_arg(map_spec, key, value, param=None): # determines whether the given param is a positional or keyword argument in docker-py and adds it to the # list of arguments - if isinstance(map_spec.get('param'), int): - spec_opts.append((map_spec.get('param'), value)) + if isinstance(map_spec.get("param"), int): + spec_opts.append((map_spec.get("param"), value)) elif param is not None: spec_kwopts[param] = value else: spec_kwopts.update(DockerAPIInterface._kwopt_to_params(map_spec, key, value)) + # positional arguments spec_opts = [] # keyword arguments @@ -491,20 +505,20 @@ class DockerAPIInterface(DockerInterface): # retrieve the option map for the docker-py object we're creating option_map = getattr(self, f"{option_map_name}_option_map") # set defaults - for key in filter(lambda k: option_map[k].get('default'), option_map.keys()): + for key in filter(lambda k: option_map[k].get("default"), option_map.keys()): map_spec = option_map[key] - _kwopt_to_arg(map_spec, key, map_spec['default']) + _kwopt_to_arg(map_spec, key, map_spec["default"]) # don't allow kwopts that start with _, those are reserved for "child" object params - for kwopt in filter(lambda k: not k.startswith('_') and k in option_map, list(kwopts.keys())): + for kwopt in filter(lambda k: not k.startswith("_") and k in option_map, list(kwopts.keys())): map_spec = option_map[kwopt] _v = kwopts.pop(kwopt) _kwopt_to_arg(map_spec, kwopt, _v) # find any child objects that need to be created and recurse to create them - for _sub_k in filter(lambda k: k.startswith('_') and 'spec_class' in option_map[k], option_map.keys()): + for _sub_k in filter(lambda k: k.startswith("_") and "spec_class" in option_map[k], option_map.keys()): map_spec = option_map[_sub_k] - param = _sub_k.lstrip('_') - _sub_v = self._create_docker_api_spec(param, map_spec['spec_class'], kwopts) - if _sub_v is not None or map_spec.get('required') or isinstance(map_spec.get('param'), int): + param = _sub_k.lstrip("_") + _sub_v = self._create_docker_api_spec(param, map_spec["spec_class"], kwopts) + if _sub_v is not None or map_spec.get("required") or isinstance(map_spec.get("param"), int): _kwopt_to_arg(map_spec, None, _sub_v, param=param) # sort positional args and make into a flat tuple if spec_opts: @@ -541,15 +555,15 @@ class DockerAPIInterface(DockerInterface): :returns: The return value of `docker.APIClient.create_host_config()` :rtype: dict """ - if 'publish_port_random' in kwopts: - port = int(kwopts.pop('publish_port_random')) - kwopts['port_bindings'] = {port: None} - kwopts['ports'] = [port] - if 'volumes' in kwopts: - paths, binds = self._volumes_to_native(kwopts.pop('volumes')) - kwopts['binds'] = binds - kwopts['volumes'] = paths - return self._create_docker_api_spec('host_config', self._client.create_host_config, kwopts) + if "publish_port_random" in kwopts: + port = int(kwopts.pop("publish_port_random")) + kwopts["port_bindings"] = {port: None} + kwopts["ports"] = [port] + if "volumes" in kwopts: + paths, binds = self._volumes_to_native(kwopts.pop("volumes")) + kwopts["binds"] = binds + kwopts["volumes"] = paths + return self._create_docker_api_spec("host_config", self._client.create_host_config, kwopts) # # docker subcommands @@ -565,7 +579,7 @@ class DockerAPIInterface(DockerInterface): host_config = self._create_host_config(kwopts) log.debug("Docker container host configuration:\n%s", safe_dumps_formatted(host_config)) log.debug("Docker container creation parameters:\n%s", safe_dumps_formatted(kwopts)) - success_test = partial(self._first, self.ps, name=kwopts['name'], running=False) + success_test = partial(self._first, self.ps, name=kwopts["name"], running=False) # this can raise exceptions, if necessary we could wrap them in a more generic "creation failed" exception class container = self._client.create_container( image, @@ -573,10 +587,10 @@ class DockerAPIInterface(DockerInterface): host_config=host_config, success_test=success_test, max_tries=5, - **kwopts + **kwopts, ) - container_id = container.get('Id') - log.debug("Starting container: %s (%s)", kwopts['name'], str(container_id)) + container_id = container.get("Id") + log.debug("Starting container: %s (%s)", kwopts["name"], str(container_id)) # start can safely be run more than once self._client.start(container=container_id) return DockerContainer.from_id(self, container_id) diff --git a/lib/galaxy/containers/docker_decorators.py b/lib/galaxy/containers/docker_decorators.py index ee5e9fa6fa2..a3e28de4860 100644 --- a/lib/galaxy/containers/docker_decorators.py +++ b/lib/galaxy/containers/docker_decorators.py @@ -28,22 +28,23 @@ def docker_columns(f): if not output: return parsed for i, c in enumerate(header): - if c != ' ' and spacect > 1: + if c != " " and spacect > 1: colidx += 1 colstarts.append(i) spacect = 0 - elif c == ' ': + elif c == " ": spacect += 1 colstarts.append(None) colheadings = [] for i in range(0, len(colstarts) - 1): - colheadings.append(header[colstarts[i]:colstarts[i + 1]].strip()) + colheadings.append(header[colstarts[i] : colstarts[i + 1]].strip()) for line in output[1:]: row = {} for i, key in enumerate(colheadings): - row[key] = line[colstarts[i]:colstarts[i + 1]].strip() + row[key] = line[colstarts[i] : colstarts[i + 1]].strip() parsed.append(row) return parsed + return parse_docker_column_output @@ -51,4 +52,5 @@ def docker_json(f): @wraps(f) def json_loads(*args, **kwargs): return json.loads(f(*args, **kwargs)) + return json_loads diff --git a/lib/galaxy/containers/docker_model.py b/lib/galaxy/containers/docker_model.py index 2d409e258db..a5f2656c46d 100644 --- a/lib/galaxy/containers/docker_model.py +++ b/lib/galaxy/containers/docker_model.py @@ -9,21 +9,21 @@ try: import docker except ImportError: from galaxy.util.bunch import Bunch + docker = Bunch(errors=Bunch(NotFound=None)) from galaxy.containers import ( Container, ContainerPort, - ContainerVolume + ContainerVolume, ) from galaxy.util import ( pretty_print_time_interval, unicodify, ) - -CPUS_LABEL = '_galaxy_cpus' -IMAGE_LABEL = '_galaxy_image' +CPUS_LABEL = "_galaxy_cpus" +IMAGE_LABEL = "_galaxy_image" CPUS_CONSTRAINT = f"node.labels.{CPUS_LABEL}" IMAGE_CONSTRAINT = f"node.labels.{IMAGE_LABEL}" @@ -31,7 +31,6 @@ log = logging.getLogger(__name__) class DockerAttributeContainer: - def __init__(self, members=None): if members is None: members = set() @@ -47,7 +46,7 @@ class DockerAttributeContainer: return hash(tuple(sorted(repr(x) for x in self._members))) def __str__(self): - return ', '.join(str(x) for x in self._members) or 'None' + return ", ".join(str(x) for x in self._members) or "None" def __iter__(self): return iter(self._members) @@ -114,11 +113,10 @@ class DockerVolume(ContainerVolume): def to_native(self): host_path = self.host_path or self.path - return (self.path, {host_path: {'bind': self.path, 'mode': self.mode}}) + return (self.path, {host_path: {"bind": self.path, "mode": self.mode}}) class DockerContainer(Container): - def __init__(self, interface, id, name=None, inspect=None): super().__init__(interface, id, name=name) self._inspect = inspect @@ -126,7 +124,7 @@ class DockerContainer(Container): @classmethod def from_id(cls, interface, id): inspect = interface.inspect(id) - return cls(interface, id, name=inspect['Name'], inspect=inspect) + return cls(interface, id, name=inspect["Name"], inspect=inspect) @property def ports(self): @@ -141,30 +139,36 @@ class DockerContainer(Container): # ] rval = [] try: - port_mappings = self.inspect['NetworkSettings']['Ports'] + port_mappings = self.inspect["NetworkSettings"]["Ports"] except KeyError: - log.warning("Failed to get ports for container %s from `docker inspect` output at " - "['NetworkSettings']['Ports']: %s: %s", self.id, exc_info=True) + log.warning( + "Failed to get ports for container %s from `docker inspect` output at " + "['NetworkSettings']['Ports']: %s: %s", + self.id, + exc_info=True, + ) return None for port_name in port_mappings: for binding in port_mappings[port_name]: - rval.append(ContainerPort( - int(port_name.split('/')[0]), - port_name.split('/')[1], - self.address, - int(binding['HostPort']), - )) + rval.append( + ContainerPort( + int(port_name.split("/")[0]), + port_name.split("/")[1], + self.address, + int(binding["HostPort"]), + ) + ) return rval @property def address(self): - if self._interface.host and self._interface.host.startswith('tcp://'): - return self._interface.host.replace('tcp://', '').split(':', 1)[0] + if self._interface.host and self._interface.host.startswith("tcp://"): + return self._interface.host.replace("tcp://", "").split(":", 1)[0] else: - return 'localhost' + return "localhost" def is_ready(self): - return self.inspect['State']['Running'] + return self.inspect["State"]["Running"] def __eq__(self, other): return self._id == other.id @@ -183,7 +187,6 @@ class DockerContainer(Container): class DockerService(Container): - def __init__(self, interface, id, name=None, image=None, inspect=None): super().__init__(interface, id, name=name) self._image = image @@ -191,15 +194,15 @@ class DockerService(Container): self._env = {} self._tasks = [] if inspect: - self._name = name or inspect['Spec']['Name'] - self._image = image or inspect['Spec']['TaskTemplate']['ContainerSpec']['Image'] + self._name = name or inspect["Spec"]["Name"] + self._image = image or inspect["Spec"]["TaskTemplate"]["ContainerSpec"]["Image"] @classmethod def from_cli(cls, interface, s, task_list): - service = cls(interface, s['ID'], name=s['NAME'], image=s['IMAGE']) + service = cls(interface, s["ID"], name=s["NAME"], image=s["IMAGE"]) for task_dict in task_list: - if task_dict['NAME'].strip().startswith(r'\_'): - continue # historical task + if task_dict["NAME"].strip().startswith(r"\_"): + continue # historical task service.task_add(DockerTask.from_cli(interface, task_dict, service=service)) return service @@ -225,29 +228,35 @@ class DockerService(Container): # ] rval = [] try: - port_mappings = self.inspect['Endpoint']['Ports'] + port_mappings = self.inspect["Endpoint"]["Ports"] except (IndexError, KeyError): - log.warning("Failed to get ports for container %s from `docker service inspect` output at " - "['Endpoint']['Ports']: %s: %s", self.id, exc_info=True) + log.warning( + "Failed to get ports for container %s from `docker service inspect` output at " + "['Endpoint']['Ports']: %s: %s", + self.id, + exc_info=True, + ) return None for binding in port_mappings: - rval.append(ContainerPort( - binding['TargetPort'], - binding['Protocol'], - self.address, # use the routing mesh - binding['PublishedPort'] - )) + rval.append( + ContainerPort( + binding["TargetPort"], + binding["Protocol"], + self.address, # use the routing mesh + binding["PublishedPort"], + ) + ) return rval @property def address(self): - if self._interface.host and self._interface.host.startswith('tcp://'): - return self._interface.host.replace('tcp://', '').split(':', 1)[0] + if self._interface.host and self._interface.host.startswith("tcp://"): + return self._interface.host.replace("tcp://", "").split(":", 1)[0] else: - return 'localhost' + return "localhost" def is_ready(self): - return self.in_state('Running', 'Running') + return self.in_state("Running", "Running") def __eq__(self, other): return self._id == other.id @@ -278,7 +287,7 @@ class DockerService(Container): state = None for task in self.tasks: state = task.state - if task.desired_state == 'running': + if task.desired_state == "running": break return state @@ -286,28 +295,26 @@ class DockerService(Container): def env(self): if not self._env: try: - for env_str in self.inspect['Spec']['TaskTemplate']['ContainerSpec']['Env']: + for env_str in self.inspect["Spec"]["TaskTemplate"]["ContainerSpec"]["Env"]: try: - self._env.update([env_str.split('=', 1)]) + self._env.update([env_str.split("=", 1)]) except ValueError: self._env[env_str] = None except KeyError as exc: - log.debug('Cannot retrieve container environment: KeyError: %s', unicodify(exc)) + log.debug("Cannot retrieve container environment: KeyError: %s", unicodify(exc)) return self._env @property def terminal(self): - """Same caveats as :meth:`state`. - """ + """Same caveats as :meth:`state`.""" for task in self.tasks: - if task.desired_state == 'running': + if task.desired_state == "running": return False return True @property def node(self): - """Same caveats as :meth:`state`. - """ + """Same caveats as :meth:`state`.""" for task in self.tasks: if task.node is not None: return task.node @@ -316,13 +323,13 @@ class DockerService(Container): @property def image(self): if self._image is None: - self._image = self.inspect['Spec']['TaskTemplate']['ContainerSpec']['Image'] + self._image = self.inspect["Spec"]["TaskTemplate"]["ContainerSpec"]["Image"] return self._image @property def cpus(self): try: - cpus = self.inspect['Spec']['TaskTemplate']['Resources']['Limits']['NanoCPUs'] / 1000000000.0 + cpus = self.inspect["Spec"]["TaskTemplate"]["Resources"]["Limits"]["NanoCPUs"] / 1000000000.0 if cpus == int(cpus): cpus = int(cpus) return cpus @@ -331,13 +338,12 @@ class DockerService(Container): @property def constraints(self): - constraints = self.inspect['Spec']['TaskTemplate']['Placement'].get('Constraints', []) + constraints = self.inspect["Spec"]["TaskTemplate"]["Placement"].get("Constraints", []) return DockerServiceConstraints.from_constraint_string_list(constraints) @property def tasks(self): - """A list of *all* tasks, including terminal ones. - """ + """A list of *all* tasks, including terminal ones.""" if not self._tasks: self._tasks = [] for task in self._interface.service_tasks(self): @@ -346,45 +352,40 @@ class DockerService(Container): @property def task_count(self): - """A count of *all* tasks, including terminal ones. - """ + """A count of *all* tasks, including terminal ones.""" return len(self.tasks) - def in_state(self, desired, current, tasks='any'): - """Indicate if one of this service's tasks matches the desired state. - """ + def in_state(self, desired, current, tasks="any"): + """Indicate if one of this service's tasks matches the desired state.""" for task in self.tasks: if task.in_state(desired, current): - if tasks == 'any': + if tasks == "any": # at least 1 task in desired state return True - elif tasks == 'all': + elif tasks == "all": # at least 1 task not in desired state return False else: - return False if tasks == 'any' else True + return False if tasks == "any" else True def constraint_add(self, name, op, value): self._interface.service_constraint_add(self.id, name, op, value) def set_cpus(self): - self.constraint_add(CPUS_LABEL, '==', self.cpus) + self.constraint_add(CPUS_LABEL, "==", self.cpus) def set_image(self): - self.constraint_add(IMAGE_LABEL, '==', self.image) + self.constraint_add(IMAGE_LABEL, "==", self.image) class DockerServiceConstraint: - def __init__(self, name=None, op=None, value=None): self._name = name self._op = op self._value = value def __eq__(self, other): - return self._name == other._name and \ - self._op == other._op and \ - self._value == other._value + return self._name == other._name and self._op == other._op and self._value == other._value def __ne__(self, other): return not self.__eq__(other) @@ -393,20 +394,20 @@ class DockerServiceConstraint: return hash((self._name, self._op, self._value)) def __repr__(self): - return f'{self.__class__.__name__}({self._name}{self._op}{self._value})' + return f"{self.__class__.__name__}({self._name}{self._op}{self._value})" def __str__(self): - return f'{self._name}{self._op}{self._value}' + return f"{self._name}{self._op}{self._value}" @staticmethod def split_constraint_string(constraint_str): - constraint = (constraint_str, '', '') - for op in '==', '!=': + constraint = (constraint_str, "", "") + for op in "==", "!=": t = constraint_str.partition(op) if len(t[0]) < len(constraint[0]): constraint = t if constraint[0] == constraint_str: - raise Exception(f'Unable to parse constraint string: {constraint_str}') + raise Exception(f"Unable to parse constraint string: {constraint_str}") return [x.strip() for x in constraint] @classmethod @@ -428,10 +429,7 @@ class DockerServiceConstraint: @property def label(self): - return DockerNodeLabel( - name=self.name.replace('node.labels.', ''), - value=self.value - ) + return DockerNodeLabel(name=self.name.replace("node.labels.", ""), value=self.value) class DockerServiceConstraints(DockerAttributeContainer): @@ -451,9 +449,7 @@ class DockerServiceConstraints(DockerAttributeContainer): class DockerNode: - - def __init__(self, interface, id=None, name=None, status=None, - availability=None, manager=False, inspect=None): + def __init__(self, interface, id=None, name=None, status=None, availability=None, manager=False, inspect=None): self._interface = interface self._id = id self._name = name @@ -462,16 +458,22 @@ class DockerNode: self._manager = manager self._inspect = inspect if inspect: - self._name = name or inspect['Description']['Hostname'] - self._status = status or inspect['Status']['State'] - self._availability = inspect['Spec']['Availability'] - self._manager = manager or inspect['Spec']['Role'] == 'manager' + self._name = name or inspect["Description"]["Hostname"] + self._status = status or inspect["Status"]["State"] + self._availability = inspect["Spec"]["Availability"] + self._manager = manager or inspect["Spec"]["Role"] == "manager" self._tasks = [] @classmethod def from_cli(cls, interface, n, task_list): - node = cls(interface, id=n['ID'], name=n['HOSTNAME'], status=n['STATUS'], - availability=n['AVAILABILITY'], manager=True if n['MANAGER STATUS'] else False) + node = cls( + interface, + id=n["ID"], + name=n["HOSTNAME"], + status=n["STATUS"], + availability=n["AVAILABILITY"], + manager=True if n["MANAGER STATUS"] else False, + ) for task_dict in task_list: node.task_add(DockerTask.from_cli(interface, task_dict, node=node)) return node @@ -498,7 +500,7 @@ class DockerNode: @property def version(self): # this changes on update so don't cache - return self._interface.node_inspect(self._id or self._name)['Version']['Index'] + return self._interface.node_inspect(self._id or self._name)["Version"]["Index"] @property def inspect(self): @@ -508,15 +510,15 @@ class DockerNode: @property def state(self): - return (f'{self._status}-{self._availability}').lower() + return (f"{self._status}-{self._availability}").lower() @property def cpus(self): - return self.inspect['Description']['Resources']['NanoCPUs'] / 1000000000 + return self.inspect["Description"]["Resources"]["NanoCPUs"] / 1000000000 @property def labels(self): - labels = self.inspect['Spec'].get('Labels', {}) or {} + labels = self.inspect["Spec"].get("Labels", {}) or {} return DockerNodeLabels.from_label_dictionary(labels) def label_add(self, label, value): @@ -534,14 +536,13 @@ class DockerNode: self.label_add(label.name, label.value) def _constraints_to_label_args(self, constraints): - constraints = filter(lambda x: x.name.startswith('node.labels.') and x.op == '==', constraints) - labels = map(lambda x: DockerNodeLabel(name=x.name.replace('node.labels.', '', 1), value=x.value), constraints) + constraints = filter(lambda x: x.name.startswith("node.labels.") and x.op == "==", constraints) + labels = map(lambda x: DockerNodeLabel(name=x.name.replace("node.labels.", "", 1), value=x.value), constraints) return labels @property def tasks(self): - """A list of *all* tasks, including terminal ones. - """ + """A list of *all* tasks, including terminal ones.""" if not self._tasks: self._tasks = [] for task in self._interface.node_tasks(self): @@ -560,15 +561,14 @@ class DockerNode: @property def task_count(self): - """A count of *all* tasks, including terminal ones. - """ + """A count of *all* tasks, including terminal ones.""" return len(self.tasks) def in_state(self, status, availability): return self._status.lower() == status.lower() and self._availability.lower() == availability.lower() def is_ok(self): - return self.in_state('Ready', 'Active') + return self.in_state("Ready", "Active") def is_managed(self): return not self._manager @@ -577,18 +577,16 @@ class DockerNode: return not self._manager and self.is_ok() and self.task_count == 0 def drain(self): - self._interface.node_update(self.id, availability='drain') + self._interface.node_update(self.id, availability="drain") class DockerNodeLabel: - def __init__(self, name=None, value=None): self._name = name self._value = value def __eq__(self, other): - return self._name == other._name and \ - self._value == other._value + return self._name == other._name and self._value == other._value def __ne__(self, other): return not self.__eq__(other) @@ -597,10 +595,10 @@ class DockerNodeLabel: return hash((self._name, self._value)) def __repr__(self): - return f'{self.__class__.__name__}({self._name}: {self._value})' + return f"{self.__class__.__name__}({self._name}: {self._value})" def __str__(self): - return f'{self._name}: {self._value}' + return f"{self._name}: {self._value}" @property def name(self): @@ -612,15 +610,11 @@ class DockerNodeLabel: @property def constraint_string(self): - return f'node.labels.{self.name}=={self.value}' + return f"node.labels.{self.name}=={self.value}" @property def constraint(self): - return DockerServiceConstraint( - name=f'node.labels.{self.name}', - op='==', - value=self.value - ) + return DockerServiceConstraint(name=f"node.labels.{self.name}", op="==", value=self.value) class DockerNodeLabels(DockerAttributeContainer): @@ -643,15 +637,26 @@ class DockerTask: # these are the possible *current* state terminal states terminal_states = ( - 'shutdown', # this is normally only a desired state but I've seen a task with it as current as well - 'complete', - 'failed', - 'rejected', - 'orphaned', + "shutdown", # this is normally only a desired state but I've seen a task with it as current as well + "complete", + "failed", + "rejected", + "orphaned", ) - def __init__(self, interface, id=None, name=None, image=None, desired_state=None, - state=None, error=None, ports=None, service=None, node=None): + def __init__( + self, + interface, + id=None, + name=None, + image=None, + desired_state=None, + state=None, + error=None, + ports=None, + service=None, + node=None, + ): self._interface = interface self._id = id self._name = name @@ -666,23 +671,41 @@ class DockerTask: @classmethod def from_cli(cls, interface, t, service=None, node=None): - state = t['CURRENT STATE'].split()[0] - return cls(interface, id=t['ID'], name=t['NAME'], image=t['IMAGE'], - desired_state=t['DESIRED STATE'], state=state, error=t['ERROR'], - ports=t['PORTS'], service=service, node=node) + state = t["CURRENT STATE"].split()[0] + return cls( + interface, + id=t["ID"], + name=t["NAME"], + image=t["IMAGE"], + desired_state=t["DESIRED STATE"], + state=state, + error=t["ERROR"], + ports=t["PORTS"], + service=service, + node=node, + ) @classmethod def from_api(cls, interface, t, service=None, node=None): - service = service or interface.service(id=t.get('ServiceID')) - node = node or interface.node(id=t.get('NodeID')) + service = service or interface.service(id=t.get("ServiceID")) + node = node or interface.node(id=t.get("NodeID")) if service: name = f"{service.name}.{str(t['Slot'])}" else: - name = t['ID'] - image = t['Spec']['ContainerSpec']['Image'].split('@', 1)[0], # remove pin - return cls(interface, id=t['ID'], name=name, image=image, desired_state=t['DesiredState'], - state=t['Status']['State'], ports=t['Status']['PortStatus'], error=t['Status']['Message'], - service=service, node=node) + name = t["ID"] + image = (t["Spec"]["ContainerSpec"]["Image"].split("@", 1)[0],) # remove pin + return cls( + interface, + id=t["ID"], + name=name, + image=image, + desired_state=t["DesiredState"], + state=t["Status"]["State"], + ports=t["Status"]["PortStatus"], + error=t["Status"]["Message"], + service=service, + node=node, + ) @property def id(self): @@ -700,15 +723,16 @@ class DockerTask: except docker.errors.NotFound: # This shouldn't be possible, appears to be some kind of Swarm bug (the node claims to have a task that # does not actually exist anymore, nor does its service exist). - log.error('Task could not be inspected because Docker claims it does not exist: %s (%s)', - self.name, self.id) + log.error( + "Task could not be inspected because Docker claims it does not exist: %s (%s)", self.name, self.id + ) return None return self._inspect @property def slot(self): try: - return self.inspect['Slot'] + return self.inspect["Slot"] except TypeError: return None @@ -716,7 +740,7 @@ class DockerTask: def node(self): if not self._node: try: - self._node = self._interface.node(id=self.inspect['NodeID']) + self._node = self._interface.node(id=self.inspect["NodeID"]) except TypeError: return None return self._node @@ -725,7 +749,7 @@ class DockerTask: def service(self): if not self._service: try: - self._service = self._interface.service(id=self.inspect['ServiceID']) + self._service = self._interface.service(id=self.inspect["ServiceID"]) except TypeError: return None return self._service @@ -733,7 +757,7 @@ class DockerTask: @property def cpus(self): try: - cpus = self.inspect['Spec']['Resources']['Reservations']['NanoCPUs'] / 1000000000.0 + cpus = self.inspect["Spec"]["Resources"]["Reservations"]["NanoCPUs"] / 1000000000.0 if cpus == int(cpus): cpus = int(cpus) return cpus @@ -744,7 +768,7 @@ class DockerTask: @property def state(self): - return (f'{self._desired_state}-{self._state}').lower() + return (f"{self._desired_state}-{self._state}").lower() @property def current_state(self): @@ -758,10 +782,10 @@ class DockerTask: def current_state_time(self): # Docker API returns a stamp w/ higher second precision than Python takes try: - stamp = self.inspect['Status']['Timestamp'] + stamp = self.inspect["Status"]["Timestamp"] except TypeError: return None - return pretty_print_time_interval(time=stamp[:stamp.index('.') + 7], precise=True, utc=stamp[-1] == 'Z') + return pretty_print_time_interval(time=stamp[: stamp.index(".") + 7], precise=True, utc=stamp[-1] == "Z") @property def desired_state(self): @@ -773,7 +797,7 @@ class DockerTask: @property def terminal(self): - return self.desired_state == 'shutdown' and self.current_state in self.terminal_states + return self.desired_state == "shutdown" and self.current_state in self.terminal_states def in_state(self, desired, current): return self.desired_state == desired.lower() and self.current_state == current.lower() diff --git a/lib/galaxy/containers/docker_swarm.py b/lib/galaxy/containers/docker_swarm.py index 6ac26c547df..840215950f9 100644 --- a/lib/galaxy/containers/docker_swarm.py +++ b/lib/galaxy/containers/docker_swarm.py @@ -6,31 +6,41 @@ import logging import os.path import subprocess from functools import partial -from typing import Any, Dict, Optional +from typing import ( + Any, + Dict, + Optional, +) try: import docker.types except ImportError: from galaxy.util.bunch import Bunch - docker = Bunch(types=Bunch( - ContainerSpec=None, - RestartPolicy=None, - Resources=None, - Placement=None, - )) + + docker = Bunch( + types=Bunch( + ContainerSpec=None, + RestartPolicy=None, + Resources=None, + Placement=None, + ) + ) from galaxy.containers.docker import ( DockerAPIInterface, DockerCLIInterface, - DockerInterface + DockerInterface, +) +from galaxy.containers.docker_decorators import ( + docker_columns, + docker_json, ) -from galaxy.containers.docker_decorators import docker_columns, docker_json from galaxy.containers.docker_model import ( CPUS_CONSTRAINT, DockerNode, DockerService, DockerTask, - IMAGE_CONSTRAINT + IMAGE_CONSTRAINT, ) from galaxy.exceptions import ContainerRunError from galaxy.util import unicodify @@ -40,25 +50,22 @@ log = logging.getLogger(__name__) SWARM_MANAGER_PATH = os.path.abspath( os.path.join( - os.path.dirname(__file__), - os.path.pardir, - os.path.pardir, - os.path.pardir, - 'scripts', - 'docker_swarm_manager.py')) + os.path.dirname(__file__), os.path.pardir, os.path.pardir, os.path.pardir, "scripts", "docker_swarm_manager.py" + ) +) class DockerSwarmInterface(DockerInterface): container_class = DockerService conf_defaults: Dict[str, Optional[Any]] = { - 'ignore_volumes': False, - 'node_prefix': None, - 'service_create_image_constraint': False, - 'service_create_cpus_constraint': False, - 'resolve_image_digest': False, - 'managed': True, - 'manager_autostart': True, + "ignore_volumes": False, + "node_prefix": None, + "service_create_image_constraint": False, + "service_create_cpus_constraint": False, + "resolve_image_digest": False, + "managed": True, + "manager_autostart": True, } publish_port_list_required = True supports_volumes = False @@ -68,48 +75,45 @@ class DockerSwarmInterface(DockerInterface): self._node_prefix = self._conf.node_prefix def run_in_container(self, command, image=None, **kwopts): - """Run a service like a detached container - """ - kwopts['replicas'] = 1 - kwopts['restart_condition'] = 'none' - if kwopts.get('publish_all_ports', False): + """Run a service like a detached container""" + kwopts["replicas"] = 1 + kwopts["restart_condition"] = "none" + if kwopts.get("publish_all_ports", False): # not supported for services # TODO: inspect image (or query registry if possible) for port list - if kwopts.get('publish_port_random', False) or kwopts.get('ports', False): + if kwopts.get("publish_port_random", False) or kwopts.get("ports", False): # assume this covers for publish_all_ports - del kwopts['publish_all_ports'] + del kwopts["publish_all_ports"] else: raise ContainerRunError( "Publishing all ports is not supported in Docker swarm" " mode, use `publish_port_random` or `ports`", image=image, - command=command + command=command, ) - if not kwopts.get('detach', True): + if not kwopts.get("detach", True): raise ContainerRunError( - "Running attached containers is not supported in Docker swarm mode", - image=image, - command=command + "Running attached containers is not supported in Docker swarm mode", image=image, command=command ) - elif kwopts.get('detach', None): - del kwopts['detach'] - if kwopts.get('volumes', None): + elif kwopts.get("detach", None): + del kwopts["detach"] + if kwopts.get("volumes", None): if self._conf.ignore_volumes: log.warning( "'volumes' kwopt is set and not supported in Docker swarm " "mode, volumes will not be passed (set 'ignore_volumes: " - "False' in containers config to fail instead): %s" % kwopts['volumes'] + "False' in containers config to fail instead): %s" % kwopts["volumes"] ) else: raise ContainerRunError( "'volumes' kwopt is set and not supported in Docker swarm " "mode (set 'ignore_volumes: True' in containers config to " - "warn instead): %s" % kwopts['volumes'], + "warn instead): %s" % kwopts["volumes"], image=image, - command=command + command=command, ) # ensure the volumes key is removed from kwopts - kwopts.pop('volumes', None) + kwopts.pop("volumes", None) service = self.service_create(command, image=image, **kwopts) self._run_swarm_manager() return service @@ -122,10 +126,18 @@ class DockerSwarmInterface(DockerInterface): if self._conf.managed and self._conf.manager_autostart: try: # sys.exectuable would be preferable to using $PATH, but sys.executable is probably uwsgi - subprocess.check_call(['python', SWARM_MANAGER_PATH, '--containers-config-file', - self.containers_config_file, '--swarm', self.key]) + subprocess.check_call( + [ + "python", + SWARM_MANAGER_PATH, + "--containers-config-file", + self.containers_config_file, + "--swarm", + self.key, + ] + ) except subprocess.CalledProcessError as exc: - log.error('Failed to launch swarm manager: %s', unicodify(exc)) + log.error("Failed to launch swarm manager: %s", unicodify(exc)) def _get_image(self, image): """Get the image string, either from the argument, or from the @@ -143,7 +155,9 @@ class DockerSwarmInterface(DockerInterface): """ if not image: image = self._conf.image - assert image is not None, "No image supplied as parameter and no image set as default in config, cannot create service" + assert ( + image is not None + ), "No image supplied as parameter and no image set as default in config, cannot create service" if self._conf.resolve_image_digest: image = self.image_repodigest(image) return image @@ -163,7 +177,7 @@ class DockerSwarmInterface(DockerInterface): def services(self, id=None, name=None): for service_dict in self.service_ls(id=id, name=name): - service_id = service_dict['ID'] + service_id = service_dict["ID"] service = DockerService(self, service_id, inspect=service_dict) if service.name.startswith(self._name_prefix): yield service @@ -174,7 +188,7 @@ class DockerSwarmInterface(DockerInterface): except StopIteration: return None - def services_in_state(self, desired, current, tasks='any'): + def services_in_state(self, desired, current, tasks="any"): for service in self.services(): if service.in_state(desired, current, tasks=tasks): yield service @@ -185,7 +199,7 @@ class DockerSwarmInterface(DockerInterface): def nodes(self, id=None, name=None): for node_dict in self.node_ls(id=id, name=name): - node_id = node_dict['ID'] + node_id = node_dict["ID"] node = DockerNode(self, node_id, inspect=node_dict) if self._node_prefix and not node.name.startswith(self._node_prefix): continue @@ -211,22 +225,22 @@ class DockerSwarmInterface(DockerInterface): # def services_waiting(self): - return self.services_in_state('Running', 'Pending') + return self.services_in_state("Running", "Pending") def services_waiting_by_constraints(self): - return self._objects_by_attribute(self.services_waiting(), 'constraints') + return self._objects_by_attribute(self.services_waiting(), "constraints") def services_completed(self): - return self.services_in_state('Shutdown', 'Complete', tasks='all') + return self.services_in_state("Shutdown", "Complete", tasks="all") def services_terminal(self): return [s for s in self.services() if s.terminal] def nodes_active(self): - return self.nodes_in_state('Ready', 'Active') + return self.nodes_in_state("Ready", "Active") def nodes_active_by_constraints(self): - return self._objects_by_attribute(self.nodes_active(), 'labels_as_constraints') + return self._objects_by_attribute(self.nodes_active(), "labels_as_constraints") # # operations @@ -239,7 +253,12 @@ class DockerSwarmInterface(DockerInterface): cleaned_service_ids.extend(self.service_rm([x.id for x in completed_services])) terminal_services = list(self.services_terminal()) for service in terminal_services: - log.warning('cleaned service in abnormal terminal state: %s (%s). state: %s', service.name, service.id, service.state) + log.warning( + "cleaned service in abnormal terminal state: %s (%s). state: %s", + service.name, + service.id, + service.state, + ) if terminal_services: cleaned_service_ids.extend(self.service_rm([x.id for x in terminal_services])) return filter(lambda x: x.id in cleaned_service_ids, completed_services + terminal_services) @@ -247,23 +266,23 @@ class DockerSwarmInterface(DockerInterface): class DockerSwarmCLIInterface(DockerSwarmInterface, DockerCLIInterface): - container_type = 'docker_swarm_cli' + container_type = "docker_swarm_cli" option_map = { # `service create` options - 'constraint': {'flag': '--constraint', 'type': 'list_of_kovtrips'}, - 'replicas': {'flag': '--replicas', 'type': 'string'}, - 'restart_condition': {'flag': '--restart-condition', 'type': 'string'}, - 'environment': {'flag': '--env', 'type': 'list_of_kvpairs'}, - 'name': {'flag': '--name', 'type': 'string'}, - 'publish_port_random': {'flag': '--publish', 'type': 'string'}, - 'cpu_limit': {'flag': '--limit-cpu', 'type': 'string'}, - 'mem_limit': {'flag': '--limit-memory', 'type': 'string'}, - 'cpu_reservation': {'flag': '--reserve-cpu', 'type': 'string'}, - 'mem_reservation': {'flag': '--reserve-memory', 'type': 'string'}, + "constraint": {"flag": "--constraint", "type": "list_of_kovtrips"}, + "replicas": {"flag": "--replicas", "type": "string"}, + "restart_condition": {"flag": "--restart-condition", "type": "string"}, + "environment": {"flag": "--env", "type": "list_of_kvpairs"}, + "name": {"flag": "--name", "type": "string"}, + "publish_port_random": {"flag": "--publish", "type": "string"}, + "cpu_limit": {"flag": "--limit-cpu", "type": "string"}, + "mem_limit": {"flag": "--limit-memory", "type": "string"}, + "cpu_reservation": {"flag": "--reserve-cpu", "type": "string"}, + "mem_reservation": {"flag": "--reserve-memory", "type": "string"}, # `service update` options - 'label_add': {'flag': '--label-add', 'type': 'list_of_kvpairs'}, - 'label_rm': {'flag': '--label-rm', 'type': 'list_of_kvpairs'}, - 'availability': {'flag': '--availability', 'type': 'string'}, + "label_add": {"flag": "--label-add", "type": "list_of_kvpairs"}, + "label_rm": {"flag": "--label-rm", "type": "list_of_kvpairs"}, + "availability": {"flag": "--availability", "type": "string"}, } # @@ -272,8 +291,8 @@ class DockerSwarmCLIInterface(DockerSwarmInterface, DockerCLIInterface): def services(self, id=None, name=None): for service_dict in self.service_ls(id=id, name=name): - service_id = service_dict['ID'] - service_name = service_dict['NAME'] + service_id = service_dict["ID"] + service_name = service_dict["NAME"] if not service_name.startswith(self._name_prefix): continue task_list = self.service_ps(service_id) @@ -281,17 +300,17 @@ class DockerSwarmCLIInterface(DockerSwarmInterface, DockerCLIInterface): def service_tasks(self, service): for task_dict in self.service_ps(service.id): - if task_dict['NAME'].strip().startswith(r'\_'): - continue # historical task + if task_dict["NAME"].strip().startswith(r"\_"): + continue # historical task yield DockerTask.from_cli(self, task_dict, service=service) def nodes(self, id=None, name=None): for node_dict in self.node_ls(id=id, name=name): - node_id = node_dict['ID'].strip(' *') - node_name = node_dict['HOSTNAME'] + node_id = node_dict["ID"].strip(" *") + node_name = node_dict["HOSTNAME"] if self._node_prefix and not node_name.startswith(self._node_prefix): continue - task_list = filter(lambda x: x['NAME'].startswith(self._name_prefix), self.node_ps(node_id)) + task_list = filter(lambda x: x["NAME"].startswith(self._name_prefix), self.node_ps(node_id)) yield DockerNode.from_cli(self, node_dict, task_list) # @@ -299,62 +318,64 @@ class DockerSwarmCLIInterface(DockerSwarmInterface, DockerCLIInterface): # def service_create(self, command, image=None, **kwopts): - if ('service_create_image_constraint' in self._conf or 'service_create_cpus_constraint' in self._conf) and 'constraint' not in kwopts: - kwopts['constraint'] = [] + if ( + "service_create_image_constraint" in self._conf or "service_create_cpus_constraint" in self._conf + ) and "constraint" not in kwopts: + kwopts["constraint"] = [] image = self._get_image(image) if self._conf.service_create_image_constraint: - kwopts['constraint'].append((IMAGE_CONSTRAINT, '==', image)) + kwopts["constraint"].append((IMAGE_CONSTRAINT, "==", image)) if self._conf.service_create_cpus_constraint: - cpus = kwopts.get('reserve_cpus', kwopts.get('limit_cpus', '1')) - kwopts['constraint'].append((CPUS_CONSTRAINT, '==', cpus)) + cpus = kwopts.get("reserve_cpus", kwopts.get("limit_cpus", "1")) + kwopts["constraint"].append((CPUS_CONSTRAINT, "==", cpus)) if self._conf.cpus: - kwopts['cpu_limit'] = self._conf.cpus - kwopts['cpu_reservation'] = self._conf.cpus + kwopts["cpu_limit"] = self._conf.cpus + kwopts["cpu_reservation"] = self._conf.cpus if self._conf.memory: - kwopts['mem_limit'] = self._conf.memory - kwopts['mem_reservation'] = self._conf.memory + kwopts["mem_limit"] = self._conf.memory + kwopts["mem_reservation"] = self._conf.memory self.set_kwopts_name(kwopts) - args = '{kwopts} {image} {command}'.format( + args = "{kwopts} {image} {command}".format( kwopts=self._stringify_kwopts(kwopts), - image=image if image else '', - command=command if command else '', + image=image if image else "", + command=command if command else "", ).strip() - service_id = self._run_docker(subcommand='service create', args=args, verbose=True) + service_id = self._run_docker(subcommand="service create", args=args, verbose=True) return DockerService.from_id(self, service_id) @docker_json def service_inspect(self, service_id): - return self._run_docker(subcommand='service inspect', args=service_id)[0] + return self._run_docker(subcommand="service inspect", args=service_id)[0] @docker_columns def service_ls(self, id=None, name=None): - return self._run_docker(subcommand='service ls', args=self._filter_by_id_or_name(id, name)) + return self._run_docker(subcommand="service ls", args=self._filter_by_id_or_name(id, name)) @docker_columns def service_ps(self, service_id): - return self._run_docker(subcommand='service ps', args=f'--no-trunc {service_id}') + return self._run_docker(subcommand="service ps", args=f"--no-trunc {service_id}") def service_rm(self, service_ids): - service_ids = ' '.join(service_ids) - return self._run_docker(subcommand='service rm', args=service_ids).splitlines() + service_ids = " ".join(service_ids) + return self._run_docker(subcommand="service rm", args=service_ids).splitlines() @docker_json def node_inspect(self, node_id): - return self._run_docker(subcommand='node inspect', args=node_id)[0] + return self._run_docker(subcommand="node inspect", args=node_id)[0] @docker_columns def node_ls(self, id=None, name=None): - return self._run_docker(subcommand='node ls', args=self._filter_by_id_or_name(id, name)) + return self._run_docker(subcommand="node ls", args=self._filter_by_id_or_name(id, name)) @docker_columns def node_ps(self, node_id): - return self._run_docker(subcommand='node ps', args=f'--no-trunc {node_id}') + return self._run_docker(subcommand="node ps", args=f"--no-trunc {node_id}") def node_update(self, node_id, **kwopts): - return self._run_docker(subcommand='node update', args='{kwopts} {node_id}'.format( - kwopts=self._stringify_kwopts(kwopts), - node_id=node_id - )) + return self._run_docker( + subcommand="node update", + args="{kwopts} {node_id}".format(kwopts=self._stringify_kwopts(kwopts), node_id=node_id), + ) @docker_json def task_inspect(self, task_id): @@ -363,51 +384,51 @@ class DockerSwarmCLIInterface(DockerSwarmInterface, DockerCLIInterface): class DockerSwarmAPIInterface(DockerSwarmInterface, DockerAPIInterface): - container_type = 'docker_swarm' + container_type = "docker_swarm" placement_option_map = { - 'constraint': {'param': 'constraints'}, + "constraint": {"param": "constraints"}, } service_mode_option_map = { - 'service_mode': {'param': 0, 'default': 'replicated'}, - 'replicas': {'default': 1}, + "service_mode": {"param": 0, "default": "replicated"}, + "replicas": {"default": 1}, } endpoint_spec_option_map: Dict[str, Dict] = { - 'ports': {}, + "ports": {}, } resources_option_map = { - 'cpus': {'params': ('cpu_limit', 'cpu_reservation'), 'map': lambda x: int(x * 1000000000)}, - 'memory': {'params': ('mem_limit', 'mem_reservation')}, + "cpus": {"params": ("cpu_limit", "cpu_reservation"), "map": lambda x: int(x * 1000000000)}, + "memory": {"params": ("mem_limit", "mem_reservation")}, } container_spec_option_map = { - 'image': {'param': 0}, - 'command': {}, - 'environment': {'param': 'env'}, - 'labels': {}, + "image": {"param": 0}, + "command": {}, + "environment": {"param": "env"}, + "labels": {}, } restart_policy_option_map = { - 'restart_condition': {'param': 'condition', 'default': 'none'}, - 'restart_delay': {'param': 'delay'}, - 'restart_max_attempts': {'param': 'max_attemps'}, + "restart_condition": {"param": "condition", "default": "none"}, + "restart_delay": {"param": "delay"}, + "restart_max_attempts": {"param": "max_attemps"}, } task_template_option_map = { - '_container_spec': {'spec_class': docker.types.ContainerSpec, 'required': True}, - '_resources': {'spec_class': docker.types.Resources}, - '_restart_policy': {'spec_class': docker.types.RestartPolicy}, - '_placement': {'spec_class': docker.types.Placement}, + "_container_spec": {"spec_class": docker.types.ContainerSpec, "required": True}, + "_resources": {"spec_class": docker.types.Resources}, + "_restart_policy": {"spec_class": docker.types.RestartPolicy}, + "_placement": {"spec_class": docker.types.Placement}, } node_spec_option_map = { - 'availability': {'param': 'Availability'}, - 'name': {'param': 'Name'}, - 'role': {'param': 'Role'}, - 'labels': {'param': 'Labels'}, + "availability": {"param": "Availability"}, + "name": {"param": "Name"}, + "role": {"param": "Role"}, + "labels": {"param": "Labels"}, } @staticmethod def create_random_port_spec(port): return { - 'Protocol': 'tcp', - 'PublishedPort': None, - 'TargetPort': port, + "Protocol": "tcp", + "PublishedPort": None, + "TargetPort": port, } # @@ -422,27 +443,27 @@ class DockerSwarmAPIInterface(DockerSwarmInterface, DockerAPIInterface): if self._conf[opt]: kwopts[opt] = self._conf[opt] # image is part of the container spec - kwopts['image'] = self._get_image(image) + kwopts["image"] = self._get_image(image) # service constraints - kwopts['constraint'] = kwopts.get('constraint', []) + kwopts["constraint"] = kwopts.get("constraint", []) if self._conf.service_create_image_constraint: - kwopts['constraint'].append(f"{IMAGE_CONSTRAINT}=={image}") + kwopts["constraint"].append(f"{IMAGE_CONSTRAINT}=={image}") if self._conf.service_create_cpus_constraint: - cpus = kwopts.get('reserve_cpus', kwopts.get('limit_cpus', '1')) - kwopts['constraint'].append(f"{CPUS_CONSTRAINT}=={cpus}") + cpus = kwopts.get("reserve_cpus", kwopts.get("limit_cpus", "1")) + kwopts["constraint"].append(f"{CPUS_CONSTRAINT}=={cpus}") # ports - if 'publish_port_random' in kwopts: - kwopts['ports'] = [DockerSwarmAPIInterface.create_random_port_spec(kwopts.pop('publish_port_random'))] + if "publish_port_random" in kwopts: + kwopts["ports"] = [DockerSwarmAPIInterface.create_random_port_spec(kwopts.pop("publish_port_random"))] # create specs - service_mode = self._create_docker_api_spec('service_mode', docker.types.ServiceMode, kwopts) - endpoint_spec = self._create_docker_api_spec('endpoint_spec', docker.types.EndpointSpec, kwopts) - task_template = self._create_docker_api_spec('task_template', docker.types.TaskTemplate, kwopts) + service_mode = self._create_docker_api_spec("service_mode", docker.types.ServiceMode, kwopts) + endpoint_spec = self._create_docker_api_spec("endpoint_spec", docker.types.EndpointSpec, kwopts) + task_template = self._create_docker_api_spec("task_template", docker.types.TaskTemplate, kwopts) self.set_kwopts_name(kwopts) log.debug("Docker service task template:\n%s", safe_dumps_formatted(task_template)) log.debug("Docker service endpoint specification:\n%s", safe_dumps_formatted(endpoint_spec)) log.debug("Docker service mode:\n%s", safe_dumps_formatted(service_mode)) log.debug("Docker service creation parameters:\n%s", safe_dumps_formatted(kwopts)) - success_test = partial(self._first, self.service_ls, name=kwopts['name']) + success_test = partial(self._first, self.service_ls, name=kwopts["name"]) # this can raise exceptions, if necessary we could wrap them in a more generic "creation failed" exception class service = self._client.create_service( task_template, @@ -450,9 +471,10 @@ class DockerSwarmAPIInterface(DockerSwarmInterface, DockerAPIInterface): endpoint_spec=endpoint_spec, success_test=success_test, max_tries=5, - **kwopts) - service_id = service.get('ID') - log.debug('Created service: %s (%s)', kwopts['name'], service_id) + **kwopts, + ) + service_id = service.get("ID") + log.debug("Created service: %s (%s)", kwopts["name"], service_id) return DockerService.from_id(self, service_id) def service_inspect(self, service_id): @@ -463,7 +485,7 @@ class DockerSwarmAPIInterface(DockerSwarmInterface, DockerAPIInterface): # roughly `docker service ps` def service_ps(self, service_id): - return self.task_ls(filters={'service': service_id}) + return self.task_ls(filters={"service": service_id}) def service_rm(self, service_ids): r = [] @@ -480,15 +502,15 @@ class DockerSwarmAPIInterface(DockerSwarmInterface, DockerAPIInterface): # roughly `docker node ps` def node_ps(self, node_id): - return self.task_ls(filters={'node': node_id}) + return self.task_ls(filters={"node": node_id}) def node_update(self, node_id, **kwopts): node = DockerNode.from_id(self, node_id) - spec = node.inspect['Spec'] - if 'label_add' in kwopts: - kwopts['labels'] = spec.get('Labels', {}) - kwopts['labels'].update(kwopts.pop('label_add')) - spec.update(self._create_docker_api_spec('node_spec', dict, kwopts)) + spec = node.inspect["Spec"] + if "label_add" in kwopts: + kwopts["labels"] = spec.get("Labels", {}) + kwopts["labels"].update(kwopts.pop("label_add")) + spec.update(self._create_docker_api_spec("node_spec", dict, kwopts)) return self._client.update_node(node.id, node.version, node_spec=spec) def task_inspect(self, task_id): diff --git a/lib/galaxy/datatypes/_schema.py b/lib/galaxy/datatypes/_schema.py index 6efb7cc4565..50c1774b614 100644 --- a/lib/galaxy/datatypes/_schema.py +++ b/lib/galaxy/datatypes/_schema.py @@ -7,44 +7,23 @@ from typing import ( from pydantic import ( BaseModel, Field, - HttpUrl + HttpUrl, ) class CompositeFileInfo(BaseModel): - name: str = Field( - ..., # Mark this field as required - title="Name", - description="The name of this composite file" - ) - optional: bool = Field( - title="Optional", - description="" # TODO add description - ) - mimetype: Optional[str] = Field( - title="MIME type", - description="The MIME type of this file" - ) + name: str = Field(..., title="Name", description="The name of this composite file") # Mark this field as required + optional: bool = Field(title="Optional", description="") # TODO add description + mimetype: Optional[str] = Field(title="MIME type", description="The MIME type of this file") description: Optional[str] = Field( - title="Description", - description="Summary description of the purpouse of this file" + title="Description", description="Summary description of the purpouse of this file" ) substitute_name_with_metadata: Optional[str] = Field( - title="Substitute name with metadata", - description="" # TODO add description - ) - is_binary: bool = Field( - title="Is binary", - description="Whether this file is a binary file" - ) - to_posix_lines: bool = Field( - title="To posix lines", - description="" # TODO add description - ) - space_to_tab: bool = Field( - title="Spaces to tabulation", - description="" # TODO add description + title="Substitute name with metadata", description="" # TODO add description ) + is_binary: bool = Field(title="Is binary", description="Whether this file is a binary file") + to_posix_lines: bool = Field(title="To posix lines", description="") # TODO add description + space_to_tab: bool = Field(title="Spaces to tabulation", description="") # TODO add description class DatatypeDetails(BaseModel): @@ -52,26 +31,21 @@ class DatatypeDetails(BaseModel): ..., # Mark this field as required title="Extension", description="The data type’s Dataset file extension", - example="bed" - ) - description: Optional[str] = Field( - title="Description", - description="A summary description for this data type" + example="bed", ) + description: Optional[str] = Field(title="Description", description="A summary description for this data type") description_url: Optional[HttpUrl] = Field( title="Description URL", description="The URL to a detailed description for this datatype", - example="https://wiki.galaxyproject.org/Learn/Datatypes#Bed" + example="https://wiki.galaxyproject.org/Learn/Datatypes#Bed", ) display_in_upload: bool = Field( default=False, title="Display in upload", - description="If True, the associated file extension will be displayed in the `File Format` select list in the `Upload File from your computer` tool in the `Get Data` tool section of the tool panel" + description="If True, the associated file extension will be displayed in the `File Format` select list in the `Upload File from your computer` tool in the `Get Data` tool section of the tool panel", ) composite_files: Optional[List[CompositeFileInfo]] = Field( - default=None, - title="Composite files", - description="A collection of files composing this data type" + default=None, title="Composite files", description="A collection of files composing this data type" ) @@ -123,7 +97,4 @@ class DatatypeConverter(BaseModel): class DatatypeConverterList(BaseModel): - __root__: List[DatatypeConverter] = Field( - title='List of data type converters', - default=[] - ) + __root__: List[DatatypeConverter] = Field(title="List of data type converters", default=[]) diff --git a/lib/galaxy/datatypes/annotation.py b/lib/galaxy/datatypes/annotation.py index 8d2e70d87b3..b8eb217bc66 100644 --- a/lib/galaxy/datatypes/annotation.py +++ b/lib/galaxy/datatypes/annotation.py @@ -2,7 +2,10 @@ import logging import tarfile from galaxy.datatypes.binary import CompressedArchive -from galaxy.datatypes.data import get_file_peek, Text +from galaxy.datatypes.data import ( + get_file_peek, + Text, +) from galaxy.datatypes.sniff import ( build_sniff_from_prefix, FilePrefix, @@ -22,8 +25,8 @@ class SnapHmm(Text): dataset.peek = get_file_peek(dataset.file_name) dataset.blurb = "SNAP HMM model" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disc' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disc" def display_peek(self, dataset): try: @@ -35,13 +38,14 @@ class SnapHmm(Text): """ SNAP model files start with zoeHMM """ - return file_prefix.startswith('zoeHMM') + return file_prefix.startswith("zoeHMM") class Augustus(CompressedArchive): """ - Class describing an Augustus prediction model + Class describing an Augustus prediction model """ + file_ext = "augustus" edam_data = "data_0950" compressed = True @@ -51,8 +55,8 @@ class Augustus(CompressedArchive): dataset.peek = "Augustus model" dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -65,19 +69,21 @@ class Augustus(CompressedArchive): Augustus archives always contain the same files """ if filename and tarfile.is_tarfile(filename): - with tarfile.open(filename, 'r') as temptar: + with tarfile.open(filename, "r") as temptar: for f in temptar: if not f.isfile(): continue - if f.name.endswith('_exon_probs.pbl') \ - or f.name.endswith('_igenic_probs.pbl') \ - or f.name.endswith('_intron_probs.pbl') \ - or f.name.endswith('_metapars.cfg') \ - or f.name.endswith('_metapars.utr.cfg') \ - or f.name.endswith('_parameters.cfg') \ - or f.name.endswith('_parameters.cgp.cfg') \ - or f.name.endswith('_utr_probs.pbl') \ - or f.name.endswith('_weightmatrix.txt'): + if ( + f.name.endswith("_exon_probs.pbl") + or f.name.endswith("_igenic_probs.pbl") + or f.name.endswith("_intron_probs.pbl") + or f.name.endswith("_metapars.cfg") + or f.name.endswith("_metapars.utr.cfg") + or f.name.endswith("_parameters.cfg") + or f.name.endswith("_parameters.cgp.cfg") + or f.name.endswith("_utr_probs.pbl") + or f.name.endswith("_weightmatrix.txt") + ): return True else: return False diff --git a/lib/galaxy/datatypes/anvio.py b/lib/galaxy/datatypes/anvio.py index 9e0d9828b57..51c8f9eecd7 100644 --- a/lib/galaxy/datatypes/anvio.py +++ b/lib/galaxy/datatypes/anvio.py @@ -19,8 +19,9 @@ class AnvioComposite(Html): Base class to use for Anvi'o composite datatypes. Generally consist of a sqlite database, plus optional additional files """ + file_ext = "anvio_composite" - composite_type = 'auto_primary_file' + composite_type = "auto_primary_file" def generate_primary_file(self, dataset=None): """ @@ -32,12 +33,12 @@ class AnvioComposite(Html): if defined_files: rval.append("

This composite dataset is composed of the following defined files:

    ") for composite_name, composite_file in defined_files: - opt_text = '' + opt_text = "" if composite_file.optional: - opt_text = ' (optional)' - missing_text = '' + opt_text = " (optional)" + missing_text = "" if not os.path.exists(os.path.join(dataset.extra_files_path, composite_name)): - missing_text = ' (missing)' + missing_text = " (missing)" rval.append(f'
  • {composite_name}{opt_text}{missing_text}
  • ') rval.append("
") defined_files = map(lambda x: x[0], defined_files) @@ -51,24 +52,24 @@ class AnvioComposite(Html): rval.append("

This composite dataset contains these undefined files:

    ") for rel_path in extra_files: rval.append(f'
  • {rel_path}
  • ') - rval.append('
') + rval.append("") if not (defined_files or extra_files): rval.append("

This composite dataset does not contain any files!

    ") - rval.append('') + rval.append("") return "\n".join(rval) def get_mime(self): """Returns the mime type of the datatype""" - return 'text/html' + return "text/html" def set_peek(self, dataset): """Set the peek and blurb text""" if not dataset.dataset.purged: - dataset.peek = 'Anvio database (multiple files)' - dataset.blurb = 'Anvio database (multiple files)' + dataset.peek = "Anvio database (multiple files)" + dataset.blurb = "Anvio database (multiple files)" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): """Create HTML content, used for displaying peek.""" @@ -80,9 +81,10 @@ class AnvioComposite(Html): class AnvioDB(AnvioComposite): """Class for AnvioDB database files.""" + _anvio_basename: Optional[str] = None MetadataElement(name="anvio_basename", default=_anvio_basename, desc="Basename", readonly=True) - file_ext = 'anvio_db' + file_ext = "anvio_db" def __init__(self, *args, **kwd): super().__init__(*args, **kwd) @@ -94,7 +96,9 @@ class AnvioDB(AnvioComposite): Set the anvio_basename based upon actual extra_files_path contents. """ super().set_meta(dataset, **kwd) - if dataset.metadata.anvio_basename is not None and os.path.exists(os.path.join(dataset.extra_files_path, dataset.metadata.anvio_basename)): + if dataset.metadata.anvio_basename is not None and os.path.exists( + os.path.join(dataset.extra_files_path, dataset.metadata.anvio_basename) + ): return found = False for basename in [dataset.metadata.anvio_basename, self._anvio_basename]: @@ -109,57 +113,64 @@ class AnvioDB(AnvioComposite): class AnvioStructureDB(AnvioDB): """Class for Anvio Structure DB database files.""" - _anvio_basename = 'STRUCTURE.db' + + _anvio_basename = "STRUCTURE.db" MetadataElement(name="anvio_basename", default=_anvio_basename, desc="Basename", readonly=True) - file_ext = 'anvio_structure_db' + file_ext = "anvio_structure_db" class AnvioGenomesDB(AnvioDB): """Class for Anvio Genomes DB database files.""" - _anvio_basename = '-GENOMES.db' + + _anvio_basename = "-GENOMES.db" MetadataElement(name="anvio_basename", default=_anvio_basename, desc="Basename", readonly=True) - file_ext = 'anvio_genomes_db' + file_ext = "anvio_genomes_db" class AnvioContigsDB(AnvioDB): """Class for Anvio Contigs DB database files.""" - _anvio_basename = 'CONTIGS.db' + + _anvio_basename = "CONTIGS.db" MetadataElement(name="anvio_basename", default=_anvio_basename, desc="Basename", readonly=True) - file_ext = 'anvio_contigs_db' + file_ext = "anvio_contigs_db" def __init__(self, *args, **kwd): super().__init__(*args, **kwd) - self.add_composite_file('CONTIGS.h5', is_binary=True, optional=True) + self.add_composite_file("CONTIGS.h5", is_binary=True, optional=True) class AnvioProfileDB(AnvioDB): """Class for Anvio Profile DB database files.""" - _anvio_basename = 'PROFILE.db' + + _anvio_basename = "PROFILE.db" MetadataElement(name="anvio_basename", default=_anvio_basename, desc="Basename", readonly=True) - file_ext = 'anvio_profile_db' + file_ext = "anvio_profile_db" def __init__(self, *args, **kwd): super().__init__(*args, **kwd) - self.add_composite_file('RUNINFO.cp', is_binary=True, optional=True) - self.add_composite_file('RUNINFO.mcp', is_binary=True, optional=True) - self.add_composite_file('AUXILIARY_DATA.db', is_binary=True, optional=True) - self.add_composite_file('RUNLOG.txt', is_binary=False, optional=True) + self.add_composite_file("RUNINFO.cp", is_binary=True, optional=True) + self.add_composite_file("RUNINFO.mcp", is_binary=True, optional=True) + self.add_composite_file("AUXILIARY_DATA.db", is_binary=True, optional=True) + self.add_composite_file("RUNLOG.txt", is_binary=False, optional=True) class AnvioPanDB(AnvioDB): """Class for Anvio Pan DB database files.""" - _anvio_basename = 'PAN.db' + + _anvio_basename = "PAN.db" MetadataElement(name="anvio_basename", default=_anvio_basename, desc="Basename", readonly=True) - file_ext = 'anvio_pan_db' + file_ext = "anvio_pan_db" class AnvioSamplesDB(AnvioDB): """Class for Anvio Samples DB database files.""" - _anvio_basename = 'SAMPLES.db' + + _anvio_basename = "SAMPLES.db" MetadataElement(name="anvio_basename", default=_anvio_basename, desc="Basename", readonly=True) - file_ext = 'anvio_samples_db' + file_ext = "anvio_samples_db" -if __name__ == '__main__': +if __name__ == "__main__": import doctest + doctest.testmod(sys.modules[__name__]) diff --git a/lib/galaxy/datatypes/assembly.py b/lib/galaxy/datatypes/assembly.py index 46b108942a2..c3855350ecf 100644 --- a/lib/galaxy/datatypes/assembly.py +++ b/lib/galaxy/datatypes/assembly.py @@ -9,8 +9,10 @@ import os import re import sys -from galaxy.datatypes import data -from galaxy.datatypes import sequence +from galaxy.datatypes import ( + data, + sequence, +) from galaxy.datatypes.metadata import MetadataElement from galaxy.datatypes.sniff import ( build_sniff_from_prefix, @@ -23,10 +25,11 @@ log = logging.getLogger(__name__) @build_sniff_from_prefix class Amos(data.Text): - """Class describing the AMOS assembly file """ + """Class describing the AMOS assembly file""" + edam_data = "data_0925" edam_format = "format_3582" - file_ext = 'afg' + file_ext = "afg" def sniff_prefix(self, file_prefix: FilePrefix): """ @@ -57,17 +60,18 @@ class Amos(data.Text): break # EOF line = line.strip() if line: # first non-empty line - if line.startswith('{'): - if re.match(r'{(RED|CTG|TLE)$', line): + if line.startswith("{"): + if re.match(r"{(RED|CTG|TLE)$", line): return True return False @build_sniff_from_prefix class Sequences(sequence.Fasta): - """Class describing the Sequences file generated by velveth """ + """Class describing the Sequences file generated by velveth""" + edam_data = "data_0925" - file_ext = 'sequences' + file_ext = "sequences" def sniff_prefix(self, file_prefix: FilePrefix): """ @@ -83,12 +87,12 @@ class Sequences(sequence.Fasta): for line in fh: line = line.strip() if line: # first non-empty line - if line.startswith('>'): - if not re.match(r'>[^\t]+\t\d+\t\d+$', line): + if line.startswith(">"): + if not re.match(r">[^\t]+\t\d+\t\d+$", line): return False # The next line.strip() must not be '', nor startwith '>' line = fh.readline().strip() - if line == '' or line.startswith('>'): + if line == "" or line.startswith(">"): return False return True else: @@ -98,9 +102,10 @@ class Sequences(sequence.Fasta): @build_sniff_from_prefix class Roadmaps(data.Text): - """Class describing the Sequences file generated by velveth """ + """Class describing the Sequences file generated by velveth""" + edam_format = "format_2561" - file_ext = 'roadmaps' + file_ext = "roadmaps" def sniff_prefix(self, file_prefix: FilePrefix): """ @@ -115,45 +120,72 @@ class Roadmaps(data.Text): for line in fh: line = line.strip() if line: # first non-empty line - if not re.match(r'\d+\t\d+\t\d+$', line): + if not re.match(r"\d+\t\d+\t\d+$", line): return False # The next line.strip() should be 'ROADMAP 1' line = fh.readline().strip() - return bool(re.match(r'ROADMAP \d+$', line)) + return bool(re.match(r"ROADMAP \d+$", line)) else: return False # we found a non-empty line, but it's not a fasta header return False class Velvet(Html): - MetadataElement(name="base_name", desc="base name for velveth dataset", default="velvet", readonly=True, set_in_upload=True) - MetadataElement(name="paired_end_reads", desc="has paired-end reads", default="False", readonly=False, set_in_upload=True) + MetadataElement( + name="base_name", desc="base name for velveth dataset", default="velvet", readonly=True, set_in_upload=True + ) + MetadataElement( + name="paired_end_reads", desc="has paired-end reads", default="False", readonly=False, set_in_upload=True + ) MetadataElement(name="long_reads", desc="has long reads", default="False", readonly=False, set_in_upload=True) - MetadataElement(name="short2_reads", desc="has 2nd short reads", default="False", readonly=False, set_in_upload=True) - composite_type = 'auto_primary_file' - file_ext = 'velvet' + MetadataElement( + name="short2_reads", desc="has 2nd short reads", default="False", readonly=False, set_in_upload=True + ) + composite_type = "auto_primary_file" + file_ext = "velvet" def __init__(self, **kwd): super().__init__(**kwd) - self.add_composite_file('Sequences', mimetype='text/html', description='Sequences', substitute_name_with_metadata=None, is_binary=False) - self.add_composite_file('Roadmaps', mimetype='text/html', description='Roadmaps', substitute_name_with_metadata=None, is_binary=False) - self.add_composite_file('Log', mimetype='text/html', description='Log', optional='True', substitute_name_with_metadata=None, is_binary=False) + self.add_composite_file( + "Sequences", + mimetype="text/html", + description="Sequences", + substitute_name_with_metadata=None, + is_binary=False, + ) + self.add_composite_file( + "Roadmaps", + mimetype="text/html", + description="Roadmaps", + substitute_name_with_metadata=None, + is_binary=False, + ) + self.add_composite_file( + "Log", + mimetype="text/html", + description="Log", + optional="True", + substitute_name_with_metadata=None, + is_binary=False, + ) def generate_primary_file(self, dataset=None): log.debug(f"Velvet log info JJ generate_primary_file {dataset}") - rval = ['Velvet Galaxy Composite Dataset

    '] - rval.append('

    This composite dataset is composed of the following files:

      ') + rval = ["Velvet Galaxy Composite Dataset

      "] + rval.append("

      This composite dataset is composed of the following files:

        ") for composite_name, composite_file in self.get_composite_files(dataset=dataset).items(): fn = composite_name log.debug(f"Velvet log info JJ generate_primary_file {fn} {composite_file}") - opt_text = '' + opt_text = "" if composite_file.optional: - opt_text = ' (optional)' - if composite_file.get('description'): - rval.append(f"
      • {fn} ({composite_file.get('description')}){opt_text}
      • ") + opt_text = " (optional)" + if composite_file.get("description"): + rval.append( + f"
      • {fn} ({composite_file.get('description')}){opt_text}
      • " + ) else: rval.append(f'
      • {fn}{opt_text}
      • ') - rval.append('
      ') + rval.append("
    ") return "\n".join(rval) def regenerate_primary_file(self, dataset): @@ -161,21 +193,21 @@ class Velvet(Html): cannot do this until we are setting metadata """ log.debug(f"Velvet log info {'JJ regenerate_primary_file'}") - gen_msg = '' + gen_msg = "" try: efp = dataset.extra_files_path - log_path = os.path.join(efp, 'Log') + log_path = os.path.join(efp, "Log") with open(log_path) as f: log_content = f.read(1000) - log_msg = re.sub(r'/\S*/', '', log_content) + log_msg = re.sub(r"/\S*/", "", log_content) log.debug(f"Velveth log info {log_msg}") - paired_end_reads = re.search(r'-(short|long)Paired', log_msg) is not None + paired_end_reads = re.search(r"-(short|long)Paired", log_msg) is not None dataset.metadata.paired_end_reads = paired_end_reads - long_reads = re.search(r'-long', log_msg) is not None + long_reads = re.search(r"-long", log_msg) is not None dataset.metadata.long_reads = long_reads - short2_reads = re.search(r'-short(Paired)?2', log_msg) is not None + short2_reads = re.search(r"-short(Paired)?2", log_msg) is not None dataset.metadata.short2_reads = short2_reads - dataset.info = re.sub(r'.*velveth \S+', 'hash_length', re.sub(r'\n', ' ', log_msg)) + dataset.info = re.sub(r".*velveth \S+", "hash_length", re.sub(r"\n", " ", log_msg)) if paired_end_reads: gen_msg = f"{gen_msg} Paired-End Reads" if long_reads: @@ -185,31 +217,34 @@ class Velvet(Html): except Exception: log.debug(f"Velveth could not read Log file in {efp}") log.debug(f"Velveth log info {gen_msg}") - rval = ['Velvet Galaxy Composite Dataset

    '] + rval = ["Velvet Galaxy Composite Dataset

    "] # rval.append('

    Generated:

    %s

    ' %(re.sub('\n','
    ',log_msg))) - rval.append(f'
    Generated:

    {gen_msg}

    ') - rval.append('
    Velveth dataset:

      ') + rval.append(f"
      Generated:

      {gen_msg}

      ") + rval.append("
      Velveth dataset:

        ") for composite_name, composite_file in self.get_composite_files(dataset=dataset).items(): fn = composite_name log.debug(f"Velvet log info JJ regenerate_primary_file {fn} {composite_file}") - if re.search('Log', fn) is None: - opt_text = '' + if re.search("Log", fn) is None: + opt_text = "" if composite_file.optional: - opt_text = ' (optional)' - if composite_file.get('description'): - rval.append(f"
      • {fn} ({composite_file.get('description')}){opt_text}
      • ") + opt_text = " (optional)" + if composite_file.get("description"): + rval.append( + f"
      • {fn} ({composite_file.get('description')}){opt_text}
      • " + ) else: rval.append(f'
      • {fn}{opt_text}
      • ') - rval.append('
      ') - with open(dataset.file_name, 'w') as f: + rval.append("
    ") + with open(dataset.file_name, "w") as f: f.write("\n".join(rval)) - f.write('\n') + f.write("\n") def set_meta(self, dataset, **kwd): Html.set_meta(self, dataset, **kwd) self.regenerate_primary_file(dataset) -if __name__ == '__main__': +if __name__ == "__main__": import doctest + doctest.testmod(sys.modules[__name__]) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 908ad8999f6..0c7b410128a 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -20,7 +20,10 @@ import h5py import numpy as np import pysam import pysam.bcftools -from bx.seq.twobit import TWOBIT_MAGIC_NUMBER, TWOBIT_MAGIC_NUMBER_SWAP +from bx.seq.twobit import ( + TWOBIT_MAGIC_NUMBER, + TWOBIT_MAGIC_NUMBER_SWAP, +) from galaxy import util from galaxy.datatypes import metadata @@ -36,11 +39,24 @@ from galaxy.datatypes.metadata import ( MetadataElement, MetadataParameter, ) -from galaxy.datatypes.sniff import build_sniff_from_prefix, FilePrefix +from galaxy.datatypes.sniff import ( + build_sniff_from_prefix, + FilePrefix, +) from galaxy.datatypes.text import Html -from galaxy.util import compression_utils, nice_size, sqlite -from galaxy.util.checkers import is_bz2, is_gzip -from . import data, dataproviders +from galaxy.util import ( + compression_utils, + nice_size, + sqlite, +) +from galaxy.util.checkers import ( + is_bz2, + is_gzip, +) +from . import ( + data, + dataproviders, +) log = logging.getLogger(__name__) # pysam 0.16.0.1 emits logs containing the word 'Error', this can confuse the stdout/stderr checkers. @@ -52,6 +68,7 @@ pysam.set_verbosity(0) class Binary(data.Data): """Binary data""" + edam_format = "format_2333" file_ext = "binary" @@ -66,19 +83,20 @@ class Binary(data.Data): def set_peek(self, dataset, **kwd): """Set the peek and blurb text""" if not dataset.dataset.purged: - dataset.peek = 'binary data' + dataset.peek = "binary data" dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def get_mime(self): """Returns the mime type of the datatype""" - return 'application/octet-stream' + return "application/octet-stream" class Ab1(Binary): """Class describing an ab1 binary sequence file""" + file_ext = "ab1" edam_format = "format_3000" edam_data = "data_0924" @@ -88,8 +106,8 @@ class Ab1(Binary): dataset.peek = "Binary ab1 sequence file" dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -100,14 +118,15 @@ class Ab1(Binary): class Idat(Binary): """Binary data in idat format""" + file_ext = "idat" edam_format = "format_2058" edam_data = "data_2603" def sniff(self, filename): try: - header = open(filename, 'rb').read(4) - if header == b'IDAT': + header = open(filename, "rb").read(4) + if header == b"IDAT": return True return False except Exception: @@ -115,15 +134,16 @@ class Idat(Binary): class Cel(Binary): - """ Cel File format described at: - http://media.affymetrix.com/support/developer/powertools/changelog/gcos-agcc/cel.html + """Cel File format described at: + http://media.affymetrix.com/support/developer/powertools/changelog/gcos-agcc/cel.html """ file_ext = "cel" edam_format = "format_1638" edam_data = "data_3110" - MetadataElement(name="version", default="3", desc="Version", readonly=True, visible=True, - optional=True, no_value="3") + MetadataElement( + name="version", default="3", desc="Version", readonly=True, visible=True, optional=True, no_value="3" + ) def sniff(self, filename): """ @@ -142,7 +162,7 @@ class Cel(Binary): >>> Cel().sniff(fname) False """ - with open(filename, 'rb') as handle: + with open(filename, "rb") as handle: header_bytes = handle.read(8) found_cel_4 = False found_cel_3 = False @@ -151,7 +171,7 @@ class Cel(Binary): found_cel_4 = True elif struct.unpack(">bb", header_bytes[:2]) == (59, 1): found_cel_agcc = True - elif header_bytes.decode("utf8", errors="ignore").startswith('[CEL]'): + elif header_bytes.decode("utf8", errors="ignore").startswith("[CEL]"): found_cel_3 = True return found_cel_3 or found_cel_4 or found_cel_agcc @@ -159,13 +179,13 @@ class Cel(Binary): """ Set metadata for Cel file. """ - with open(dataset.file_name, 'rb') as handle: + with open(dataset.file_name, "rb") as handle: header_bytes = handle.read(8) if struct.unpack("bb", header_bytes[:2]) == (59, 1): dataset.metadata.version = "agcc" - elif header_bytes.decode("utf8", errors="ignore").startswith('[CEL]'): + elif header_bytes.decode("utf8", errors="ignore").startswith("[CEL]"): dataset.metadata.version = "3" def set_peek(self, dataset): @@ -173,27 +193,29 @@ class Cel(Binary): dataset.blurb = f"Cel version: {dataset.metadata.version}" dataset.peek = get_file_peek(dataset.file_name) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" class MashSketch(Binary): """ - Mash Sketch file. - Sketches are used by the MinHash algorithm to allow fast distance estimations - with low storage and memory requirements. To make a sketch, each k-mer in a sequence - is hashed, which creates a pseudo-random identifier. By sorting these identifiers (hashes), - a small subset from the top of the sorted list can represent the entire sequence (these are min-hashes). - The more similar another sequence is, the more min-hashes it is likely to share. + Mash Sketch file. + Sketches are used by the MinHash algorithm to allow fast distance estimations + with low storage and memory requirements. To make a sketch, each k-mer in a sequence + is hashed, which creates a pseudo-random identifier. By sorting these identifiers (hashes), + a small subset from the top of the sorted list can represent the entire sequence (these are min-hashes). + The more similar another sequence is, the more min-hashes it is likely to share. """ + file_ext = "msh" class CompressedArchive(Binary): """ - Class describing an compressed binary file - This class can be sublass'ed to implement archive filetypes that will not be unpacked by upload.py. + Class describing an compressed binary file + This class can be sublass'ed to implement archive filetypes that will not be unpacked by upload.py. """ + file_ext = "compressed_archive" compressed = True @@ -202,8 +224,8 @@ class CompressedArchive(Binary): dataset.peek = "Compressed binary file" dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -214,6 +236,7 @@ class CompressedArchive(Binary): class Meryldb(CompressedArchive): """MerylDB is a tar.gz archive, with 128 files. 64 data files and 64 index files.""" + file_ext = "meryldb" def sniff(self, filename): @@ -229,14 +252,14 @@ class Meryldb(CompressedArchive): """ try: if filename and tarfile.is_tarfile(filename): - with tarfile.open(filename, 'r') as temptar: + with tarfile.open(filename, "r") as temptar: _tar_content = temptar.getnames() # 64 data files ad 64 indices + 2 folders if len(_tar_content) == 130: - if len([_ for _ in _tar_content if _.endswith('.merylIndex')]) == 64: + if len([_ for _ in _tar_content if _.endswith(".merylIndex")]) == 64: return True except Exception as e: - log.warning('%s, sniff Exception: %s', self, e) + log.warning("%s, sniff Exception: %s", self, e) return False @@ -257,8 +280,8 @@ class Bref3(Binary): dataset.peek = "Binary bref3 file" dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -273,13 +296,15 @@ class DynamicCompressedArchive(CompressedArchive): uncompressed_datatype_instance: Data def matches_any(self, target_datatypes) -> bool: - """Treat two aspects of compressed datatypes separately. - """ + """Treat two aspects of compressed datatypes separately.""" compressed_target_datatypes = [] uncompressed_target_datatypes = [] for target_datatype in target_datatypes: - if hasattr(target_datatype, "uncompressed_datatype_instance") and target_datatype.compressed_format == self.compressed_format: + if ( + hasattr(target_datatype, "uncompressed_datatype_instance") + and target_datatype.compressed_format == self.compressed_format + ): uncompressed_target_datatypes.append(target_datatype.uncompressed_datatype_instance) else: compressed_target_datatypes.append(target_datatype) @@ -304,9 +329,10 @@ class Bz2DynamicCompressedArchive(DynamicCompressedArchive): class CompressedZipArchive(CompressedArchive): """ - Class describing an compressed binary file - This class can be sublass'ed to implement archive filetypes that will not be unpacked by upload.py. + Class describing an compressed binary file + This class can be sublass'ed to implement archive filetypes that will not be unpacked by upload.py. """ + file_ext = "zip" def set_peek(self, dataset): @@ -314,8 +340,8 @@ class CompressedZipArchive(CompressedArchive): dataset.peek = "Compressed zip file" dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -328,7 +354,7 @@ class CompressedZipArchive(CompressedArchive): zf_files = zf.infolist() count = 0 for f in zf_files: - if f.file_size > 0 and not f.filename.startswith('__MACOSX/') and not f.filename.endswith('.DS_Store'): + if f.file_size > 0 and not f.filename.startswith("__MACOSX/") and not f.filename.endswith(".DS_Store"): count += 1 if count > 1: return True @@ -336,6 +362,7 @@ class CompressedZipArchive(CompressedArchive): class GenericAsn1Binary(Binary): """Class for generic ASN.1 binary format""" + file_ext = "asn1-binary" edam_format = "format_1966" edam_data = "data_0849" @@ -348,14 +375,16 @@ class _BamOrSam: def set_meta(self, dataset, overwrite=True, **kwd): try: - bam_file = pysam.AlignmentFile(dataset.file_name, mode='rb') + bam_file = pysam.AlignmentFile(dataset.file_name, mode="rb") # TODO: Reference names, lengths, read_groups and headers can become very large, truncate when necessary dataset.metadata.reference_names = list(bam_file.references) dataset.metadata.reference_lengths = list(bam_file.lengths) dataset.metadata.bam_header = dict(bam_file.header.items()) - dataset.metadata.read_groups = [read_group['ID'] for read_group in dataset.metadata.bam_header.get('RG', []) if 'ID' in read_group] - dataset.metadata.sort_order = dataset.metadata.bam_header.get('HD', {}).get('SO', None) - dataset.metadata.bam_version = dataset.metadata.bam_header.get('HD', {}).get('VN', None) + dataset.metadata.read_groups = [ + read_group["ID"] for read_group in dataset.metadata.bam_header.get("RG", []) if "ID" in read_group + ] + dataset.metadata.sort_order = dataset.metadata.bam_header.get("HD", {}).get("SO", None) + dataset.metadata.bam_version = dataset.metadata.bam_header.get("HD", {}).get("VN", None) except Exception: # Per Dan, don't log here because doing so will cause datasets that # fail metadata to end in the error state @@ -364,21 +393,90 @@ class _BamOrSam: class BamNative(CompressedArchive, _BamOrSam): """Class describing a BAM binary file that is not necessarily sorted""" + edam_format = "format_2572" edam_data = "data_0863" file_ext = "unsorted.bam" sort_flag: Optional[str] = None MetadataElement(name="columns", default=12, desc="Number of columns", readonly=True, visible=False, no_value=0) - MetadataElement(name="column_types", default=['str', 'int', 'str', 'int', 'int', 'str', 'str', 'int', 'int', 'str', 'str', 'str'], desc="Column types", param=metadata.ColumnTypesParameter, readonly=True, visible=False, no_value=[]) - MetadataElement(name="column_names", default=['QNAME', 'FLAG', 'RNAME', 'POS', 'MAPQ', 'CIGAR', 'MRNM', 'MPOS', 'ISIZE', 'SEQ', 'QUAL', 'OPT'], desc="Column names", readonly=True, visible=False, optional=True, no_value=[]) + MetadataElement( + name="column_types", + default=["str", "int", "str", "int", "int", "str", "str", "int", "int", "str", "str", "str"], + desc="Column types", + param=metadata.ColumnTypesParameter, + readonly=True, + visible=False, + no_value=[], + ) + MetadataElement( + name="column_names", + default=["QNAME", "FLAG", "RNAME", "POS", "MAPQ", "CIGAR", "MRNM", "MPOS", "ISIZE", "SEQ", "QUAL", "OPT"], + desc="Column names", + readonly=True, + visible=False, + optional=True, + no_value=[], + ) - MetadataElement(name="bam_version", default=None, desc="BAM Version", param=MetadataParameter, readonly=True, visible=False, optional=True) - MetadataElement(name="sort_order", default=None, desc="Sort Order", param=MetadataParameter, readonly=True, visible=False, optional=True) - MetadataElement(name="read_groups", default=[], desc="Read Groups", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value=[]) - MetadataElement(name="reference_names", default=[], desc="Chromosome Names", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value=[]) - MetadataElement(name="reference_lengths", default=[], desc="Chromosome Lengths", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value=[]) - MetadataElement(name="bam_header", default={}, desc="Dictionary of BAM Headers", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value={}) + MetadataElement( + name="bam_version", + default=None, + desc="BAM Version", + param=MetadataParameter, + readonly=True, + visible=False, + optional=True, + ) + MetadataElement( + name="sort_order", + default=None, + desc="Sort Order", + param=MetadataParameter, + readonly=True, + visible=False, + optional=True, + ) + MetadataElement( + name="read_groups", + default=[], + desc="Read Groups", + param=MetadataParameter, + readonly=True, + visible=False, + optional=True, + no_value=[], + ) + MetadataElement( + name="reference_names", + default=[], + desc="Chromosome Names", + param=MetadataParameter, + readonly=True, + visible=False, + optional=True, + no_value=[], + ) + MetadataElement( + name="reference_lengths", + default=[], + desc="Chromosome Lengths", + param=MetadataParameter, + readonly=True, + visible=False, + optional=True, + no_value=[], + ) + MetadataElement( + name="bam_header", + default={}, + desc="Dictionary of BAM Headers", + param=MetadataParameter, + readonly=True, + visible=False, + optional=True, + no_value={}, + ) def set_meta(self, dataset, overwrite=True, **kwd): _BamOrSam().set_meta(dataset) @@ -391,7 +489,7 @@ class BamNative(CompressedArchive, _BamOrSam): :param split_files: List of bam file paths to merge :param output_file: Write merged bam file to this location """ - pysam.merge('-O', 'BAM', output_file, *split_files) + pysam.merge("-O", "BAM", output_file, *split_files) def init_meta(self, dataset, copy_from=None): Binary.init_meta(self, dataset, copy_from=copy_from) @@ -405,7 +503,7 @@ class BamNative(CompressedArchive, _BamOrSam): # The first 4 bytes of any bam file is 'BAM\1', and the file is binary. try: header = gzip.open(filename).read(4) - if header == b'BAM\1': + if header == b"BAM\1": return True return False except Exception: @@ -416,8 +514,8 @@ class BamNative(CompressedArchive, _BamOrSam): dataset.peek = "Binary bam alignments file" dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -447,13 +545,15 @@ class BamNative(CompressedArchive, _BamOrSam): # Don't re-sort if already sorted return tmp_dir = tempfile.mkdtemp() - tmp_sorted_dataset_file_name_prefix = os.path.join(tmp_dir, 'sorted') + tmp_sorted_dataset_file_name_prefix = os.path.join(tmp_dir, "sorted") sorted_file_name = f"{tmp_sorted_dataset_file_name_prefix}.bam" - slots = os.environ.get('GALAXY_SLOTS', 1) + slots = os.environ.get("GALAXY_SLOTS", 1) sort_args = [] if self.sort_flag: sort_args = [self.sort_flag] - sort_args.extend([f"-@{slots}", file_name, '-T', tmp_sorted_dataset_file_name_prefix, '-O', 'BAM', '-o', sorted_file_name]) + sort_args.extend( + [f"-@{slots}", file_name, "-T", tmp_sorted_dataset_file_name_prefix, "-O", "BAM", "-o", sorted_file_name] + ) try: pysam.sort(*sort_args) except Exception: @@ -472,8 +572,8 @@ class BamNative(CompressedArchive, _BamOrSam): ck_data = "" header_line_count = 0 if offset == 0: - ck_data = bamfile.text.replace('\t', ' ') - header_line_count = bamfile.text.count('\n') + ck_data = bamfile.text.replace("\t", " ") + header_line_count = bamfile.text.count("\n") else: bamfile.seek(offset) for line_number, alignment in enumerate(bamfile): @@ -486,7 +586,7 @@ class BamNative(CompressedArchive, _BamOrSam): bamline = alignment.tostring(bamfile) # Galaxy display each tag as separate column because 'tostring()' funcition put tabs in between each tag of tags column. # Below code will remove spaces between each tag. - bamline_modified = ('\t').join(bamline.split()[:11] + [(' ').join(bamline.split()[11:])]) + bamline_modified = ("\t").join(bamline.split()[:11] + [(" ").join(bamline.split()[11:])]) ck_data = f"{ck_data}\n{bamline_modified}" else: # Nothing to enumerate; we've either offset to the end @@ -497,10 +597,9 @@ class BamNative(CompressedArchive, _BamOrSam): offset = -1 ck_data = f"Could not display BAM file, error was:\n{e}" else: - ck_data = '' + ck_data = "" offset = -1 - return dumps({'ck_data': util.unicodify(ck_data), - 'offset': offset}) + return dumps({"ck_data": util.unicodify(ck_data), "offset": offset}) def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, offset=None, ck_size=None, **kwd): headers = kwd.get("headers", {}) @@ -519,42 +618,66 @@ class BamNative(CompressedArchive, _BamOrSam): column_number = dataset.metadata.columns if column_number is None: column_number = 1 - return trans.fill_template("/dataset/tabular_chunked.mako", - dataset=dataset, - chunk=self.get_chunk(trans, dataset, 0), - column_number=column_number, - column_names=column_names, - column_types=column_types), headers + return ( + trans.fill_template( + "/dataset/tabular_chunked.mako", + dataset=dataset, + chunk=self.get_chunk(trans, dataset, 0), + column_number=column_number, + column_names=column_names, + column_types=column_types, + ), + headers, + ) def validate(self, dataset, **kwd): if not BamNative.is_bam(dataset.file_name): return DatatypeValidation.invalid("This dataset does not appear to a BAM file.") elif self.dataset_content_needs_grooming(dataset.file_name): - return DatatypeValidation.invalid("This BAM file does not appear to have the correct sorting for declared datatype.") + return DatatypeValidation.invalid( + "This BAM file does not appear to have the correct sorting for declared datatype." + ) return DatatypeValidation.validated() @dataproviders.decorators.has_dataproviders class Bam(BamNative): """Class describing a BAM binary file""" + edam_format = "format_2572" edam_data = "data_0863" file_ext = "bam" track_type = "ReadTrack" data_sources = {"data": "bai", "index": "bigwig"} - MetadataElement(name="bam_index", desc="BAM Index File", param=metadata.FileParameter, file_ext="bai", readonly=True, visible=False, optional=True) - MetadataElement(name="bam_csi_index", desc="BAM CSI Index File", param=metadata.FileParameter, file_ext="bam.csi", readonly=True, visible=False, optional=True) + MetadataElement( + name="bam_index", + desc="BAM Index File", + param=metadata.FileParameter, + file_ext="bai", + readonly=True, + visible=False, + optional=True, + ) + MetadataElement( + name="bam_csi_index", + desc="BAM CSI Index File", + param=metadata.FileParameter, + file_ext="bam.csi", + readonly=True, + visible=False, + optional=True, + ) def get_index_flag(self, file_name): """ Return pysam flag for bai index (default) or csi index (contig size > (2**29 - 1) ) """ - index_flag = '-b' # bai index + index_flag = "-b" # bai index try: with pysam.AlignmentFile(file_name) as alignment_file: - if max(alignment_file.header.lengths) > (2 ** 29) - 1: - index_flag = '-c' # csi index + if max(alignment_file.header.lengths) > (2**29) - 1: + index_flag = "-c" # csi index except Exception: # File may not have a header, that's OK pass @@ -572,12 +695,20 @@ class Bam(BamNative): # If pysam fails to index a file it will write to stderr, # and this causes the set_meta script to fail. So instead # we start another process and discard stderr. - if index_flag == '-b': + if index_flag == "-b": # IOError: No such file or directory: '-b' if index_flag is set to -b (pysam 0.15.4) - cmd = ['python', '-c', f"import pysam; pysam.set_verbosity(0); pysam.index('{file_name}', '{index_name}')"] + cmd = [ + "python", + "-c", + f"import pysam; pysam.set_verbosity(0); pysam.index('{file_name}', '{index_name}')", + ] else: - cmd = ['python', '-c', f"import pysam; pysam.set_verbosity(0); pysam.index('{index_flag}', '{file_name}', '{index_name}')"] - with open(os.devnull, 'w') as devnull: + cmd = [ + "python", + "-c", + f"import pysam; pysam.set_verbosity(0); pysam.index('{index_flag}', '{file_name}', '{index_name}')", + ] + with open(os.devnull, "w") as devnull: subprocess.check_call(cmd, stderr=devnull, shell=False) needs_sorting = False except subprocess.CalledProcessError: @@ -592,15 +723,15 @@ class Bam(BamNative): # These metadata values are not accessible by users, always overwrite super().set_meta(dataset=dataset, overwrite=overwrite, **kwd) index_flag = self.get_index_flag(dataset.file_name) - if index_flag == '-b': - spec_key = 'bam_index' + if index_flag == "-b": + spec_key = "bam_index" index_file = dataset.metadata.bam_index else: - spec_key = 'bam_csi_index' + spec_key = "bam_csi_index" index_file = dataset.metadata.bam_csi_index if not index_file: index_file = dataset.metadata.spec[spec_key].param.new_file(dataset=dataset) - if index_flag == '-b': + if index_flag == "-b": # IOError: No such file or directory: '-b' if index_flag is set to -b (pysam 0.15.4) pysam.index(dataset.file_name, index_file.file_name) else: @@ -616,28 +747,28 @@ class Bam(BamNative): # bam does not use '#' to indicate comments/headers - we need to strip out those headers from the std. providers # TODO:?? seems like there should be an easier way to do/inherit this - metadata.comment_char? # TODO: incorporate samtools options to control output: regions first, then flags, etc. - @dataproviders.decorators.dataprovider_factory('line', dataproviders.line.FilteredLineDataProvider.settings) + @dataproviders.decorators.dataprovider_factory("line", dataproviders.line.FilteredLineDataProvider.settings) def line_dataprovider(self, dataset, **settings): samtools_source = dataproviders.dataset.SamtoolsDataProvider(dataset) - settings['comment_char'] = '@' + settings["comment_char"] = "@" return dataproviders.line.FilteredLineDataProvider(samtools_source, **settings) - @dataproviders.decorators.dataprovider_factory('regex-line', dataproviders.line.RegexLineDataProvider.settings) + @dataproviders.decorators.dataprovider_factory("regex-line", dataproviders.line.RegexLineDataProvider.settings) def regex_line_dataprovider(self, dataset, **settings): samtools_source = dataproviders.dataset.SamtoolsDataProvider(dataset) - settings['comment_char'] = '@' + settings["comment_char"] = "@" return dataproviders.line.RegexLineDataProvider(samtools_source, **settings) - @dataproviders.decorators.dataprovider_factory('column', dataproviders.column.ColumnarDataProvider.settings) + @dataproviders.decorators.dataprovider_factory("column", dataproviders.column.ColumnarDataProvider.settings) def column_dataprovider(self, dataset, **settings): samtools_source = dataproviders.dataset.SamtoolsDataProvider(dataset) - settings['comment_char'] = '@' + settings["comment_char"] = "@" return dataproviders.column.ColumnarDataProvider(samtools_source, **settings) - @dataproviders.decorators.dataprovider_factory('dict', dataproviders.column.DictDataProvider.settings) + @dataproviders.decorators.dataprovider_factory("dict", dataproviders.column.DictDataProvider.settings) def dict_dataprovider(self, dataset, **settings): samtools_source = dataproviders.dataset.SamtoolsDataProvider(dataset) - settings['comment_char'] = '@' + settings["comment_char"] = "@" return dataproviders.column.DictDataProvider(samtools_source, **settings) # these can't be used directly - may need BamColumn, BamDict (Bam metadata -> column/dict) @@ -652,20 +783,20 @@ class Bam(BamNative): # settings['comment_char'] = '@' # return super().dataset_dict_dataprovider(dataset, **settings) - @dataproviders.decorators.dataprovider_factory('header', dataproviders.line.RegexLineDataProvider.settings) + @dataproviders.decorators.dataprovider_factory("header", dataproviders.line.RegexLineDataProvider.settings) def header_dataprovider(self, dataset, **settings): # in this case we can use an option of samtools view to provide just what we need (w/o regex) - samtools_source = dataproviders.dataset.SamtoolsDataProvider(dataset, '-H') + samtools_source = dataproviders.dataset.SamtoolsDataProvider(dataset, "-H") return dataproviders.line.RegexLineDataProvider(samtools_source, **settings) - @dataproviders.decorators.dataprovider_factory('id-seq-qual', dataproviders.column.DictDataProvider.settings) + @dataproviders.decorators.dataprovider_factory("id-seq-qual", dataproviders.column.DictDataProvider.settings) def id_seq_qual_dataprovider(self, dataset, **settings): - settings['indeces'] = [0, 9, 10] - settings['column_types'] = ['str', 'str', 'str'] - settings['column_names'] = ['id', 'seq', 'qual'] + settings["indeces"] = [0, 9, 10] + settings["column_types"] = ["str", "str", "str"] + settings["column_names"] = ["id", "seq", "qual"] return self.dict_dataprovider(dataset, **settings) - @dataproviders.decorators.dataprovider_factory('genomic-region', dataproviders.column.ColumnarDataProvider.settings) + @dataproviders.decorators.dataprovider_factory("genomic-region", dataproviders.column.ColumnarDataProvider.settings) def genomic_region_dataprovider(self, dataset, **settings): # GenomicRegionDataProvider currently requires a dataset as source - may not be necc. # TODO:?? consider (at least) the possible use of a kwarg: metadata_source (def. to source.dataset), @@ -675,18 +806,20 @@ class Bam(BamNative): # 2, 3, 3, **settings) # instead, set manually and use in-class column gen - settings['indeces'] = [2, 3, 3] - settings['column_types'] = ['str', 'int', 'int'] + settings["indeces"] = [2, 3, 3] + settings["column_types"] = ["str", "int", "int"] return self.column_dataprovider(dataset, **settings) - @dataproviders.decorators.dataprovider_factory('genomic-region-dict', dataproviders.column.DictDataProvider.settings) + @dataproviders.decorators.dataprovider_factory( + "genomic-region-dict", dataproviders.column.DictDataProvider.settings + ) def genomic_region_dict_dataprovider(self, dataset, **settings): - settings['indeces'] = [2, 3, 3] - settings['column_types'] = ['str', 'int', 'int'] - settings['column_names'] = ['chrom', 'start', 'end'] + settings["indeces"] = [2, 3, 3] + settings["column_types"] = ["str", "int", "int"] + settings["column_names"] = ["chrom", "start", "end"] return self.dict_dataprovider(dataset, **settings) - @dataproviders.decorators.dataprovider_factory('samtools') + @dataproviders.decorators.dataprovider_factory("samtools") def samtools_dataprovider(self, dataset, **settings): """Generic samtools interface - all options available through settings.""" dataset_source = dataproviders.dataset.DatasetDataProvider(dataset) @@ -695,6 +828,7 @@ class Bam(BamNative): class ProBam(Bam): """Class describing a BAM binary file - extended for proteomics data""" + edam_format = "format_3826" edam_data = "data_0863" file_ext = "probam" @@ -707,8 +841,9 @@ class BamInputSorted(BamNative): or ordered by their queryname. This notaby keeps alignments produced by paired end sequencing adjacent. """ - sort_flag = '-n' - file_ext = 'qname_input_sorted.bam' + + sort_flag = "-n" + file_ext = "qname_input_sorted.bam" def sniff(self, file_name): # We never want to sniff to this datatype @@ -722,13 +857,13 @@ class BamInputSorted(BamNative): # is to actually index them. with pysam.AlignmentFile(filename=file_name) as f: # The only sure thing we know here is that the sort order can't be coordinate - return f.header.get('HD', {}).get('SO') == 'coordinate' + return f.header.get("HD", {}).get("SO") == "coordinate" class BamQuerynameSorted(BamInputSorted): """A class for queryname sorted BAM files.""" - sort_flag = '-n' + sort_flag = "-n" file_ext = "qname_sorted.bam" def sniff(self, file_name): @@ -741,7 +876,7 @@ class BamQuerynameSorted(BamInputSorted): # The best way to ensure that BAM files are coordinate-sorted and indexable # is to actually index them. with pysam.AlignmentFile(filename=file_name) as f: - return f.header.get('HD', {}).get('SO') != 'queryname' + return f.header.get("HD", {}).get("SO") != "queryname" class CRAM(Binary): @@ -749,8 +884,24 @@ class CRAM(Binary): edam_format = "format_3462" edam_data = "data_0863" - MetadataElement(name="cram_version", default=None, desc="CRAM Version", param=MetadataParameter, readonly=True, visible=False, optional=False) - MetadataElement(name="cram_index", desc="CRAM Index File", param=metadata.FileParameter, file_ext="crai", readonly=True, visible=False, optional=True) + MetadataElement( + name="cram_version", + default=None, + desc="CRAM Version", + param=MetadataParameter, + readonly=True, + visible=False, + optional=False, + ) + MetadataElement( + name="cram_index", + desc="CRAM Index File", + param=metadata.FileParameter, + file_ext="crai", + readonly=True, + visible=False, + optional=True, + ) def set_meta(self, dataset, overwrite=True, **kwd): major_version, minor_version = self.get_cram_version(dataset.file_name) @@ -758,7 +909,7 @@ class CRAM(Binary): dataset.metadata.cram_version = f"{str(major_version)}.{str(minor_version)}" if not dataset.metadata.cram_index: - index_file = dataset.metadata.spec['cram_index'].param.new_file(dataset=dataset) + index_file = dataset.metadata.spec["cram_index"].param.new_file(dataset=dataset) if self.set_index_file(dataset, index_file): dataset.metadata.cram_index = index_file @@ -768,7 +919,7 @@ class CRAM(Binary): header = bytearray(fh.read(6)) return header[4], header[5] except Exception as exc: - log.warning('%s, get_cram_version Exception: %s', self, exc) + log.warning("%s, get_cram_version Exception: %s", self, exc) return -1, -1 def set_index_file(self, dataset, index_file): @@ -776,20 +927,20 @@ class CRAM(Binary): pysam.index(dataset.file_name, index_file.file_name) return True except Exception as exc: - log.warning('%s, set_index_file Exception: %s', self, exc) + log.warning("%s, set_index_file Exception: %s", self, exc) return False def set_peek(self, dataset): if not dataset.dataset.purged: - dataset.peek = 'CRAM binary alignment file' - dataset.blurb = 'binary data' + dataset.peek = "CRAM binary alignment file" + dataset.blurb = "binary data" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def sniff(self, filename): try: - header = open(filename, 'rb').read(4) + header = open(filename, "rb").read(4) if header == b"CRAM": return True return False @@ -807,37 +958,48 @@ class Bcf(BaseBcf): Class describing a (BGZF-compressed) BCF file """ + file_ext = "bcf" - MetadataElement(name="bcf_index", desc="BCF Index File", param=metadata.FileParameter, file_ext="csi", readonly=True, visible=False, optional=True) + MetadataElement( + name="bcf_index", + desc="BCF Index File", + param=metadata.FileParameter, + file_ext="csi", + readonly=True, + visible=False, + optional=True, + ) def sniff(self, filename): # BCF is compressed in the BGZF format, and must not be uncompressed in Galaxy. try: header = gzip.open(filename).read(3) # The first 3 bytes of any BCF file are 'BCF', and the file is binary. - if header == b'BCF': + if header == b"BCF": return True return False except Exception: return False def set_meta(self, dataset, overwrite=True, **kwd): - """ Creates the index for the BCF file. """ + """Creates the index for the BCF file.""" # These metadata values are not accessible by users, always overwrite index_file = dataset.metadata.bcf_index if not index_file: - index_file = dataset.metadata.spec['bcf_index'].param.new_file(dataset=dataset) + index_file = dataset.metadata.spec["bcf_index"].param.new_file(dataset=dataset) # Create the bcf index - dataset_symlink = os.path.join(os.path.dirname(index_file.file_name), - '__dataset_%d_%s' % (dataset.id, os.path.basename(index_file.file_name))) + dataset_symlink = os.path.join( + os.path.dirname(index_file.file_name), + "__dataset_%d_%s" % (dataset.id, os.path.basename(index_file.file_name)), + ) os.symlink(dataset.file_name, dataset_symlink) try: - cmd = ['python', '-c', f"import pysam.bcftools; pysam.bcftools.index('{dataset_symlink}')"] + cmd = ["python", "-c", f"import pysam.bcftools; pysam.bcftools.index('{dataset_symlink}')"] subprocess.check_call(cmd) shutil.move(f"{dataset_symlink}.csi", index_file.file_name) except Exception as e: - raise Exception(f'Error setting BCF metadata: {util.unicodify(e)}') + raise Exception(f"Error setting BCF metadata: {util.unicodify(e)}") finally: # Remove temp file and symlink os.remove(dataset_symlink) @@ -856,13 +1018,14 @@ class BcfUncompressed(BaseBcf): >>> BcfUncompressed().sniff(fname) False """ + file_ext = "bcf_uncompressed" def sniff(self, filename): try: - header = open(filename, mode='rb').read(3) + header = open(filename, mode="rb").read(3) # The first 3 bytes of any BCF file are 'BCF', and the file is binary. - if header == b'BCF': + if header == b"BCF": return True return False except Exception: @@ -881,6 +1044,7 @@ class H5(Binary): >>> H5().sniff(fname) False """ + file_ext = "h5" edam_format = "format_3590" @@ -891,7 +1055,7 @@ class H5(Binary): def sniff(self, filename): # The first 8 bytes of any hdf5 file are 0x894844460d0a1a0a try: - header = open(filename, 'rb').read(8) + header = open(filename, "rb").read(8) if header == self._magic: return True return False @@ -903,8 +1067,8 @@ class H5(Binary): dataset.peek = "Binary HDF5 file" dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -925,6 +1089,7 @@ class Loom(H5): >>> Loom().sniff(fname) False """ + file_ext = "loom" edam_format = "format_3590" @@ -932,28 +1097,71 @@ class Loom(H5): MetadataElement(name="description", default="", desc="description", readonly=True, visible=True, no_value="") MetadataElement(name="url", default="", desc="url", readonly=True, visible=True, no_value="") MetadataElement(name="doi", default="", desc="doi", readonly=True, visible=True, no_value="") - MetadataElement(name="loom_spec_version", default="", desc="loom_spec_version", readonly=True, visible=True, no_value="") + MetadataElement( + name="loom_spec_version", default="", desc="loom_spec_version", readonly=True, visible=True, no_value="" + ) MetadataElement(name="creation_date", default=None, desc="creation_date", readonly=True, visible=True) - MetadataElement(name="shape", default=(), desc="shape", param=metadata.ListParameter, readonly=True, visible=True, no_value=()) + MetadataElement( + name="shape", default=(), desc="shape", param=metadata.ListParameter, readonly=True, visible=True, no_value=() + ) MetadataElement(name="layers_count", default=0, desc="layers_count", readonly=True, visible=True, no_value=0) - MetadataElement(name="layers_names", desc="layers_names", default=[], param=metadata.SelectParameter, multiple=True, readonly=True) + MetadataElement( + name="layers_names", + desc="layers_names", + default=[], + param=metadata.SelectParameter, + multiple=True, + readonly=True, + ) MetadataElement(name="row_attrs_count", default=0, desc="row_attrs_count", readonly=True, visible=True, no_value=0) - MetadataElement(name="row_attrs_names", desc="row_attrs_names", default=[], param=metadata.SelectParameter, multiple=True, readonly=True) + MetadataElement( + name="row_attrs_names", + desc="row_attrs_names", + default=[], + param=metadata.SelectParameter, + multiple=True, + readonly=True, + ) MetadataElement(name="col_attrs_count", default=0, desc="col_attrs_count", readonly=True, visible=True, no_value=0) - MetadataElement(name="col_attrs_names", desc="col_attrs_names", default=[], param=metadata.SelectParameter, multiple=True, readonly=True) - MetadataElement(name="col_graphs_count", default=0, desc="col_graphs_count", readonly=True, visible=True, no_value=0) - MetadataElement(name="col_graphs_names", desc="col_graphs_names", default=[], param=metadata.SelectParameter, multiple=True, readonly=True) - MetadataElement(name="row_graphs_count", default=0, desc="row_graphs_count", readonly=True, visible=True, no_value=0) - MetadataElement(name="row_graphs_names", desc="row_graphs_names", default=[], param=metadata.SelectParameter, multiple=True, readonly=True) + MetadataElement( + name="col_attrs_names", + desc="col_attrs_names", + default=[], + param=metadata.SelectParameter, + multiple=True, + readonly=True, + ) + MetadataElement( + name="col_graphs_count", default=0, desc="col_graphs_count", readonly=True, visible=True, no_value=0 + ) + MetadataElement( + name="col_graphs_names", + desc="col_graphs_names", + default=[], + param=metadata.SelectParameter, + multiple=True, + readonly=True, + ) + MetadataElement( + name="row_graphs_count", default=0, desc="row_graphs_count", readonly=True, visible=True, no_value=0 + ) + MetadataElement( + name="row_graphs_names", + desc="row_graphs_names", + default=[], + param=metadata.SelectParameter, + multiple=True, + readonly=True, + ) def sniff(self, filename): if super().sniff(filename): - with h5py.File(filename, 'r') as loom_file: + with h5py.File(filename, "r") as loom_file: # Check the optional but distinctive LOOM_SPEC_VERSION attribute - if bool(loom_file.attrs.get('LOOM_SPEC_VERSION')): + if bool(loom_file.attrs.get("LOOM_SPEC_VERSION")): return True # Check some mandatory H5 datasets and groups - for el in ('matrix', 'row_attrs', 'col_attrs'): + for el in ("matrix", "row_attrs", "col_attrs"): if loom_file.get(el) is None: return False else: @@ -965,8 +1173,8 @@ class Loom(H5): dataset.peek = "Binary Loom file" dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -977,29 +1185,29 @@ class Loom(H5): def set_meta(self, dataset, overwrite=True, **kwd): super().set_meta(dataset, overwrite=overwrite, **kwd) try: - with h5py.File(dataset.file_name, 'r') as loom_file: - dataset.metadata.title = loom_file.attrs.get('title') - dataset.metadata.description = loom_file.attrs.get('description') - dataset.metadata.url = loom_file.attrs.get('url') - dataset.metadata.doi = loom_file.attrs.get('doi') - loom_spec_version = loom_file.attrs.get('LOOM_SPEC_VERSION') + with h5py.File(dataset.file_name, "r") as loom_file: + dataset.metadata.title = loom_file.attrs.get("title") + dataset.metadata.description = loom_file.attrs.get("description") + dataset.metadata.url = loom_file.attrs.get("url") + dataset.metadata.doi = loom_file.attrs.get("doi") + loom_spec_version = loom_file.attrs.get("LOOM_SPEC_VERSION") if isinstance(loom_spec_version, np.ndarray): loom_spec_version = loom_spec_version[0] if isinstance(loom_spec_version, bytes): loom_spec_version = loom_spec_version.decode() dataset.metadata.loom_spec_version = loom_spec_version - dataset.creation_date = loom_file.attrs.get('creation_date') - dataset.metadata.shape = tuple(loom_file['matrix'].shape) + dataset.creation_date = loom_file.attrs.get("creation_date") + dataset.metadata.shape = tuple(loom_file["matrix"].shape) - tmp = list(loom_file.get('layers', {}).keys()) + tmp = list(loom_file.get("layers", {}).keys()) dataset.metadata.layers_count = len(tmp) dataset.metadata.layers_names = tmp - tmp = list(loom_file['row_attrs'].keys()) + tmp = list(loom_file["row_attrs"].keys()) dataset.metadata.row_attrs_count = len(tmp) dataset.metadata.row_attrs_names = tmp - tmp = list(loom_file['col_attrs'].keys()) + tmp = list(loom_file["col_attrs"].keys()) dataset.metadata.col_attrs_count = len(tmp) dataset.metadata.col_attrs_names = tmp @@ -1007,15 +1215,15 @@ class Loom(H5): # and row_graphs are mandatory groups, but files created by # Bioconductor LoomExperiment do not always have them: # https://github.com/Bioconductor/LoomExperiment/issues/7 - tmp = list(loom_file.get('col_graphs', {}).keys()) + tmp = list(loom_file.get("col_graphs", {}).keys()) dataset.metadata.col_graphs_count = len(tmp) dataset.metadata.col_graphs_names = tmp - tmp = list(loom_file.get('row_graphs', {}).keys()) + tmp = list(loom_file.get("row_graphs", {}).keys()) dataset.metadata.row_graphs_count = len(tmp) dataset.metadata.row_graphs_names = tmp except Exception as e: - log.warning('%s, set_meta Exception: %s', self, e) + log.warning("%s, set_meta Exception: %s", self, e) class Anndata(H5): @@ -1041,57 +1249,92 @@ class Anndata(H5): >>> Anndata().sniff(get_test_fname('adata_unk.h5ad')) True """ - file_ext = 'h5ad' + + file_ext = "h5ad" MetadataElement(name="title", default="", desc="title", readonly=True, visible=True, no_value="") MetadataElement(name="description", default="", desc="description", readonly=True, visible=True, no_value="") MetadataElement(name="url", default="", desc="url", readonly=True, visible=True, no_value="") MetadataElement(name="doi", default="", desc="doi", readonly=True, visible=True, no_value="") - MetadataElement(name="anndata_spec_version", default="", desc="anndata_spec_version", readonly=True, visible=True, no_value="") + MetadataElement( + name="anndata_spec_version", default="", desc="anndata_spec_version", readonly=True, visible=True, no_value="" + ) MetadataElement(name="creation_date", default=None, desc="creation_date", readonly=True, visible=True) MetadataElement(name="layers_count", default=0, desc="layers_count", readonly=True, visible=True, no_value=0) - MetadataElement(name="layers_names", desc="layers_names", default=[], param=metadata.SelectParameter, multiple=True, readonly=True) + MetadataElement( + name="layers_names", + desc="layers_names", + default=[], + param=metadata.SelectParameter, + multiple=True, + readonly=True, + ) MetadataElement(name="row_attrs_count", default=0, desc="row_attrs_count", readonly=True, visible=True, no_value=0) # obs_names: Cell1, Cell2, Cell3,... # obs_layers: louvain, leidein, isBcell # obs_count: number of obs_layers # obs_size: number of obs_names MetadataElement(name="obs_names", desc="obs_names", default=[], multiple=True, readonly=True) - MetadataElement(name="obs_layers", desc="obs_layers", default=[], param=metadata.SelectParameter, multiple=True, readonly=True) + MetadataElement( + name="obs_layers", desc="obs_layers", default=[], param=metadata.SelectParameter, multiple=True, readonly=True + ) MetadataElement(name="obs_count", default=0, desc="obs_count", readonly=True, visible=True, no_value=0) MetadataElement(name="obs_size", default=-1, desc="obs_size", readonly=True, visible=True, no_value=0) - MetadataElement(name="obsm_layers", desc="obsm_layers", default=[], param=metadata.SelectParameter, multiple=True, readonly=True) + MetadataElement( + name="obsm_layers", desc="obsm_layers", default=[], param=metadata.SelectParameter, multiple=True, readonly=True + ) MetadataElement(name="obsm_count", default=0, desc="obsm_count", readonly=True, visible=True, no_value=0) - MetadataElement(name="raw_var_layers", desc="raw_var_layers", default=[], param=metadata.SelectParameter, multiple=True, readonly=True) + MetadataElement( + name="raw_var_layers", + desc="raw_var_layers", + default=[], + param=metadata.SelectParameter, + multiple=True, + readonly=True, + ) MetadataElement(name="raw_var_count", default=0, desc="raw_var_count", readonly=True, visible=True, no_value=0) MetadataElement(name="raw_var_size", default=0, desc="raw_var_size", readonly=True, visible=True, no_value=0) - MetadataElement(name="var_layers", desc="var_layers", default=[], param=metadata.SelectParameter, multiple=True, readonly=True) + MetadataElement( + name="var_layers", desc="var_layers", default=[], param=metadata.SelectParameter, multiple=True, readonly=True + ) MetadataElement(name="var_count", default=0, desc="var_count", readonly=True, visible=True, no_value=0) MetadataElement(name="var_size", default=-1, desc="var_size", readonly=True, visible=True, no_value=0) - MetadataElement(name="varm_layers", desc="varm_layers", default=[], param=metadata.SelectParameter, multiple=True, readonly=True) + MetadataElement( + name="varm_layers", desc="varm_layers", default=[], param=metadata.SelectParameter, multiple=True, readonly=True + ) MetadataElement(name="varm_count", default=0, desc="varm_count", readonly=True, visible=True, no_value=0) - MetadataElement(name="uns_layers", desc="uns_layers", default=[], param=metadata.SelectParameter, multiple=True, readonly=True) + MetadataElement( + name="uns_layers", desc="uns_layers", default=[], param=metadata.SelectParameter, multiple=True, readonly=True + ) MetadataElement(name="uns_count", default=0, desc="uns_count", readonly=True, visible=True, no_value=0) - MetadataElement(name="shape", default=(-1, -1), desc="shape", param=metadata.ListParameter, readonly=True, visible=True, no_value=(0, 0)) + MetadataElement( + name="shape", + default=(-1, -1), + desc="shape", + param=metadata.ListParameter, + readonly=True, + visible=True, + no_value=(0, 0), + ) def sniff(self, filename): if super().sniff(filename): try: - with h5py.File(filename, 'r') as f: - return all(attr in f for attr in ['X', 'obs', 'var']) + with h5py.File(filename, "r") as f: + return all(attr in f for attr in ["X", "obs", "var"]) except Exception: return False return False def set_meta(self, dataset, overwrite=True, **kwd): super().set_meta(dataset, overwrite=overwrite, **kwd) - with h5py.File(dataset.file_name, 'r') as anndata_file: - dataset.metadata.title = anndata_file.attrs.get('title') - dataset.metadata.description = anndata_file.attrs.get('description') - dataset.metadata.url = anndata_file.attrs.get('url') - dataset.metadata.doi = anndata_file.attrs.get('doi') - dataset.creation_date = anndata_file.attrs.get('creation_date') - dataset.metadata.shape = anndata_file.attrs.get('shape', dataset.metadata.shape) + with h5py.File(dataset.file_name, "r") as anndata_file: + dataset.metadata.title = anndata_file.attrs.get("title") + dataset.metadata.description = anndata_file.attrs.get("description") + dataset.metadata.url = anndata_file.attrs.get("url") + dataset.metadata.doi = anndata_file.attrs.get("doi") + dataset.creation_date = anndata_file.attrs.get("creation_date") + dataset.metadata.shape = anndata_file.attrs.get("shape", dataset.metadata.shape) # none of the above appear to work in any dataset tested, but could be useful for # future AnnData datasets dataset.metadata.layers_count = len(anndata_file) @@ -1099,7 +1342,7 @@ class Anndata(H5): def _layercountsize(tmp, lennames=0): "From TMP and LENNAMES, return layers, their number, and the length of one of the layers (all equal)." - if hasattr(tmp, 'dtype'): + if hasattr(tmp, "dtype"): layers = list(tmp.dtype.names) count = len(tmp.dtype) size = int(tmp.size) @@ -1109,7 +1352,7 @@ class Anndata(H5): size = lennames return (layers, count, size) - if 'obs' in dataset.metadata.layers_names: + if "obs" in dataset.metadata.layers_names: tmp = anndata_file["obs"] obs_index = None if "index" in tmp: @@ -1119,7 +1362,7 @@ class Anndata(H5): # Determine cell labels if obs_index: dataset.metadata.obs_names = list(tmp[obs_index]) - elif hasattr(tmp, 'dtype'): + elif hasattr(tmp, "dtype"): if "index" in tmp.dtype.names: # Yes, we call tmp["index"], and not tmp.dtype["index"] # here, despite the above tests. @@ -1136,11 +1379,11 @@ class Anndata(H5): dataset.metadata.obs_count = y dataset.metadata.obs_size = z - if 'obsm' in dataset.metadata.layers_names: + if "obsm" in dataset.metadata.layers_names: tmp = anndata_file["obsm"] dataset.metadata.obsm_layers, dataset.metadata.obsm_count, _ = _layercountsize(tmp) - if 'raw.var' in dataset.metadata.layers_names: + if "raw.var" in dataset.metadata.layers_names: tmp = anndata_file["raw.var"] # full set of genes would never need to be previewed # dataset.metadata.raw_var_names = tmp["index"] @@ -1149,7 +1392,7 @@ class Anndata(H5): dataset.metadata.raw_var_count = y dataset.metadata.raw_var_size = z - if 'var' in dataset.metadata.layers_names: + if "var" in dataset.metadata.layers_names: tmp = anndata_file["var"] var_index = None if "index" in tmp: @@ -1169,22 +1412,22 @@ class Anndata(H5): dataset.metadata.var_count = y dataset.metadata.var_size = z - if 'varm' in dataset.metadata.layers_names: + if "varm" in dataset.metadata.layers_names: tmp = anndata_file["varm"] dataset.metadata.varm_layers, dataset.metadata.varm_count, _ = _layercountsize(tmp) - if 'uns' in dataset.metadata.layers_names: + if "uns" in dataset.metadata.layers_names: tmp = anndata_file["uns"] dataset.metadata.uns_layers, dataset.metadata.uns_count, _ = _layercountsize(tmp) # Resolving the problematic shape parameter - if 'X' in dataset.metadata.layers_names: + if "X" in dataset.metadata.layers_names: # Shape we determine here due to the non-standard representation of 'X' dimensions - shape = anndata_file['X'].attrs.get("shape") + shape = anndata_file["X"].attrs.get("shape") if shape is not None: dataset.metadata.shape = tuple(shape) - elif hasattr(anndata_file['X'], 'shape'): - dataset.metadata.shape = tuple(anndata_file['X'].shape) + elif hasattr(anndata_file["X"], "shape"): + dataset.metadata.shape = tuple(anndata_file["X"].shape) if dataset.metadata.shape is None: dataset.metadata.shape = (int(dataset.metadata.obs_size), int(dataset.metadata.var_size)) @@ -1200,7 +1443,7 @@ class Anndata(H5): layer, count, "layer" if count == 1 else "layers", - ', '.join(sorted(names)) + ", ".join(sorted(names)), ) return "" @@ -1214,8 +1457,8 @@ class Anndata(H5): dataset.peek = peekstr dataset.blurb = f"Anndata file ({nice_size(dataset.get_size())})" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -1237,14 +1480,17 @@ class Grib(Binary): >>> Grib().sniff_prefix(fname) False """ + file_ext = "grib" # GRIB not yet in EDAM (work in progress). For now, so set to binary edam_format = "format_2333" - MetadataElement(name="grib_edition", default=1, desc="GRIB edition", readonly=True, visible=True, optional=True, no_value=0) + MetadataElement( + name="grib_edition", default=1, desc="GRIB edition", readonly=True, visible=True, optional=True, no_value=0 + ) def __init__(self, **kwd): super().__init__(**kwd) - self._magic = b'GRIB' + self._magic = b"GRIB" def sniff_prefix(self, file_prefix: FilePrefix): # The first 4 bytes of any GRIB file are GRIB @@ -1264,8 +1510,8 @@ class Grib(Binary): dataset.peek = "Binary GRIB file" dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -1282,7 +1528,7 @@ class Grib(Binary): def _get_grib_edition(self, filename): _uint8struct = struct.Struct(b">B") edition = 0 - with open(filename, 'rb') as f: + with open(filename, "rb") as f: f.seek(4) tmp = f.read(4) edition = _uint8struct.unpack_from(tmp, 3)[0] @@ -1300,15 +1546,15 @@ class GmxBinary(Binary): def sniff_prefix(self, sniff_prefix): # The first 4 bytes of any GROMACS binary file containing the magic number - return sniff_prefix.magic_header('>1i') == self.magic_number + return sniff_prefix.magic_header(">1i") == self.magic_number def set_peek(self, dataset): if not dataset.dataset.purged: dataset.peek = f"Binary GROMACS {self.file_ext} file" dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -1382,22 +1628,41 @@ class Edr(GmxBinary): """ file_ext = "edr" - magic_number = -55555 # reference: https://github.com/gromacs/gromacs/blob/cec211b2c835ba6e8ea849fb1bf67d7fc19693a4/src/gromacs/fileio/enxio.cpp + magic_number = ( + -55555 + ) # reference: https://github.com/gromacs/gromacs/blob/cec211b2c835ba6e8ea849fb1bf67d7fc19693a4/src/gromacs/fileio/enxio.cpp class Biom2(H5): """ Class describing a biom2 file (http://biom-format.org/documentation/biom_format.html) """ + MetadataElement(name="id", default=None, desc="table id", readonly=True, visible=True) MetadataElement(name="format_url", default=None, desc="format-url", readonly=True, visible=True) - MetadataElement(name="format_version", default=None, desc="format-version (equal to format)", readonly=True, visible=True) + MetadataElement( + name="format_version", default=None, desc="format-version (equal to format)", readonly=True, visible=True + ) MetadataElement(name="format", default=None, desc="format (equal to format=version)", readonly=True, visible=True) MetadataElement(name="type", default=None, desc="table type", readonly=True, visible=True) MetadataElement(name="generated_by", default=None, desc="generated by", readonly=True, visible=True) MetadataElement(name="creation_date", default=None, desc="creation date", readonly=True, visible=True) - MetadataElement(name="nnz", default=-1, desc="nnz: The number of non-zero elements in the table", readonly=True, visible=True, no_value=-1) - MetadataElement(name="shape", default=(), desc="shape: The number of rows and columns in the dataset", readonly=True, visible=True, no_value=()) + MetadataElement( + name="nnz", + default=-1, + desc="nnz: The number of non-zero elements in the table", + readonly=True, + visible=True, + no_value=-1, + ) + MetadataElement( + name="shape", + default=(), + desc="shape: The number of rows and columns in the dataset", + readonly=True, + visible=True, + no_value=(), + ) file_ext = "biom2" edam_format = "format_3746" @@ -1416,47 +1681,47 @@ class Biom2(H5): False """ if super().sniff(filename): - with h5py.File(filename, 'r') as f: - required_fields = {'id', 'format-url', 'type', 'generated-by', 'creation-date', 'nnz', 'shape'} + with h5py.File(filename, "r") as f: + required_fields = {"id", "format-url", "type", "generated-by", "creation-date", "nnz", "shape"} return required_fields.issubset(f.attrs.keys()) return False def set_meta(self, dataset, overwrite=True, **kwd): super().set_meta(dataset, overwrite=overwrite, **kwd) try: - with h5py.File(dataset.file_name, 'r') as f: + with h5py.File(dataset.file_name, "r") as f: attributes = f.attrs - dataset.metadata.id = util.unicodify(attributes['id']) - dataset.metadata.format_url = util.unicodify(attributes['format-url']) - if 'format-version' in attributes: # biom 2.1 - dataset.metadata.format_version = '.'.join(str(_) for _ in attributes['format-version']) + dataset.metadata.id = util.unicodify(attributes["id"]) + dataset.metadata.format_url = util.unicodify(attributes["format-url"]) + if "format-version" in attributes: # biom 2.1 + dataset.metadata.format_version = ".".join(str(_) for _ in attributes["format-version"]) dataset.metadata.format = dataset.metadata.format_version - elif 'format' in attributes: # biom 2.0 - dataset.metadata.format = util.unicodify(attributes['format']) + elif "format" in attributes: # biom 2.0 + dataset.metadata.format = util.unicodify(attributes["format"]) dataset.metadata.format_version = dataset.metadata.format - dataset.metadata.type = util.unicodify(attributes['type']) - dataset.metadata.shape = tuple(int(_) for _ in attributes['shape']) - dataset.metadata.generated_by = util.unicodify(attributes['generated-by']) - dataset.metadata.creation_date = util.unicodify(attributes['creation-date']) - dataset.metadata.nnz = int(attributes['nnz']) + dataset.metadata.type = util.unicodify(attributes["type"]) + dataset.metadata.shape = tuple(int(_) for _ in attributes["shape"]) + dataset.metadata.generated_by = util.unicodify(attributes["generated-by"]) + dataset.metadata.creation_date = util.unicodify(attributes["creation-date"]) + dataset.metadata.nnz = int(attributes["nnz"]) except Exception as e: - log.warning('%s, set_meta Exception: %s', self, util.unicodify(e)) + log.warning("%s, set_meta Exception: %s", self, util.unicodify(e)) def set_peek(self, dataset): if not dataset.dataset.purged: - lines = ['Biom2 (HDF5) file'] + lines = ["Biom2 (HDF5) file"] try: with h5py.File(dataset.file_name) as f: for k, v in f.attrs.items(): - lines.append(f'{k}: {util.unicodify(v)}') + lines.append(f"{k}: {util.unicodify(v)}") except Exception as e: - log.warning('%s, set_peek Exception: %s', self, util.unicodify(e)) - dataset.peek = '\n'.join(lines) + log.warning("%s, set_peek Exception: %s", self, util.unicodify(e)) + dataset.peek = "\n".join(lines) dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -1493,10 +1758,10 @@ class Cool(H5): URL = "https://github.com/mirnylab/cooler" if super().sniff(filename): - keys = ['chroms', 'bins', 'pixels', 'indexes'] - with h5py.File(filename, 'r') as handle: - fmt = util.unicodify(handle.attrs.get('format')) - url = util.unicodify(handle.attrs.get('format-url')) + keys = ["chroms", "bins", "pixels", "indexes"] + with h5py.File(filename, "r") as handle: + fmt = util.unicodify(handle.attrs.get("format")) + url = util.unicodify(handle.attrs.get("format-url")) if fmt == MAGIC or url == URL: if not all(name in handle.keys() for name in keys): return False @@ -1508,8 +1773,8 @@ class Cool(H5): dataset.peek = "Cool (HDF5) file for storing genomic interaction data." dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -1549,16 +1814,16 @@ class MCool(H5): URL = "https://github.com/mirnylab/cooler" if super().sniff(filename): - keys0 = ['resolutions'] - with h5py.File(filename, 'r') as handle: + keys0 = ["resolutions"] + with h5py.File(filename, "r") as handle: if not all(name in handle.keys() for name in keys0): return False - res0 = next(iter(handle['resolutions'].keys())) - keys = ['chroms', 'bins', 'pixels', 'indexes'] - fmt = util.unicodify(handle['resolutions'][res0].attrs.get('format')) - url = util.unicodify(handle['resolutions'][res0].attrs.get('format-url')) + res0 = next(iter(handle["resolutions"].keys())) + keys = ["chroms", "bins", "pixels", "indexes"] + fmt = util.unicodify(handle["resolutions"][res0].attrs.get("format")) + url = util.unicodify(handle["resolutions"][res0].attrs.get("format-url")) if fmt == MAGIC or url == URL: - if not all(name in handle['resolutions'][res0].keys() for name in keys): + if not all(name in handle["resolutions"][res0].keys() for name in keys): return False return True return False @@ -1568,8 +1833,8 @@ class MCool(H5): dataset.peek = "Multi-resolution Cool (HDF5) file for storing genomic interaction data." dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -1582,13 +1847,22 @@ class H5MLM(H5): """ Machine learning model generated by Galaxy-ML. """ + file_ext = "h5mlm" URL = "https://github.com/goeckslab/Galaxy-ML" - max_peek_size = 1000 # 1 KB - max_preview_size = 1000000 # 1 MB + max_peek_size = 1000 # 1 KB + max_preview_size = 1000000 # 1 MB - MetadataElement(name="hyper_params", desc="Hyperparameter File", param=FileParameter, file_ext="tabular", readonly=True, visible=False, optional=True) + MetadataElement( + name="hyper_params", + desc="Hyperparameter File", + param=FileParameter, + file_ext="tabular", + readonly=True, + visible=False, + optional=True, + ) def set_meta(self, dataset, overwrite=True, **kwd): try: @@ -1605,7 +1879,7 @@ class H5MLM(H5): f.write("\t".join(p) + "\n") dataset.metadata.hyper_params = params_file except Exception as e: - log.warning('%s, set_meta Exception: %s', self, e) + log.warning("%s, set_meta Exception: %s", self, e) def sniff(self, filename): if super().sniff(filename): @@ -1624,7 +1898,7 @@ class H5MLM(H5): repr_ = util.unicodify(handle.attrs.get("-repr-")) return repr_ except Exception as e: - log.warning('%s, get_repr Except: %s', self, e) + log.warning("%s, get_repr Except: %s", self, e) return "" def get_config_string(self, filename): @@ -1633,17 +1907,17 @@ class H5MLM(H5): config = util.unicodify(handle["-model_config-"][()]) return config except Exception as e: - log.warning('%s, get model configuration Except: %s', self, e) + log.warning("%s, get model configuration Except: %s", self, e) return "" def set_peek(self, dataset): if not dataset.dataset.purged: repr_ = self.get_repr(dataset.file_name) - dataset.peek = repr_[:self.max_peek_size] + dataset.peek = repr_[: self.max_peek_size] dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -1662,17 +1936,17 @@ class H5MLM(H5): rval = {} try: with h5py.File(dataset.file_name, "r") as handle: - rval['Attributes'] = {} + rval["Attributes"] = {} attributes = handle.attrs - for k in (set(attributes.keys()) - {'-URL-', '-repr-'}): - rval['Attributes'][k] = util.unicodify(attributes.get(k)) + for k in set(attributes.keys()) - {"-URL-", "-repr-"}: + rval["Attributes"][k] = util.unicodify(attributes.get(k)) except Exception as e: log.warning(e) config = self.get_config_string(dataset.file_name) - rval['Config'] = json.loads(config) if config else '' + rval["Config"] = json.loads(config) if config else "" rval = json.dumps(rval, sort_keys=True, indent=2) - rval = rval[:self.max_preview_size] + rval = rval[: self.max_preview_size] repr_ = self.get_repr(dataset.file_name) @@ -1683,26 +1957,29 @@ class LudwigModel(Html): """ Composite datatype that encloses multiple files for a Ludwig trained model. """ - composite_type = 'auto_primary_file' + + composite_type = "auto_primary_file" file_ext = "ludwig_model" def __init__(self, **kwd): super().__init__(**kwd) - self.add_composite_file('model_hyperparameters.json', description='Model hyperparameters', is_binary=False) - self.add_composite_file('model_weights', description='Model weights', is_binary=True) - self.add_composite_file('training_set_metadata.json', description='Training set metadata', is_binary=False) - self.add_composite_file('training_progress.json', description='Training progress', is_binary=False, optional=True) + self.add_composite_file("model_hyperparameters.json", description="Model hyperparameters", is_binary=False) + self.add_composite_file("model_weights", description="Model weights", is_binary=True) + self.add_composite_file("training_set_metadata.json", description="Training set metadata", is_binary=False) + self.add_composite_file( + "training_progress.json", description="Training progress", is_binary=False, optional=True + ) def generate_primary_file(self, dataset=None): - rval = ['Ludwig Model Composite Dataset.

    '] - rval.append('

    This model dataset is composed of the following items:

      ') + rval = ["Ludwig Model Composite Dataset.

      "] + rval.append("

      This model dataset is composed of the following items:

        ") for composite_name, composite_file in self.get_composite_files(dataset=dataset).items(): - description = composite_file.get('description') - link_text = f'{composite_name} ({description})' if description else composite_name - opt_text = ' (optional)' if composite_file.optional else '' + description = composite_file.get("description") + link_text = f"{composite_name} ({description})" if description else composite_name + opt_text = " (optional)" if composite_file.optional else "" rval.append(f'
      • {link_text}{opt_text}
      • ') - rval.append('
      ') + rval.append("
    ") return "\n".join(rval) @@ -1718,17 +1995,36 @@ class HexrdMaterials(H5): >>> HexrdMaterials().sniff(fname) False """ + file_ext = "hexrd.materials.h5" edam_format = "format_3590" - MetadataElement(name="materials", desc="materials", default=[], param=metadata.SelectParameter, multiple=True, readonly=True) - MetadataElement(name="SpaceGroupNumber", default={}, param=DictParameter, desc="SpaceGroupNumber", readonly=True, visible=True, no_value={}) - MetadataElement(name="LatticeParameters", default={}, param=DictParameter, desc="LatticeParameters", readonly=True, visible=True, no_value={}) + MetadataElement( + name="materials", desc="materials", default=[], param=metadata.SelectParameter, multiple=True, readonly=True + ) + MetadataElement( + name="SpaceGroupNumber", + default={}, + param=DictParameter, + desc="SpaceGroupNumber", + readonly=True, + visible=True, + no_value={}, + ) + MetadataElement( + name="LatticeParameters", + default={}, + param=DictParameter, + desc="LatticeParameters", + readonly=True, + visible=True, + no_value={}, + ) def sniff(self, filename): if super().sniff(filename): - req = {'AtomData', 'Atomtypes', 'CrystalSystem', 'LatticeParameters'} - with h5py.File(filename, 'r') as mat_file: + req = {"AtomData", "Atomtypes", "CrystalSystem", "LatticeParameters"} + with h5py.File(filename, "r") as mat_file: for k in mat_file.keys(): if isinstance(mat_file[k], h5py._hl.group.Group) and set(mat_file[k].keys()) >= req: return True @@ -1737,38 +2033,41 @@ class HexrdMaterials(H5): def set_meta(self, dataset, overwrite=True, **kwd): super().set_meta(dataset, overwrite=overwrite, **kwd) try: - with h5py.File(dataset.file_name, 'r') as mat_file: + with h5py.File(dataset.file_name, "r") as mat_file: dataset.metadata.materials = list(mat_file.keys()) sgn = dict() lp = dict() for m in mat_file.keys(): - if 'SpaceGroupNumber' in mat_file[m] and len(mat_file[m]['SpaceGroupNumber']) > 0: - sgn[m] = mat_file[m]['SpaceGroupNumber'][0].item() - if 'LatticeParameters' in mat_file[m]: - lp[m] = mat_file[m]['LatticeParameters'][0:].tolist() + if "SpaceGroupNumber" in mat_file[m] and len(mat_file[m]["SpaceGroupNumber"]) > 0: + sgn[m] = mat_file[m]["SpaceGroupNumber"][0].item() + if "LatticeParameters" in mat_file[m]: + lp[m] = mat_file[m]["LatticeParameters"][0:].tolist() dataset.metadata.SpaceGroupNumber = sgn dataset.metadata.LatticeParameters = lp except Exception as e: - log.warning('%s, set_meta Exception: %s', self, e) + log.warning("%s, set_meta Exception: %s", self, e) def set_peek(self, dataset): if not dataset.dataset.purged: - lines = ['Material SpaceGroup Lattice'] + lines = ["Material SpaceGroup Lattice"] if dataset.metadata.materials: for m in dataset.metadata.materials: try: - lines.append(f'{m} {dataset.metadata.SpaceGroupNumber[m]} {dataset.metadata.LatticeParameters[m]}') + lines.append( + f"{m} {dataset.metadata.SpaceGroupNumber[m]} {dataset.metadata.LatticeParameters[m]}" + ) except Exception: continue - dataset.peek = '\n'.join(lines) + dataset.peek = "\n".join(lines) dataset.blurb = f"Materials: {' '.join(dataset.metadata.materials)}" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" class Scf(Binary): """Class describing an scf binary sequence file""" + edam_format = "format_1632" edam_data = "data_0924" file_ext = "scf" @@ -1778,8 +2077,8 @@ class Scf(Binary): dataset.peek = "Binary scf sequence file" dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -1790,7 +2089,8 @@ class Scf(Binary): @build_sniff_from_prefix class Sff(Binary): - """ Standard Flowgram Format (SFF) """ + """Standard Flowgram Format (SFF)""" + edam_format = "format_3284" edam_data = "data_0924" file_ext = "sff" @@ -1798,15 +2098,15 @@ class Sff(Binary): def sniff_prefix(self, sniff_prefix): # The first 4 bytes of any sff file is '.sff', and the file is binary. For details # about the format, see http://www.ncbi.nlm.nih.gov/Traces/trace.cgi?cmd=show&f=formats&m=doc&s=format - return sniff_prefix.startswith_bytes(b'.sff') + return sniff_prefix.startswith_bytes(b".sff") def set_peek(self, dataset): if not dataset.dataset.purged: dataset.peek = "Binary sff file" dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -1822,6 +2122,7 @@ class BigWig(Binary): The supplemental info in the paper has the binary details: http://bioinformatics.oxfordjournals.org/cgi/content/abstract/btq351v1 """ + edam_format = "format_3006" edam_data = "data_3002" file_ext = "bigwig" @@ -1841,8 +2142,8 @@ class BigWig(Binary): dataset.peek = f"Binary UCSC {self._name} file" dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -1853,6 +2154,7 @@ class BigWig(Binary): class BigBed(BigWig): """BigBed support from UCSC.""" + edam_format = "format_3004" edam_data = "data_3002" file_ext = "bigbed" @@ -1867,6 +2169,7 @@ class BigBed(BigWig): @build_sniff_from_prefix class TwoBit(Binary): """Class describing a TwoBit format nucleotide file""" + edam_format = "format_3009" edam_data = "data_0848" file_ext = "twobit" @@ -1891,10 +2194,29 @@ class TwoBit(Binary): @dataproviders.decorators.has_dataproviders class SQlite(Binary): - """Class describing a Sqlite database """ - MetadataElement(name="tables", default=[], param=ListParameter, desc="Database Tables", readonly=True, visible=True, no_value=[]) - MetadataElement(name="table_columns", default={}, param=DictParameter, desc="Database Table Columns", readonly=True, visible=True, no_value={}) - MetadataElement(name="table_row_count", default={}, param=DictParameter, desc="Database Table Row Count", readonly=True, visible=True, no_value={}) + """Class describing a Sqlite database""" + + MetadataElement( + name="tables", default=[], param=ListParameter, desc="Database Tables", readonly=True, visible=True, no_value=[] + ) + MetadataElement( + name="table_columns", + default={}, + param=DictParameter, + desc="Database Table Columns", + readonly=True, + visible=True, + no_value={}, + ) + MetadataElement( + name="table_row_count", + default={}, + param=DictParameter, + desc="Database Table Row Count", + readonly=True, + visible=True, + no_value={}, + ) file_ext = "sqlite" edam_format = "format_3621" @@ -1913,30 +2235,30 @@ class SQlite(Binary): for table, _ in rslt: tables.append(table) try: - col_query = f'SELECT * FROM {table} LIMIT 0' + col_query = f"SELECT * FROM {table} LIMIT 0" cur = conn.cursor().execute(col_query) cols = [col[0] for col in cur.description] columns[table] = cols except Exception as exc: - log.warning('%s, set_meta Exception: %s', self, exc) + log.warning("%s, set_meta Exception: %s", self, exc) for table in tables: try: row_query = f"SELECT count(*) FROM {table}" rowcounts[table] = c.execute(row_query).fetchone()[0] except Exception as exc: - log.warning('%s, set_meta Exception: %s', self, exc) + log.warning("%s, set_meta Exception: %s", self, exc) dataset.metadata.tables = tables dataset.metadata.table_columns = columns dataset.metadata.table_row_count = rowcounts except Exception as exc: - log.warning('%s, set_meta Exception: %s', self, exc) + log.warning("%s, set_meta Exception: %s", self, exc) def sniff(self, filename): # The first 16 bytes of any SQLite3 database file is 'SQLite format 3\0', and the file is binary. For details # about the format, see http://www.sqlite.org/fileformat.html try: - header = open(filename, 'rb').read(16) - if header == b'SQLite format 3\0': + header = open(filename, "rb").read(16) + if header == b"SQLite format 3\0": return True return False except Exception: @@ -1955,24 +2277,24 @@ class SQlite(Binary): return False return True except Exception as e: - log.warning('%s, sniff Exception: %s', self, e) + log.warning("%s, sniff Exception: %s", self, e) return False def set_peek(self, dataset): if not dataset.dataset.purged: dataset.peek = "SQLite Database" - lines = ['SQLite Database'] + lines = ["SQLite Database"] if dataset.metadata.tables: for table in dataset.metadata.tables: try: - lines.append(f'{table} [{dataset.metadata.table_row_count[table]}]') + lines.append(f"{table} [{dataset.metadata.table_row_count[table]}]") except Exception: continue - dataset.peek = '\n'.join(lines) + dataset.peek = "\n".join(lines) dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -1980,26 +2302,36 @@ class SQlite(Binary): except Exception: return f"SQLite Database ({nice_size(dataset.get_size())})" - @dataproviders.decorators.dataprovider_factory('sqlite', dataproviders.dataset.SQliteDataProvider.settings) + @dataproviders.decorators.dataprovider_factory("sqlite", dataproviders.dataset.SQliteDataProvider.settings) def sqlite_dataprovider(self, dataset, **settings): dataset_source = dataproviders.dataset.DatasetDataProvider(dataset) return dataproviders.dataset.SQliteDataProvider(dataset_source, **settings) - @dataproviders.decorators.dataprovider_factory('sqlite-table', dataproviders.dataset.SQliteDataTableProvider.settings) + @dataproviders.decorators.dataprovider_factory( + "sqlite-table", dataproviders.dataset.SQliteDataTableProvider.settings + ) def sqlite_datatableprovider(self, dataset, **settings): dataset_source = dataproviders.dataset.DatasetDataProvider(dataset) return dataproviders.dataset.SQliteDataTableProvider(dataset_source, **settings) - @dataproviders.decorators.dataprovider_factory('sqlite-dict', dataproviders.dataset.SQliteDataDictProvider.settings) + @dataproviders.decorators.dataprovider_factory("sqlite-dict", dataproviders.dataset.SQliteDataDictProvider.settings) def sqlite_datadictprovider(self, dataset, **settings): dataset_source = dataproviders.dataset.DatasetDataProvider(dataset) return dataproviders.dataset.SQliteDataDictProvider(dataset_source, **settings) class GeminiSQLite(SQlite): - """Class describing a Gemini Sqlite database """ - MetadataElement(name="gemini_version", default='0.10.0', param=MetadataParameter, desc="Gemini Version", - readonly=True, visible=True, no_value='0.10.0') + """Class describing a Gemini Sqlite database""" + + MetadataElement( + name="gemini_version", + default="0.10.0", + param=MetadataParameter, + desc="Gemini Version", + readonly=True, + visible=True, + no_value="0.10.0", + ) file_ext = "gemini.sqlite" edam_format = "format_3622" edam_data = "data_3498" @@ -2011,36 +2343,46 @@ class GeminiSQLite(SQlite): c = conn.cursor() tables_query = "SELECT version FROM version" result = c.execute(tables_query).fetchall() - for version, in result: + for (version,) in result: dataset.metadata.gemini_version = version # TODO: Can/should we detect even more attributes, such as use of PED file, what was input annotation type, etc. except Exception as e: - log.warning('%s, set_meta Exception: %s', self, e) + log.warning("%s, set_meta Exception: %s", self, e) def sniff(self, filename): if super().sniff(filename): - table_names = ["gene_detailed", "gene_summary", "resources", "sample_genotype_counts", - "sample_genotypes", "samples", "variant_impacts", "variants", "version"] + table_names = [ + "gene_detailed", + "gene_summary", + "resources", + "sample_genotype_counts", + "sample_genotypes", + "samples", + "variant_impacts", + "variants", + "version", + ] return self.sniff_table_names(filename, table_names) return False def set_peek(self, dataset): if not dataset.dataset.purged: - dataset.peek = "Gemini SQLite Database, version %s" % (dataset.metadata.gemini_version or 'unknown') + dataset.peek = "Gemini SQLite Database, version %s" % (dataset.metadata.gemini_version or "unknown") dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: return dataset.peek except Exception: - return "Gemini SQLite Database, version %s" % (dataset.metadata.gemini_version or 'unknown') + return "Gemini SQLite Database, version %s" % (dataset.metadata.gemini_version or "unknown") class ChiraSQLite(SQlite): - """Class describing a ChiRAViz Sqlite database """ + """Class describing a ChiRAViz Sqlite database""" + file_ext = "chira.sqlite" def set_meta(self, dataset, overwrite=True, **kwd): @@ -2048,18 +2390,28 @@ class ChiraSQLite(SQlite): def sniff(self, filename): if super().sniff(filename): - self.sniff_table_names(filename, ['Chimeras']) + self.sniff_table_names(filename, ["Chimeras"]) return False class CuffDiffSQlite(SQlite): - """Class describing a CuffDiff SQLite database """ - MetadataElement(name="cuffdiff_version", default='2.2.1', param=MetadataParameter, desc="CuffDiff Version", - readonly=True, visible=True, no_value='2.2.1') - MetadataElement(name="genes", default=[], param=MetadataParameter, desc="Genes", - readonly=True, visible=True, no_value=[]) - MetadataElement(name="samples", default=[], param=MetadataParameter, desc="Samples", - readonly=True, visible=True, no_value=[]) + """Class describing a CuffDiff SQLite database""" + + MetadataElement( + name="cuffdiff_version", + default="2.2.1", + param=MetadataParameter, + desc="CuffDiff Version", + readonly=True, + visible=True, + no_value="2.2.1", + ) + MetadataElement( + name="genes", default=[], param=MetadataParameter, desc="Genes", readonly=True, visible=True, no_value=[] + ) + MetadataElement( + name="samples", default=[], param=MetadataParameter, desc="Samples", readonly=True, visible=True, no_value=[] + ) file_ext = "cuffdiff.sqlite" # TODO: Update this when/if there is a specific EDAM format for CuffDiff SQLite data. edam_format = "format_3621" @@ -2073,50 +2425,51 @@ class CuffDiffSQlite(SQlite): c = conn.cursor() tables_query = "SELECT value FROM runInfo where param = 'version'" result = c.execute(tables_query).fetchall() - for version, in result: + for (version,) in result: dataset.metadata.cuffdiff_version = version - genes_query = 'SELECT gene_id, gene_short_name FROM genes ORDER BY gene_short_name' + genes_query = "SELECT gene_id, gene_short_name FROM genes ORDER BY gene_short_name" result = c.execute(genes_query).fetchall() for gene_id, gene_name in result: if gene_name is None: continue - gene = f'{gene_id}: {gene_name}' + gene = f"{gene_id}: {gene_name}" if gene not in genes: genes.append(gene) - samples_query = 'SELECT DISTINCT(sample_name) as sample_name FROM samples ORDER BY sample_name' + samples_query = "SELECT DISTINCT(sample_name) as sample_name FROM samples ORDER BY sample_name" result = c.execute(samples_query).fetchall() - for sample_name, in result: + for (sample_name,) in result: if sample_name not in samples: samples.append(sample_name) dataset.metadata.genes = genes dataset.metadata.samples = samples except Exception as e: - log.warning('%s, set_meta Exception: %s', self, e) + log.warning("%s, set_meta Exception: %s", self, e) def sniff(self, filename): if super().sniff(filename): # These tables should be in any CuffDiff SQLite output. - table_names = ['CDS', 'genes', 'isoforms', 'replicates', 'runInfo', 'samples', 'TSS'] + table_names = ["CDS", "genes", "isoforms", "replicates", "runInfo", "samples", "TSS"] return self.sniff_table_names(filename, table_names) return False def set_peek(self, dataset): if not dataset.dataset.purged: - dataset.peek = "CuffDiff SQLite Database, version %s" % (dataset.metadata.cuffdiff_version or 'unknown') + dataset.peek = "CuffDiff SQLite Database, version %s" % (dataset.metadata.cuffdiff_version or "unknown") dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: return dataset.peek except Exception: - return "CuffDiff SQLite Database, version %s" % (dataset.metadata.cuffdiff_version or 'unknown') + return "CuffDiff SQLite Database, version %s" % (dataset.metadata.cuffdiff_version or "unknown") class MzSQlite(SQlite): - """Class describing a Proteomics Sqlite database """ + """Class describing a Proteomics Sqlite database""" + file_ext = "mz.sqlite" def set_meta(self, dataset, overwrite=True, **kwd): @@ -2124,8 +2477,19 @@ class MzSQlite(SQlite): def sniff(self, filename): if super().sniff(filename): - table_names = ["DBSequence", "Modification", "Peaks", "Peptide", "PeptideEvidence", - "Score", "SearchDatabase", "Source", "SpectraData", "Spectrum", "SpectrumIdentification"] + table_names = [ + "DBSequence", + "Modification", + "Peaks", + "Peptide", + "PeptideEvidence", + "Score", + "SearchDatabase", + "Source", + "SpectraData", + "Spectrum", + "SpectrumIdentification", + ] return self.sniff_table_names(filename, table_names) return False @@ -2142,6 +2506,7 @@ class PQP(SQlite): >>> PQP().sniff(fname) False """ + file_ext = "pqp" def set_meta(self, dataset, overwrite=True, **kwd): @@ -2155,10 +2520,19 @@ class PQP(SQlite): """ if not super().sniff(filename): return False - table_names = ['COMPOUND', 'PEPTIDE', 'PEPTIDE_PROTEIN_MAPPING', 'PRECURSOR', - 'PRECURSOR_COMPOUND_MAPPING', 'PRECURSOR_PEPTIDE_MAPPING', 'PROTEIN', - 'TRANSITION', 'TRANSITION_PEPTIDE_MAPPING', 'TRANSITION_PRECURSOR_MAPPING'] - osw_table_names = ['FEATURE', 'FEATURE_MS1', 'FEATURE_MS2', 'FEATURE_TRANSITION', 'RUN'] + table_names = [ + "COMPOUND", + "PEPTIDE", + "PEPTIDE_PROTEIN_MAPPING", + "PRECURSOR", + "PRECURSOR_COMPOUND_MAPPING", + "PRECURSOR_PEPTIDE_MAPPING", + "PROTEIN", + "TRANSITION", + "TRANSITION_PEPTIDE_MAPPING", + "TRANSITION_PRECURSOR_MAPPING", + ] + osw_table_names = ["FEATURE", "FEATURE_MS1", "FEATURE_MS2", "FEATURE_TRANSITION", "RUN"] return self.sniff_table_names(filename, table_names) and not self.sniff_table_names(filename, osw_table_names) @@ -2174,6 +2548,7 @@ class OSW(SQlite): >>> OSW().sniff(fname) False """ + file_ext = "osw" def set_meta(self, dataset, overwrite=True, **kwd): @@ -2184,10 +2559,23 @@ class OSW(SQlite): # see also here https://github.com/OpenMS/OpenMS/issues/4365 if not super().sniff(filename): return False - table_names = ['COMPOUND', 'PEPTIDE', 'PEPTIDE_PROTEIN_MAPPING', 'PRECURSOR', - 'PRECURSOR_COMPOUND_MAPPING', 'PRECURSOR_PEPTIDE_MAPPING', 'PROTEIN', - 'TRANSITION', 'TRANSITION_PEPTIDE_MAPPING', 'TRANSITION_PRECURSOR_MAPPING', - 'FEATURE', 'FEATURE_MS1', 'FEATURE_MS2', 'FEATURE_TRANSITION', 'RUN'] + table_names = [ + "COMPOUND", + "PEPTIDE", + "PEPTIDE_PROTEIN_MAPPING", + "PRECURSOR", + "PRECURSOR_COMPOUND_MAPPING", + "PRECURSOR_PEPTIDE_MAPPING", + "PROTEIN", + "TRANSITION", + "TRANSITION_PEPTIDE_MAPPING", + "TRANSITION_PRECURSOR_MAPPING", + "FEATURE", + "FEATURE_MS1", + "FEATURE_MS2", + "FEATURE_TRANSITION", + "RUN", + ] return self.sniff_table_names(filename, table_names) @@ -2203,6 +2591,7 @@ class SQmass(SQlite): >>> SQmass().sniff(fname) False """ + file_ext = "sqmass" def set_meta(self, dataset, overwrite=True, **kwd): @@ -2216,9 +2605,17 @@ class SQmass(SQlite): class BlibSQlite(SQlite): - """Class describing a Proteomics Spectral Library Sqlite database """ - MetadataElement(name="blib_version", default='1.8', param=MetadataParameter, desc="Blib Version", - readonly=True, visible=True, no_value='1.8') + """Class describing a Proteomics Spectral Library Sqlite database""" + + MetadataElement( + name="blib_version", + default="1.8", + param=MetadataParameter, + desc="Blib Version", + readonly=True, + visible=True, + no_value="1.8", + ) file_ext = "blib" def set_meta(self, dataset, overwrite=True, **kwd): @@ -2228,14 +2625,22 @@ class BlibSQlite(SQlite): c = conn.cursor() tables_query = "SELECT majorVersion,minorVersion FROM LibInfo" (majorVersion, minorVersion) = c.execute(tables_query).fetchall()[0] - dataset.metadata.blib_version = f'{majorVersion}.{minorVersion}' + dataset.metadata.blib_version = f"{majorVersion}.{minorVersion}" except Exception as e: - log.warning('%s, set_meta Exception: %s', self, e) + log.warning("%s, set_meta Exception: %s", self, e) def sniff(self, filename): if super().sniff(filename): - table_names = ['IonMobilityTypes', 'LibInfo', 'Modifications', 'RefSpectra', - 'RefSpectraPeakAnnotations', 'RefSpectraPeaks', 'ScoreTypes', 'SpectrumSourceFiles'] + table_names = [ + "IonMobilityTypes", + "LibInfo", + "Modifications", + "RefSpectra", + "RefSpectraPeakAnnotations", + "RefSpectraPeaks", + "ScoreTypes", + "SpectrumSourceFiles", + ] return self.sniff_table_names(filename, table_names) return False @@ -2254,8 +2659,16 @@ class DlibSQlite(SQlite): >>> DlibSQlite().sniff(fname) False """ - MetadataElement(name="dlib_version", default='1.8', param=MetadataParameter, desc="Dlib Version", - readonly=True, visible=True, no_value='1.8') + + MetadataElement( + name="dlib_version", + default="1.8", + param=MetadataParameter, + desc="Dlib Version", + readonly=True, + visible=True, + no_value="1.8", + ) file_ext = "dlib" def set_meta(self, dataset, overwrite=True, **kwd): @@ -2265,13 +2678,13 @@ class DlibSQlite(SQlite): c = conn.cursor() tables_query = "SELECT Value FROM metadata WHERE Key = 'version'" version = c.execute(tables_query).fetchall()[0] - dataset.metadata.dlib_version = f'{version}' + dataset.metadata.dlib_version = f"{version}" except Exception as e: - log.warning('%s, set_meta Exception: %s', self, e) + log.warning("%s, set_meta Exception: %s", self, e) def sniff(self, filename): if super().sniff(filename): - table_names = ['entries', 'metadata', 'peptidetoprotein'] + table_names = ["entries", "metadata", "peptidetoprotein"] return self.sniff_table_names(filename, table_names) return False @@ -2290,8 +2703,16 @@ class ElibSQlite(SQlite): >>> ElibSQlite().sniff(fname) False """ - MetadataElement(name="version", default='0.1.14', param=MetadataParameter, desc="Elib Version", - readonly=True, visible=True, no_value='0.1.14') + + MetadataElement( + name="version", + default="0.1.14", + param=MetadataParameter, + desc="Elib Version", + readonly=True, + visible=True, + no_value="0.1.14", + ) file_ext = "elib" def set_meta(self, dataset, overwrite=True, **kwd): @@ -2301,14 +2722,23 @@ class ElibSQlite(SQlite): c = conn.cursor() tables_query = "SELECT Value FROM metadata WHERE Key = 'version'" version = c.execute(tables_query).fetchall()[0] - dataset.metadata.dlib_version = f'{version}' + dataset.metadata.dlib_version = f"{version}" except Exception as e: - log.warning('%s, set_meta Exception: %s', self, e) + log.warning("%s, set_meta Exception: %s", self, e) def sniff(self, filename): if super().sniff(filename): - table_names = ['entries', 'fragmentquants', 'metadata', 'peptidelocalizations', 'peptidequants', - 'peptidescores', 'peptidetoprotein', 'proteinscores', 'retentiontimes'] + table_names = [ + "entries", + "fragmentquants", + "metadata", + "peptidelocalizations", + "peptidequants", + "peptidescores", + "peptidetoprotein", + "proteinscores", + "retentiontimes", + ] if self.sniff_table_names(filename, table_names): try: conn = sqlite.connect(filename) @@ -2317,7 +2747,7 @@ class ElibSQlite(SQlite): count = c.execute(row_query).fetchone()[0] return int(count) > 0 except Exception as e: - log.warning('%s, sniff Exception: %s', self, e) + log.warning("%s, sniff Exception: %s", self, e) return False @@ -2333,6 +2763,7 @@ class IdpDB(SQlite): >>> IdpDB().sniff(fname) False """ + file_ext = "idpdb" def set_meta(self, dataset, overwrite=True, **kwd): @@ -2340,8 +2771,14 @@ class IdpDB(SQlite): def sniff(self, filename): if super().sniff(filename): - table_names = ["About", "Analysis", "AnalysisParameter", "PeptideSpectrumMatch", - "Spectrum", "SpectrumSource"] + table_names = [ + "About", + "Analysis", + "AnalysisParameter", + "PeptideSpectrumMatch", + "Spectrum", + "SpectrumSource", + ] return self.sniff_table_names(filename, table_names) return False @@ -2350,8 +2787,8 @@ class IdpDB(SQlite): dataset.peek = "IDPickerDB SQLite file" dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -2362,62 +2799,85 @@ class IdpDB(SQlite): class GAFASQLite(SQlite): """Class describing a GAFA SQLite database""" - MetadataElement(name='gafa_schema_version', default='0.3.0', param=MetadataParameter, desc='GAFA schema version', - readonly=True, visible=True, no_value='0.3.0') - file_ext = 'gafa.sqlite' + + MetadataElement( + name="gafa_schema_version", + default="0.3.0", + param=MetadataParameter, + desc="GAFA schema version", + readonly=True, + visible=True, + no_value="0.3.0", + ) + file_ext = "gafa.sqlite" def set_meta(self, dataset, overwrite=True, **kwd): super().set_meta(dataset, overwrite=overwrite, **kwd) try: conn = sqlite.connect(dataset.file_name) c = conn.cursor() - version_query = 'SELECT version FROM meta' + version_query = "SELECT version FROM meta" results = c.execute(version_query).fetchall() if len(results) == 0: - raise Exception('version not found in meta table') + raise Exception("version not found in meta table") elif len(results) > 1: - raise Exception('Multiple versions found in meta table') + raise Exception("Multiple versions found in meta table") dataset.metadata.gafa_schema_version = results[0][0] except Exception as e: log.warning("%s, set_meta Exception: %s", self, e) def sniff(self, filename): if super().sniff(filename): - table_names = frozenset({'gene', 'gene_family', 'gene_family_member', 'meta', 'transcript'}) + table_names = frozenset({"gene", "gene_family", "gene_family_member", "meta", "transcript"}) return self.sniff_table_names(filename, table_names) return False class NcbiTaxonomySQlite(SQlite): """Class describing the NCBI Taxonomy database stored in SQLite as done by rust-ncbitaxonomy""" - MetadataElement(name='ncbitaxonomy_schema_version', default='20200501095116', param=MetadataParameter, desc='ncbitaxonomy schema version', - readonly=True, visible=True, no_value='20200501095116') - MetadataElement(name="taxon_count", default=[], param=MetadataParameter, desc="Count of taxa in the taxonomy", - readonly=True, visible=True, no_value=[]) - file_ext = 'ncbitaxonomy.sqlite' + MetadataElement( + name="ncbitaxonomy_schema_version", + default="20200501095116", + param=MetadataParameter, + desc="ncbitaxonomy schema version", + readonly=True, + visible=True, + no_value="20200501095116", + ) + MetadataElement( + name="taxon_count", + default=[], + param=MetadataParameter, + desc="Count of taxa in the taxonomy", + readonly=True, + visible=True, + no_value=[], + ) + + file_ext = "ncbitaxonomy.sqlite" def set_meta(self, dataset, overwrite=True, **kwd): super().set_meta(dataset, overwrite=overwrite, **kwd) try: conn = sqlite.connect(dataset.file_name) c = conn.cursor() - version_query = 'SELECT version FROM __diesel_schema_migrations ORDER BY run_on DESC LIMIT 1' + version_query = "SELECT version FROM __diesel_schema_migrations ORDER BY run_on DESC LIMIT 1" results = c.execute(version_query).fetchall() if len(results) == 0: - raise Exception('version not found in __diesel_schema_migrations table') + raise Exception("version not found in __diesel_schema_migrations table") dataset.metadata.ncbitaxonomy_schema_version = results[0][0] - taxons_query = 'SELECT count(name) FROM taxonomy' + taxons_query = "SELECT count(name) FROM taxonomy" results = c.execute(taxons_query).fetchall() if len(results) == 0: - raise Exception('could not count size of taxonomy table') + raise Exception("could not count size of taxonomy table") dataset.metadata.taxon_count = results[0][0] except Exception as e: log.warning("%s, set_meta Exception: %s", self, e) def sniff(self, filename): if super().sniff(filename): - table_names = frozenset({'__diesel_schema_migrations', 'taxonomy'}) + table_names = frozenset({"__diesel_schema_migrations", "taxonomy"}) return self.sniff_table_names(filename, table_names) return False @@ -2425,12 +2885,12 @@ class NcbiTaxonomySQlite(SQlite): if not dataset.dataset.purged: dataset.peek = "NCBI Taxonomy SQLite Database, version {} ({} taxons)".format( getattr(dataset.metadata, "ncbitaxonomy_schema_version", "unknown"), - getattr(dataset.metadata, "taxon_count", "unknown") + getattr(dataset.metadata, "taxon_count", "unknown"), ) dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -2438,12 +2898,13 @@ class NcbiTaxonomySQlite(SQlite): except Exception: return "NCBI Taxonomy SQLite Database, version {} ({} taxons)".format( getattr(dataset.metadata, "ncbitaxonomy_schema_version", "unknown"), - getattr(dataset.metadata, "taxon_count", "unknown") + getattr(dataset.metadata, "taxon_count", "unknown"), ) class Xlsx(Binary): """Class for Excel 2007 (xlsx) files""" + file_ext = "xlsx" compressed = True @@ -2452,7 +2913,13 @@ class Xlsx(Binary): try: if zipfile.is_zipfile(filename): tempzip = zipfile.ZipFile(filename) - if "[Content_Types].xml" in tempzip.namelist() and tempzip.read("[Content_Types].xml").find(b'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml') != -1: + if ( + "[Content_Types].xml" in tempzip.namelist() + and tempzip.read("[Content_Types].xml").find( + b"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml" + ) + != -1 + ): return True return False except Exception: @@ -2461,24 +2928,25 @@ class Xlsx(Binary): class ExcelXls(Binary): """Class describing an Excel (xls) file""" + file_ext = "excel.xls" edam_format = "format_3468" def sniff(self, filename): - mime_type = subprocess.check_output(['file', '--mime-type', filename]) + mime_type = subprocess.check_output(["file", "--mime-type", filename]) return b"application/vnd.ms-excel" in mime_type def get_mime(self): """Returns the mime type of the datatype""" - return 'application/vnd.ms-excel' + return "application/vnd.ms-excel" def set_peek(self, dataset): if not dataset.dataset.purged: dataset.peek = "Microsoft Excel XLS file" dataset.blurb = data.nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -2489,28 +2957,29 @@ class ExcelXls(Binary): @build_sniff_from_prefix class Sra(Binary): - """ Sequence Read Archive (SRA) datatype originally from mdshw5/sra-tools-galaxy""" - file_ext = 'sra' + """Sequence Read Archive (SRA) datatype originally from mdshw5/sra-tools-galaxy""" + + file_ext = "sra" def sniff_prefix(self, sniff_prefix): - """ The first 8 bytes of any NCBI sra file is 'NCBI.sra', and the file is binary. + """The first 8 bytes of any NCBI sra file is 'NCBI.sra', and the file is binary. For details about the format, see http://www.ncbi.nlm.nih.gov/books/n/helpsra/SRA_Overview_BK/#SRA_Overview_BK.4_SRA_Data_Structure """ - return sniff_prefix.startswith_bytes(b'NCBI.sra') + return sniff_prefix.startswith_bytes(b"NCBI.sra") def set_peek(self, dataset): if not dataset.dataset.purged: - dataset.peek = 'Binary sra file' + dataset.peek = "Binary sra file" dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: return dataset.peek except Exception: - return f'Binary sra file ({nice_size(dataset.get_size())})' + return f"Binary sra file ({nice_size(dataset.get_size())})" @build_sniff_from_prefix @@ -2532,11 +3001,20 @@ class RData(CompressedArchive): >>> dataset.metadata.version '3' """ - VERSION_2_PREFIX = b'RDX2\nX\n' - VERSION_3_PREFIX = b'RDX3\nX\n' - file_ext = 'rdata' - MetadataElement(name="version", default=None, desc="serialisation version", param=MetadataParameter, readonly=True, visible=False, optional=False) + VERSION_2_PREFIX = b"RDX2\nX\n" + VERSION_3_PREFIX = b"RDX3\nX\n" + file_ext = "rdata" + + MetadataElement( + name="version", + default=None, + desc="serialisation version", + param=MetadataParameter, + readonly=True, + visible=False, + optional=False, + ) def set_meta(self, dataset, overwrite=True, **kwd): super().set_meta(dataset, overwrite=overwrite, **kwd) @@ -2590,17 +3068,47 @@ class RDS(CompressedArchive): >>> dataset.metadata.minrversion '3.5.0' """ - file_ext = 'rds' - MetadataElement(name="version", default=None, desc="serialisation version", param=MetadataParameter, readonly=True, visible=False, optional=False) - MetadataElement(name="rversion", default=None, desc="R version", param=MetadataParameter, readonly=True, visible=False, optional=False) - MetadataElement(name="minrversion", default=None, desc="minimum R version", param=MetadataParameter, readonly=False, visible=True, optional=False) + file_ext = "rds" + + MetadataElement( + name="version", + default=None, + desc="serialisation version", + param=MetadataParameter, + readonly=True, + visible=False, + optional=False, + ) + MetadataElement( + name="rversion", + default=None, + desc="R version", + param=MetadataParameter, + readonly=True, + visible=False, + optional=False, + ) + MetadataElement( + name="minrversion", + default=None, + desc="minimum R version", + param=MetadataParameter, + readonly=False, + visible=True, + optional=False, + ) def set_meta(self, dataset, overwrite=True, **kwd): super().set_meta(dataset, overwrite=overwrite, **kwd) _, fh = compression_utils.get_fileobj_raw(dataset.file_name, "rb") try: - _, dataset.metadata.version, dataset.metadata.rversion, dataset.metadata.minrversion = self._parse_rds_header(fh.read(14)) + ( + _, + dataset.metadata.version, + dataset.metadata.rversion, + dataset.metadata.minrversion, + ) = self._parse_rds_header(fh.read(14)) except Exception: pass finally: @@ -2623,9 +3131,9 @@ class RDS(CompressedArchive): - the minimum r version needed to read the file """ header = header_bytes[:2] - if header == b'X\n': + if header == b"X\n": mode = "X" - elif header == b'A\n': + elif header == b"A\n": mode = "A" else: raise Exception() @@ -2644,13 +3152,12 @@ class RDS(CompressedArchive): class OxliBinary(Binary): - @staticmethod def _sniff(filename, oxlitype): try: - with open(filename, 'rb') as fileobj: + with open(filename, "rb") as fileobj: header = fileobj.read(4) - if header == b'OXLI': + if header == b"OXLI": fileobj.read(1) # skip the version number ftype = fileobj.read(1) if binascii.hexlify(ftype) == oxlitype: @@ -2679,7 +3186,8 @@ class OxliCountGraph(OxliBinary): >>> OxliCountGraph().sniff(fname) True """ - file_ext = 'oxlicg' + + file_ext = "oxlicg" def sniff(self, filename): return OxliBinary._sniff(filename, b"01") @@ -2704,7 +3212,8 @@ class OxliNodeGraph(OxliBinary): >>> OxliNodeGraph().sniff(fname) True """ - file_ext = 'oxling' + + file_ext = "oxling" def sniff(self, filename): return OxliBinary._sniff(filename, b"02") @@ -2730,7 +3239,8 @@ class OxliTagSet(OxliBinary): >>> OxliTagSet().sniff(fname) True """ - file_ext = 'oxlits' + + file_ext = "oxlits" def sniff(self, filename): return OxliBinary._sniff(filename, b"03") @@ -2751,7 +3261,8 @@ class OxliStopTags(OxliBinary): >>> OxliStopTags().sniff(fname) True """ - file_ext = 'oxlist' + + file_ext = "oxlist" def sniff(self, filename): return OxliBinary._sniff(filename, b"04") @@ -2777,7 +3288,8 @@ class OxliSubset(OxliBinary): >>> OxliSubset().sniff(fname) True """ - file_ext = 'oxliss' + + file_ext = "oxliss" def sniff(self, filename): return OxliBinary._sniff(filename, b"05") @@ -2804,7 +3316,8 @@ class OxliGraphLabels(OxliBinary): >>> OxliGraphLabels().sniff(fname) True """ - file_ext = 'oxligl' + + file_ext = "oxligl" def sniff(self, filename): return OxliBinary._sniff(filename, b"06") @@ -2822,33 +3335,40 @@ class PostgresqlArchive(CompressedArchive): >>> PostgresqlArchive().sniff(fname) False """ - MetadataElement(name="version", default=None, param=MetadataParameter, desc="PostgreSQL database version", - readonly=True, visible=True) + + MetadataElement( + name="version", + default=None, + param=MetadataParameter, + desc="PostgreSQL database version", + readonly=True, + visible=True, + ) file_ext = "postgresql" def set_meta(self, dataset, overwrite=True, **kwd): super().set_meta(dataset, overwrite=overwrite, **kwd) try: if dataset and tarfile.is_tarfile(dataset.file_name): - with tarfile.open(dataset.file_name, 'r') as temptar: - pg_version_file = temptar.extractfile('postgresql/db/PG_VERSION') + with tarfile.open(dataset.file_name, "r") as temptar: + pg_version_file = temptar.extractfile("postgresql/db/PG_VERSION") dataset.metadata.version = util.unicodify(pg_version_file.read()).strip() except Exception as e: - log.warning('%s, set_meta Exception: %s', self, util.unicodify(e)) + log.warning("%s, set_meta Exception: %s", self, util.unicodify(e)) def sniff(self, filename): if filename and tarfile.is_tarfile(filename): - with tarfile.open(filename, 'r') as temptar: - return 'postgresql/db/PG_VERSION' in temptar.getnames() + with tarfile.open(filename, "r") as temptar: + return "postgresql/db/PG_VERSION" in temptar.getnames() return False def set_peek(self, dataset): if not dataset.dataset.purged: dataset.peek = f"PostgreSQL Archive ({nice_size(dataset.get_size())})" - dataset.blurb = "PostgreSQL version %s" % (dataset.metadata.version or 'unknown') + dataset.blurb = "PostgreSQL version %s" % (dataset.metadata.version or "unknown") else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -2866,43 +3386,43 @@ class Fast5Archive(CompressedArchive): >>> Fast5Archive().sniff(fname) True """ - MetadataElement(name="fast5_count", default='0', param=MetadataParameter, desc="Read Count", - readonly=True, visible=True) + + MetadataElement( + name="fast5_count", default="0", param=MetadataParameter, desc="Read Count", readonly=True, visible=True + ) file_ext = "fast5.tar" def set_meta(self, dataset, overwrite=True, **kwd): super().set_meta(dataset, overwrite=overwrite, **kwd) try: if dataset and tarfile.is_tarfile(dataset.file_name): - with tarfile.open(dataset.file_name, 'r') as temptar: - dataset.metadata.fast5_count = sum( - 1 for f in temptar if f.name.endswith('.fast5') - ) + with tarfile.open(dataset.file_name, "r") as temptar: + dataset.metadata.fast5_count = sum(1 for f in temptar if f.name.endswith(".fast5")) except Exception as e: - log.warning('%s, set_meta Exception: %s', self, e) + log.warning("%s, set_meta Exception: %s", self, e) def sniff(self, filename): try: if filename and tarfile.is_tarfile(filename): - with tarfile.open(filename, 'r') as temptar: + with tarfile.open(filename, "r") as temptar: for f in temptar: if not f.isfile(): continue - if f.name.endswith('.fast5'): + if f.name.endswith(".fast5"): return True else: return False except Exception as e: - log.warning('%s, sniff Exception: %s', self, e) + log.warning("%s, sniff Exception: %s", self, e) return False def set_peek(self, dataset): if not dataset.dataset.purged: dataset.peek = f"FAST5 Archive ({nice_size(dataset.get_size())})" - dataset.blurb = "%s sequences" % (dataset.metadata.fast5_count or 'unknown') + dataset.blurb = "%s sequences" % (dataset.metadata.fast5_count or "unknown") else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -2926,6 +3446,7 @@ class Fast5ArchiveGz(Fast5Archive): >>> Fast5ArchiveGz().sniff(fname) False """ + file_ext = "fast5.tar.gz" def sniff(self, filename): @@ -2949,6 +3470,7 @@ class Fast5ArchiveBz2(Fast5Archive): >>> Fast5ArchiveBz2().sniff(fname) False """ + file_ext = "fast5.tar.bz2" def sniff(self, filename): @@ -2958,11 +3480,24 @@ class Fast5ArchiveBz2(Fast5Archive): class SearchGuiArchive(CompressedArchive): - """Class describing a SearchGUI archive """ - MetadataElement(name="searchgui_version", default='1.28.0', param=MetadataParameter, desc="SearchGui Version", - readonly=True, visible=True) - MetadataElement(name="searchgui_major_version", default='1', param=MetadataParameter, desc="SearchGui Major Version", - readonly=True, visible=True) + """Class describing a SearchGUI archive""" + + MetadataElement( + name="searchgui_version", + default="1.28.0", + param=MetadataParameter, + desc="SearchGui Version", + readonly=True, + visible=True, + ) + MetadataElement( + name="searchgui_major_version", + default="1", + param=MetadataParameter, + desc="SearchGui Major Version", + readonly=True, + visible=True, + ) file_ext = "searchgui_archive" def set_meta(self, dataset, overwrite=True, **kwd): @@ -2970,44 +3505,45 @@ class SearchGuiArchive(CompressedArchive): try: if dataset and zipfile.is_zipfile(dataset.file_name): with zipfile.ZipFile(dataset.file_name) as tempzip: - if 'searchgui.properties' in tempzip.namelist(): - with tempzip.open('searchgui.properties') as fh: + if "searchgui.properties" in tempzip.namelist(): + with tempzip.open("searchgui.properties") as fh: for line in io.TextIOWrapper(fh): - if line.startswith('searchgui.version'): - version = line.split('=')[1].strip() + if line.startswith("searchgui.version"): + version = line.split("=")[1].strip() dataset.metadata.searchgui_version = version - dataset.metadata.searchgui_major_version = version.split('.')[0] + dataset.metadata.searchgui_major_version = version.split(".")[0] except Exception as e: - log.warning('%s, set_meta Exception: %s', self, e) + log.warning("%s, set_meta Exception: %s", self, e) def sniff(self, filename): try: if filename and zipfile.is_zipfile(filename): - with zipfile.ZipFile(filename, 'r') as tempzip: - is_searchgui = 'searchgui.properties' in tempzip.namelist() + with zipfile.ZipFile(filename, "r") as tempzip: + is_searchgui = "searchgui.properties" in tempzip.namelist() return is_searchgui except Exception as e: - log.warning('%s, sniff Exception: %s', self, e) + log.warning("%s, sniff Exception: %s", self, e) return False def set_peek(self, dataset): if not dataset.dataset.purged: - dataset.peek = "SearchGUI Archive, version %s" % (dataset.metadata.searchgui_version or 'unknown') + dataset.peek = "SearchGUI Archive, version %s" % (dataset.metadata.searchgui_version or "unknown") dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: return dataset.peek except Exception: - return "SearchGUI Archive, version %s" % (dataset.metadata.searchgui_version or 'unknown') + return "SearchGUI Archive, version %s" % (dataset.metadata.searchgui_version or "unknown") @build_sniff_from_prefix class NetCDF(Binary): """Binary data in netCDF format""" + file_ext = "netcdf" edam_format = "format_3650" edam_data = "data_0943" @@ -3017,8 +3553,8 @@ class NetCDF(Binary): dataset.peek = "Binary netCDF file" dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -3027,7 +3563,7 @@ class NetCDF(Binary): return f"Binary netCDF file ({nice_size(dataset.get_size())})" def sniff_prefix(self, sniff_prefix): - return sniff_prefix.startswith_bytes(b'CDF') + return sniff_prefix.startswith_bytes(b"CDF") class Dcd(Binary): @@ -3042,18 +3578,19 @@ class Dcd(Binary): >>> Dcd().sniff(fname) False """ + file_ext = "dcd" edam_data = "data_3842" def __init__(self, **kwd): super().__init__(**kwd) - self._magic_number = b'CORD' + self._magic_number = b"CORD" def sniff(self, filename): # Match the keyword 'CORD' at position 4 or 8 - intsize dependent # Not checking for endianness try: - with open(filename, 'rb') as header: + with open(filename, "rb") as header: intsize = 4 header.seek(intsize) if header.read(intsize) == self._magic_number: @@ -3072,8 +3609,8 @@ class Dcd(Binary): dataset.peek = "Binary CHARMM/NAMD dcd file" dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -3094,17 +3631,18 @@ class Vel(Binary): >>> Vel().sniff(fname) False """ + file_ext = "vel" def __init__(self, **kwd): super().__init__(**kwd) - self._magic_number = b'VELD' + self._magic_number = b"VELD" def sniff(self, filename): # Match the keyword 'VELD' at position 4 or 8 - intsize dependent # Not checking for endianness try: - with open(filename, 'rb') as header: + with open(filename, "rb") as header: intsize = 4 header.seek(intsize) if header.read(intsize) == self._magic_number: @@ -3123,8 +3661,8 @@ class Vel(Binary): dataset.peek = "Binary CHARMM velocity file" dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -3145,6 +3683,7 @@ class DAA(Binary): >>> DAA().sniff(fname) False """ + file_ext = "daa" def __init__(self, **kwd): @@ -3168,6 +3707,7 @@ class RMA6(Binary): >>> RMA6().sniff(fname) False """ + file_ext = "rma6" def __init__(self, **kwd): @@ -3190,6 +3730,7 @@ class DMND(Binary): >>> DMND().sniff(fname) False """ + file_ext = "dmnd" def __init__(self, **kwd): @@ -3205,6 +3746,7 @@ class ICM(Binary): """ Class describing an ICM (interpolated context model) file, used by Glimmer """ + file_ext = "icm" edam_data = "data_0950" @@ -3213,12 +3755,18 @@ class ICM(Binary): dataset.peek = "Binary ICM (interpolated context model) file" dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def sniff(self, dataset): line = open(dataset).read(100) - if '>ver = ' in line and 'len = ' in line and 'depth = ' in line and 'periodicity =' in line and 'nodes = ' in line: + if ( + ">ver = " in line + and "len = " in line + and "depth = " in line + and "periodicity =" in line + and "nodes = " in line + ): return True return False @@ -3236,6 +3784,7 @@ class Parquet(Binary): >>> Parquet().sniff(fname) False """ + file_ext = "parquet" def __init__(self, **kwd): @@ -3257,6 +3806,7 @@ class BafTar(CompressedArchive): >>> BafTar().sniff(fname) False """ + edam_data = "data_2536" # mass spectrometry data edam_format = "format_3712" # TODO: add more raw formats to EDAM? file_ext = "brukerbaf.d.tar" @@ -3278,8 +3828,8 @@ class BafTar(CompressedArchive): dataset.peek = self.get_type() dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -3289,7 +3839,8 @@ class BafTar(CompressedArchive): class YepTar(BafTar): - """ A tar'd up .d directory containing Agilent/Bruker YEP format data """ + """A tar'd up .d directory containing Agilent/Bruker YEP format data""" + file_ext = "agilentbrukeryep.d.tar" def get_signature_file(self): @@ -3300,7 +3851,8 @@ class YepTar(BafTar): class TdfTar(BafTar): - """ A tar'd up .d directory containing Bruker TDF format data """ + """A tar'd up .d directory containing Bruker TDF format data""" + file_ext = "brukertdf.d.tar" def get_signature_file(self): @@ -3311,7 +3863,8 @@ class TdfTar(BafTar): class MassHunterTar(BafTar): - """ A tar'd up .d directory containing Agilent MassHunter format data """ + """A tar'd up .d directory containing Agilent MassHunter format data""" + file_ext = "agilentmasshunter.d.tar" def get_signature_file(self): @@ -3322,7 +3875,8 @@ class MassHunterTar(BafTar): class MassLynxTar(BafTar): - """ A tar'd up .d directory containing Waters MassLynx format data """ + """A tar'd up .d directory containing Waters MassLynx format data""" + file_ext = "watersmasslynx.raw.tar" def get_signature_file(self): @@ -3346,6 +3900,7 @@ class WiffTar(BafTar): >>> WiffTar().sniff(fname) False """ + file_ext = "wiff.tar" def sniff(self, filename): @@ -3368,20 +3923,21 @@ class Pretext(Binary): >>> Pretext().sniff(fname) True """ + file_ext = "pretext" def sniff_prefix(self, sniff_prefix): # The first 4 bytes of any pretext file is 'pstm', and the rest of the # file contains binary data. - return sniff_prefix.startswith_bytes(b'pstm') + return sniff_prefix.startswith_bytes(b"pstm") def set_peek(self, dataset): if not dataset.dataset.purged: dataset.peek = "Binary pretext file" dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -3401,6 +3957,7 @@ class JP2(Binary): >>> JP2().sniff(fname) False """ + file_ext = "jp2" def __init__(self, **kwd): @@ -3410,7 +3967,7 @@ class JP2(Binary): def sniff(self, filename): # The first 12 bytes of any jp2 file are 0000000C6A5020200D0A870A try: - header = open(filename, 'rb').read(12) + header = open(filename, "rb").read(12) if header == self._magic: return True return False @@ -3422,8 +3979,8 @@ class JP2(Binary): dataset.peek = "Binary JPEG 2000 file" dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -3444,6 +4001,7 @@ class Npz(CompressedArchive): >>> Npz().sniff(fname) False """ + file_ext = "npz" # edam_format = "format_4003" @@ -3470,15 +4028,15 @@ class Npz(CompressedArchive): dataset.metadata.nfiles = len(npz.files) dataset.metadata.files = npz.files except Exception as e: - log.warning('%s, set_meta Exception: %s', self, e) + log.warning("%s, set_meta Exception: %s", self, e) def set_peek(self, dataset): if not dataset.dataset.purged: dataset.peek = f"Binary Numpy npz {dataset.metadata.nfiles} files ({nice_size(dataset.get_size())})" dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -3499,10 +4057,22 @@ class HexrdImagesNpz(Npz): >>> HexrdImagesNpz().sniff(fname) False """ + file_ext = "hexrd.images.npz" - MetadataElement(name="panel_id", default='', desc="Detector Panel ID", param=MetadataParameter, readonly=True, visible=True, optional=True, no_value='') - MetadataElement(name="shape", default=(), desc="shape", param=metadata.ListParameter, readonly=True, visible=True, no_value=()) + MetadataElement( + name="panel_id", + default="", + desc="Detector Panel ID", + param=MetadataParameter, + readonly=True, + visible=True, + optional=True, + no_value="", + ) + MetadataElement( + name="shape", default=(), desc="shape", param=metadata.ListParameter, readonly=True, visible=True, no_value=() + ) MetadataElement(name="nframes", default=0, desc="nframes", readonly=True, visible=True, no_value=0) MetadataElement(name="omegas", desc="has omegas", default="False", visible=False) @@ -3512,11 +4082,11 @@ class HexrdImagesNpz(Npz): def sniff(self, filename): if super().sniff(filename): try: - req_files = {'0_row', '0_col', '0_data', 'shape', 'nframes', 'dtype'} + req_files = {"0_row", "0_col", "0_data", "shape", "nframes", "dtype"} with np.load(filename) as npz: return set(npz.files) >= req_files except Exception as e: - log.warning('%s, sniff Exception: %s', self, e) + log.warning("%s, sniff Exception: %s", self, e) return False return False @@ -3524,24 +4094,26 @@ class HexrdImagesNpz(Npz): super().set_meta(dataset, **kwd) try: with np.load(dataset.file_name) as npz: - if 'panel_id' in npz.files: - dataset.metadata.panel_id = str(npz['panel_id']) - if 'omega' in npz.files: + if "panel_id" in npz.files: + dataset.metadata.panel_id = str(npz["panel_id"]) + if "omega" in npz.files: dataset.metadata.omegas = "True" - dataset.metadata.shape = npz['shape'].tolist() - dataset.metadata.nframes = npz['nframes'].tolist() + dataset.metadata.shape = npz["shape"].tolist() + dataset.metadata.nframes = npz["nframes"].tolist() except Exception as e: - log.warning('%s, set_meta Exception: %s', self, e) + log.warning("%s, set_meta Exception: %s", self, e) def set_peek(self, dataset): if not dataset.dataset.purged: - lines = [f"Binary Hexrd Image npz {dataset.metadata.nfiles} files ({nice_size(dataset.get_size())})", - f"Panel: {dataset.metadata.panel_id} Frames: {dataset.metadata.nframes} Shape: {dataset.metadata.shape}"] - dataset.peek = '\n'.join(lines) + lines = [ + f"Binary Hexrd Image npz {dataset.metadata.nfiles} files ({nice_size(dataset.get_size())})", + f"Panel: {dataset.metadata.panel_id} Frames: {dataset.metadata.nframes} Shape: {dataset.metadata.shape}", + ] + dataset.peek = "\n".join(lines) dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -3562,9 +4134,12 @@ class HexrdEtaOmeNpz(Npz): >>> HexrdEtaOmeNpz().sniff(fname) False """ + file_ext = "hexrd.eta_ome.npz" - MetadataElement(name="HKLs", default=(), desc="HKLs", param=metadata.ListParameter, readonly=True, visible=True, no_value=()) + MetadataElement( + name="HKLs", default=(), desc="HKLs", param=metadata.ListParameter, readonly=True, visible=True, no_value=() + ) MetadataElement(name="nframes", default=0, desc="nframes", readonly=True, visible=True, no_value=0) def __init__(self, **kwd): @@ -3573,11 +4148,11 @@ class HexrdEtaOmeNpz(Npz): def sniff(self, filename): if super().sniff(filename): try: - req_files = {'dataStore', 'etas', 'etaEdges', 'iHKLList', 'omegas', 'omeEdges', 'planeData_hkls'} + req_files = {"dataStore", "etas", "etaEdges", "iHKLList", "omegas", "omeEdges", "planeData_hkls"} with np.load(filename) as npz: return set(npz.files) >= req_files except Exception as e: - log.warning('%s, sniff Exception: %s', self, e) + log.warning("%s, sniff Exception: %s", self, e) return False return False @@ -3585,20 +4160,22 @@ class HexrdEtaOmeNpz(Npz): super().set_meta(dataset, **kwd) try: with np.load(dataset.file_name) as npz: - dataset.metadata.HKLs = npz['iHKLList'].tolist() - dataset.metadata.nframes = len(npz['omegas']) + dataset.metadata.HKLs = npz["iHKLList"].tolist() + dataset.metadata.nframes = len(npz["omegas"]) except Exception as e: - log.warning('%s, set_meta Exception: %s', self, e) + log.warning("%s, set_meta Exception: %s", self, e) def set_peek(self, dataset): if not dataset.dataset.purged: - lines = [f"Binary Hexrd Eta-Ome npz {dataset.metadata.nfiles} files ({nice_size(dataset.get_size())})", - f"Eta-Ome HKLs: {dataset.metadata.HKLs} Frames: {dataset.metadata.nframes}"] - dataset.peek = '\n'.join(lines) + lines = [ + f"Binary Hexrd Eta-Ome npz {dataset.metadata.nfiles} files ({nice_size(dataset.get_size())})", + f"Eta-Ome HKLs: {dataset.metadata.HKLs} Frames: {dataset.metadata.nframes}", + ] + dataset.peek = "\n".join(lines) dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -3607,6 +4184,7 @@ class HexrdEtaOmeNpz(Npz): return "Binary Numpy npz file (%s)" % (nice_size(dataset.get_size())) -if __name__ == '__main__': +if __name__ == "__main__": import doctest + doctest.testmod(sys.modules[__name__]) diff --git a/lib/galaxy/datatypes/blast.py b/lib/galaxy/datatypes/blast.py index 7d03e000532..31bfbd0b81b 100644 --- a/lib/galaxy/datatypes/blast.py +++ b/lib/galaxy/datatypes/blast.py @@ -42,7 +42,7 @@ from galaxy.util import smart_str from .data import ( Data, get_file_peek, - Text + Text, ) from .xml import GenericXml @@ -52,6 +52,7 @@ log = logging.getLogger(__name__) @build_sniff_from_prefix class BlastXml(GenericXml): """NCBI Blast XML Output data""" + file_ext = "blastxml" edam_format = "format_3331" edam_data = "data_0857" @@ -60,10 +61,10 @@ class BlastXml(GenericXml): """Set the peek and blurb text""" if not dataset.dataset.purged: dataset.peek = get_file_peek(dataset.file_name) - dataset.blurb = 'NCBI Blast XML data' + dataset.blurb = "NCBI Blast XML data" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def sniff_prefix(self, file_prefix: FilePrefix): """Determines whether the file is blastxml @@ -84,11 +85,13 @@ class BlastXml(GenericXml): if line.strip() != '': return False line = handle.readline() - if line.strip() not in ['', - '']: + if line.strip() not in [ + '', + '', + ]: return False line = handle.readline() - if line.strip() != '': + if line.strip() != "": return False return True @@ -99,8 +102,7 @@ class BlastXml(GenericXml): # For one file only, use base class method (move/copy) return Text.merge(split_files, output_file) if not split_files: - raise ValueError("Given no BLAST XML files, %r, to merge into %s" - % (split_files, output_file)) + raise ValueError("Given no BLAST XML files, %r, to merge into %s" % (split_files, output_file)) with open(output_file, "w") as out: h = None old_header = None @@ -129,8 +131,10 @@ class BlastXml(GenericXml): raise ValueError(f"{f} is not an XML file!") line = h.readline() header += line - if line.strip() not in ['', - '']: + if line.strip() not in [ + '', + '', + ]: out.write(header) # for diagnosis h.close() raise ValueError(f"{f} is not a BLAST XML file!") @@ -158,8 +162,10 @@ class BlastXml(GenericXml): elif old_header is not None and old_header[:300] != header[:300]: # Enough to check and match h.close() - raise ValueError("BLAST XML headers don't match for %s and %s - have:\n%s\n...\n\nAnd:\n%s\n...\n" - % (split_files[0], f, old_header[:300], header[:300])) + raise ValueError( + "BLAST XML headers don't match for %s and %s - have:\n%s\n...\n\nAnd:\n%s\n...\n" + % (split_files[0], f, old_header[:300], header[:300]) + ) else: out.write(" \n") for line in h: @@ -182,8 +188,8 @@ class _BlastDb(Data): dataset.peek = "BLAST database (multiple files)" dataset.blurb = "BLAST database (multiple files)" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): """Create HTML content, used for displaying peek.""" @@ -192,22 +198,16 @@ class _BlastDb(Data): except Exception: return "BLAST database (multiple files)" - def display_data(self, trans, data, preview=False, filename=None, - to_ext=None, size=None, offset=None, **kwd): + def display_data(self, trans, data, preview=False, filename=None, to_ext=None, size=None, offset=None, **kwd): """ If preview is `True` allows us to format the data shown in the central pane via the "eye" icon. If preview is `False` triggers download. """ headers = kwd.get("headers", {}) if not preview: - return super().display_data(trans, - data=data, - preview=preview, - filename=filename, - to_ext=to_ext, - size=size, - offset=offset, - **kwd) + return super().display_data( + trans, data=data, preview=preview, filename=filename, to_ext=to_ext, size=size, offset=offset, **kwd + ) if self.file_ext == "blastdbn": title = "This is a nucleotide BLAST database" elif self.file_ext == "blastdbp": @@ -220,7 +220,7 @@ class _BlastDb(Data): msg = "" try: # Try to use any text recorded in the dummy index file: - with open(data.file_name, encoding='utf-8') as handle: + with open(data.file_name, encoding="utf-8") as handle: msg = handle.read().strip() except Exception: pass @@ -242,25 +242,46 @@ class _BlastDb(Data): class BlastNucDb(_BlastDb): """Class for nucleotide BLAST database files.""" - file_ext = 'blastdbn' - composite_type = 'basic' + + file_ext = "blastdbn" + composite_type = "basic" def __init__(self, **kwd): super().__init__(**kwd) - self.add_composite_file('blastdb.nhr', is_binary=True) # sequence headers - self.add_composite_file('blastdb.nin', is_binary=True) # index file - self.add_composite_file('blastdb.nsq', is_binary=True) # nucleotide sequences - self.add_composite_file('blastdb.nal', is_binary=False, optional=True) # alias ( -gi_mask option of makeblastdb) - self.add_composite_file('blastdb.nhd', is_binary=True, optional=True) # sorted sequence hash values ( -hash_index option of makeblastdb) - self.add_composite_file('blastdb.nhi', is_binary=True, optional=True) # index of sequence hash values ( -hash_index option of makeblastdb) - self.add_composite_file('blastdb.nnd', is_binary=True, optional=True) # sorted GI values ( -parse_seqids option of makeblastdb and gi present in the description lines) - self.add_composite_file('blastdb.nni', is_binary=True, optional=True) # index of GI values ( -parse_seqids option of makeblastdb and gi present in the description lines) - self.add_composite_file('blastdb.nog', is_binary=True, optional=True) # OID->GI lookup file ( -hash_index or -parse_seqids option of makeblastdb) - self.add_composite_file('blastdb.nsd', is_binary=True, optional=True) # sorted sequence accession values ( -hash_index or -parse_seqids option of makeblastdb) - self.add_composite_file('blastdb.nsi', is_binary=True, optional=True) # index of sequence accession values ( -hash_index or -parse_seqids option of makeblastdb) -# self.add_composite_file('blastdb.00.idx', is_binary=True, optional=True) # first volume of the MegaBLAST index generated by makembindex -# The previous line should be repeated for each index volume, with filename extensions like '.01.idx', '.02.idx', etc. - self.add_composite_file('blastdb.shd', is_binary=True, optional=True) # MegaBLAST index superheader (-old_style_index false option of makembindex) + self.add_composite_file("blastdb.nhr", is_binary=True) # sequence headers + self.add_composite_file("blastdb.nin", is_binary=True) # index file + self.add_composite_file("blastdb.nsq", is_binary=True) # nucleotide sequences + self.add_composite_file( + "blastdb.nal", is_binary=False, optional=True + ) # alias ( -gi_mask option of makeblastdb) + self.add_composite_file( + "blastdb.nhd", is_binary=True, optional=True + ) # sorted sequence hash values ( -hash_index option of makeblastdb) + self.add_composite_file( + "blastdb.nhi", is_binary=True, optional=True + ) # index of sequence hash values ( -hash_index option of makeblastdb) + self.add_composite_file( + "blastdb.nnd", is_binary=True, optional=True + ) # sorted GI values ( -parse_seqids option of makeblastdb and gi present in the description lines) + self.add_composite_file( + "blastdb.nni", is_binary=True, optional=True + ) # index of GI values ( -parse_seqids option of makeblastdb and gi present in the description lines) + self.add_composite_file( + "blastdb.nog", is_binary=True, optional=True + ) # OID->GI lookup file ( -hash_index or -parse_seqids option of makeblastdb) + self.add_composite_file( + "blastdb.nsd", is_binary=True, optional=True + ) # sorted sequence accession values ( -hash_index or -parse_seqids option of makeblastdb) + self.add_composite_file( + "blastdb.nsi", is_binary=True, optional=True + ) # index of sequence accession values ( -hash_index or -parse_seqids option of makeblastdb) + # self.add_composite_file('blastdb.00.idx', is_binary=True, optional=True) # first volume of the MegaBLAST index generated by makembindex + # The previous line should be repeated for each index volume, with filename extensions like '.01.idx', '.02.idx', etc. + self.add_composite_file( + "blastdb.shd", is_binary=True, optional=True + ) # MegaBLAST index superheader (-old_style_index false option of makembindex) + + # self.add_composite_file('blastdb.naa', is_binary=True, optional=True) # index of a WriteDB column for e.g. mask data # self.add_composite_file('blastdb.nab', is_binary=True, optional=True) # data of a WriteDB column # self.add_composite_file('blastdb.nac', is_binary=True, optional=True) # multiple byte order for a WriteDB column @@ -269,22 +290,25 @@ class BlastNucDb(_BlastDb): class BlastProtDb(_BlastDb): """Class for protein BLAST database files.""" - file_ext = 'blastdbp' - composite_type = 'basic' + + file_ext = "blastdbp" + composite_type = "basic" def __init__(self, **kwd): super().__init__(**kwd) -# Component file comments are as in BlastNucDb except where noted - self.add_composite_file('blastdb.phr', is_binary=True) - self.add_composite_file('blastdb.pin', is_binary=True) - self.add_composite_file('blastdb.psq', is_binary=True) # protein sequences - self.add_composite_file('blastdb.phd', is_binary=True, optional=True) - self.add_composite_file('blastdb.phi', is_binary=True, optional=True) - self.add_composite_file('blastdb.pnd', is_binary=True, optional=True) - self.add_composite_file('blastdb.pni', is_binary=True, optional=True) - self.add_composite_file('blastdb.pog', is_binary=True, optional=True) - self.add_composite_file('blastdb.psd', is_binary=True, optional=True) - self.add_composite_file('blastdb.psi', is_binary=True, optional=True) + # Component file comments are as in BlastNucDb except where noted + self.add_composite_file("blastdb.phr", is_binary=True) + self.add_composite_file("blastdb.pin", is_binary=True) + self.add_composite_file("blastdb.psq", is_binary=True) # protein sequences + self.add_composite_file("blastdb.phd", is_binary=True, optional=True) + self.add_composite_file("blastdb.phi", is_binary=True, optional=True) + self.add_composite_file("blastdb.pnd", is_binary=True, optional=True) + self.add_composite_file("blastdb.pni", is_binary=True, optional=True) + self.add_composite_file("blastdb.pog", is_binary=True, optional=True) + self.add_composite_file("blastdb.psd", is_binary=True, optional=True) + self.add_composite_file("blastdb.psi", is_binary=True, optional=True) + + # self.add_composite_file('blastdb.paa', is_binary=True, optional=True) # self.add_composite_file('blastdb.pab', is_binary=True, optional=True) # self.add_composite_file('blastdb.pac', is_binary=True, optional=True) @@ -293,26 +317,28 @@ class BlastProtDb(_BlastDb): class BlastDomainDb(_BlastDb): """Class for domain BLAST database files.""" - file_ext = 'blastdbd' - composite_type = 'basic' + + file_ext = "blastdbd" + composite_type = "basic" def __init__(self, **kwd): super().__init__(**kwd) - self.add_composite_file('blastdb.phr', is_binary=True) - self.add_composite_file('blastdb.pin', is_binary=True) - self.add_composite_file('blastdb.psq', is_binary=True) - self.add_composite_file('blastdb.freq', is_binary=True, optional=True) - self.add_composite_file('blastdb.loo', is_binary=True, optional=True) - self.add_composite_file('blastdb.psd', is_binary=True, optional=True) - self.add_composite_file('blastdb.psi', is_binary=True, optional=True) - self.add_composite_file('blastdb.rps', is_binary=True, optional=True) - self.add_composite_file('blastdb.aux', is_binary=True, optional=True) + self.add_composite_file("blastdb.phr", is_binary=True) + self.add_composite_file("blastdb.pin", is_binary=True) + self.add_composite_file("blastdb.psq", is_binary=True) + self.add_composite_file("blastdb.freq", is_binary=True, optional=True) + self.add_composite_file("blastdb.loo", is_binary=True, optional=True) + self.add_composite_file("blastdb.psd", is_binary=True, optional=True) + self.add_composite_file("blastdb.psi", is_binary=True, optional=True) + self.add_composite_file("blastdb.rps", is_binary=True, optional=True) + self.add_composite_file("blastdb.aux", is_binary=True, optional=True) class LastDb(Data): """Class for LAST database files.""" - file_ext = 'lastdb' - composite_type = 'basic' + + file_ext = "lastdb" + composite_type = "basic" def set_peek(self, dataset): """Set the peek and blurb text.""" @@ -320,8 +346,8 @@ class LastDb(Data): dataset.peek = "LAST database (multiple files)" dataset.blurb = "LAST database (multiple files)" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): """Create HTML content, used for displaying peek.""" @@ -332,36 +358,57 @@ class LastDb(Data): def __init__(self, **kwd): super().__init__(**kwd) - self.add_composite_file('lastdb.bck', is_binary=True) - self.add_composite_file('lastdb.des', description="Description file", is_binary=False) - self.add_composite_file('lastdb.prj', description="Project resume file", is_binary=False) - self.add_composite_file('lastdb.sds', is_binary=True) - self.add_composite_file('lastdb.ssp', is_binary=True) - self.add_composite_file('lastdb.suf', is_binary=True) - self.add_composite_file('lastdb.tis', is_binary=True) + self.add_composite_file("lastdb.bck", is_binary=True) + self.add_composite_file("lastdb.des", description="Description file", is_binary=False) + self.add_composite_file("lastdb.prj", description="Project resume file", is_binary=False) + self.add_composite_file("lastdb.sds", is_binary=True) + self.add_composite_file("lastdb.ssp", is_binary=True) + self.add_composite_file("lastdb.suf", is_binary=True) + self.add_composite_file("lastdb.tis", is_binary=True) class BlastNucDb5(_BlastDb): """Class for nucleotide BLAST database files.""" - file_ext = 'blastdbn5' - composite_type = 'basic' + + file_ext = "blastdbn5" + composite_type = "basic" def __init__(self, **kwd): super().__init__(**kwd) - self.add_composite_file('blastdb.nhr', is_binary=True) # sequence headers - self.add_composite_file('blastdb.nin', is_binary=True) # index file - self.add_composite_file('blastdb.nsq', is_binary=True) # nucleotide sequences - self.add_composite_file('blastdb.nal', is_binary=False, optional=True) # alias ( -gi_mask option of makeblastdb) - self.add_composite_file('blastdb.nhd', is_binary=True, optional=True) # sorted sequence hash values ( -hash_index option of makeblastdb) - self.add_composite_file('blastdb.nhi', is_binary=True, optional=True) # index of sequence hash values ( -hash_index option of makeblastdb) - self.add_composite_file('blastdb.nnd', is_binary=True, optional=True) # sorted GI values ( -parse_seqids option of makeblastdb and gi present in the description lines) - self.add_composite_file('blastdb.nni', is_binary=True, optional=True) # index of GI values ( -parse_seqids option of makeblastdb and gi present in the description lines) - self.add_composite_file('blastdb.nog', is_binary=True, optional=True) # OID->GI lookup file ( -hash_index or -parse_seqids option of makeblastdb) - self.add_composite_file('blastdb.nsd', is_binary=True, optional=True) # sorted sequence accession values ( -hash_index or -parse_seqids option of makeblastdb) - self.add_composite_file('blastdb.nsi', is_binary=True, optional=True) # index of sequence accession values ( -hash_index or -parse_seqids option of makeblastdb) -# self.add_composite_file('blastdb.00.idx', is_binary=True, optional=True) # first volume of the MegaBLAST index generated by makembindex -# The previous line should be repeated for each index volume, with filename extensions like '.01.idx', '.02.idx', etc. - self.add_composite_file('blastdb.shd', is_binary=True, optional=True) # MegaBLAST index superheader (-old_style_index false option of makembindex) + self.add_composite_file("blastdb.nhr", is_binary=True) # sequence headers + self.add_composite_file("blastdb.nin", is_binary=True) # index file + self.add_composite_file("blastdb.nsq", is_binary=True) # nucleotide sequences + self.add_composite_file( + "blastdb.nal", is_binary=False, optional=True + ) # alias ( -gi_mask option of makeblastdb) + self.add_composite_file( + "blastdb.nhd", is_binary=True, optional=True + ) # sorted sequence hash values ( -hash_index option of makeblastdb) + self.add_composite_file( + "blastdb.nhi", is_binary=True, optional=True + ) # index of sequence hash values ( -hash_index option of makeblastdb) + self.add_composite_file( + "blastdb.nnd", is_binary=True, optional=True + ) # sorted GI values ( -parse_seqids option of makeblastdb and gi present in the description lines) + self.add_composite_file( + "blastdb.nni", is_binary=True, optional=True + ) # index of GI values ( -parse_seqids option of makeblastdb and gi present in the description lines) + self.add_composite_file( + "blastdb.nog", is_binary=True, optional=True + ) # OID->GI lookup file ( -hash_index or -parse_seqids option of makeblastdb) + self.add_composite_file( + "blastdb.nsd", is_binary=True, optional=True + ) # sorted sequence accession values ( -hash_index or -parse_seqids option of makeblastdb) + self.add_composite_file( + "blastdb.nsi", is_binary=True, optional=True + ) # index of sequence accession values ( -hash_index or -parse_seqids option of makeblastdb) + # self.add_composite_file('blastdb.00.idx', is_binary=True, optional=True) # first volume of the MegaBLAST index generated by makembindex + # The previous line should be repeated for each index volume, with filename extensions like '.01.idx', '.02.idx', etc. + self.add_composite_file( + "blastdb.shd", is_binary=True, optional=True + ) # MegaBLAST index superheader (-old_style_index false option of makembindex) + + # self.add_composite_file('blastdb.naa', is_binary=True, optional=True) # index of a WriteDB column for e.g. mask data # self.add_composite_file('blastdb.nab', is_binary=True, optional=True) # data of a WriteDB column # self.add_composite_file('blastdb.nac', is_binary=True, optional=True) # multiple byte order for a WriteDB column @@ -370,22 +417,25 @@ class BlastNucDb5(_BlastDb): class BlastProtDb5(_BlastDb): """Class for protein BLAST database files.""" - file_ext = 'blastdbp5' - composite_type = 'basic' + + file_ext = "blastdbp5" + composite_type = "basic" def __init__(self, **kwd): super().__init__(**kwd) -# Component file comments are as in BlastNucDb except where noted - self.add_composite_file('blastdb.phr', is_binary=True) - self.add_composite_file('blastdb.pin', is_binary=True) - self.add_composite_file('blastdb.psq', is_binary=True) # protein sequences - self.add_composite_file('blastdb.phd', is_binary=True, optional=True) - self.add_composite_file('blastdb.phi', is_binary=True, optional=True) - self.add_composite_file('blastdb.pnd', is_binary=True, optional=True) - self.add_composite_file('blastdb.pni', is_binary=True, optional=True) - self.add_composite_file('blastdb.pog', is_binary=True, optional=True) - self.add_composite_file('blastdb.psd', is_binary=True, optional=True) - self.add_composite_file('blastdb.psi', is_binary=True, optional=True) + # Component file comments are as in BlastNucDb except where noted + self.add_composite_file("blastdb.phr", is_binary=True) + self.add_composite_file("blastdb.pin", is_binary=True) + self.add_composite_file("blastdb.psq", is_binary=True) # protein sequences + self.add_composite_file("blastdb.phd", is_binary=True, optional=True) + self.add_composite_file("blastdb.phi", is_binary=True, optional=True) + self.add_composite_file("blastdb.pnd", is_binary=True, optional=True) + self.add_composite_file("blastdb.pni", is_binary=True, optional=True) + self.add_composite_file("blastdb.pog", is_binary=True, optional=True) + self.add_composite_file("blastdb.psd", is_binary=True, optional=True) + self.add_composite_file("blastdb.psi", is_binary=True, optional=True) + + # self.add_composite_file('blastdb.paa', is_binary=True, optional=True) # self.add_composite_file('blastdb.pab', is_binary=True, optional=True) # self.add_composite_file('blastdb.pac', is_binary=True, optional=True) @@ -394,17 +444,18 @@ class BlastProtDb5(_BlastDb): class BlastDomainDb5(_BlastDb): """Class for domain BLAST database files.""" - file_ext = 'blastdbd5' - composite_type = 'basic' + + file_ext = "blastdbd5" + composite_type = "basic" def __init__(self, **kwd): super().__init__(**kwd) - self.add_composite_file('blastdb.phr', is_binary=True) - self.add_composite_file('blastdb.pin', is_binary=True) - self.add_composite_file('blastdb.psq', is_binary=True) - self.add_composite_file('blastdb.freq', is_binary=True, optional=True) - self.add_composite_file('blastdb.loo', is_binary=True, optional=True) - self.add_composite_file('blastdb.psd', is_binary=True, optional=True) - self.add_composite_file('blastdb.psi', is_binary=True, optional=True) - self.add_composite_file('blastdb.rps', is_binary=True, optional=True) - self.add_composite_file('blastdb.aux', is_binary=True, optional=True) + self.add_composite_file("blastdb.phr", is_binary=True) + self.add_composite_file("blastdb.pin", is_binary=True) + self.add_composite_file("blastdb.psq", is_binary=True) + self.add_composite_file("blastdb.freq", is_binary=True, optional=True) + self.add_composite_file("blastdb.loo", is_binary=True, optional=True) + self.add_composite_file("blastdb.psd", is_binary=True, optional=True) + self.add_composite_file("blastdb.psi", is_binary=True, optional=True) + self.add_composite_file("blastdb.rps", is_binary=True, optional=True) + self.add_composite_file("blastdb.aux", is_binary=True, optional=True) diff --git a/lib/galaxy/datatypes/checkers.py b/lib/galaxy/datatypes/checkers.py index ab2fce29024..600fecd8f18 100644 --- a/lib/galaxy/datatypes/checkers.py +++ b/lib/galaxy/datatypes/checkers.py @@ -14,12 +14,12 @@ from galaxy.util.checkers import ( ) __all__ = ( - 'check_binary', - 'check_bz2', - 'check_gzip', - 'check_html', - 'check_image', - 'check_zip', - 'is_gzip', - 'is_bz2', + "check_binary", + "check_bz2", + "check_gzip", + "check_html", + "check_image", + "check_zip", + "is_gzip", + "is_bz2", ) diff --git a/lib/galaxy/datatypes/constructive_solid_geometry.py b/lib/galaxy/datatypes/constructive_solid_geometry.py index 3bee5711b3e..7c8f6686fc2 100644 --- a/lib/galaxy/datatypes/constructive_solid_geometry.py +++ b/lib/galaxy/datatypes/constructive_solid_geometry.py @@ -10,8 +10,10 @@ from typing import List from galaxy import util from galaxy.datatypes import data from galaxy.datatypes.binary import Binary -from galaxy.datatypes.data import get_file_peek -from galaxy.datatypes.data import nice_size +from galaxy.datatypes.data import ( + get_file_peek, + nice_size, +) from galaxy.datatypes.metadata import MetadataElement from galaxy.datatypes.sniff import ( build_sniff_from_prefix, @@ -21,7 +23,7 @@ from galaxy.datatypes.tabular import Tabular MAX_HEADER_LINES = 500 MAX_LINE_LEN = 2000 -COLOR_OPTS = ['COLOR_SCALARS', 'red', 'green', 'blue'] +COLOR_OPTS = ["COLOR_SCALARS", "red", "green", "blue"] @build_sniff_from_prefix @@ -32,16 +34,21 @@ class Ply: normal direction that can be attached to these elements. A PLY file contains the description of exactly one object. """ - subtype = '' + + subtype = "" # Add metadata elements. - MetadataElement(name="file_format", default=None, desc="File format", - readonly=True, optional=True, visible=True) - MetadataElement(name="vertex", default=None, desc="Vertex", - readonly=True, optional=True, visible=True) - MetadataElement(name="face", default=None, desc="Face", - readonly=True, optional=True, visible=True) - MetadataElement(name="other_elements", default=[], desc="Other elements", - readonly=True, optional=True, visible=True, no_value=[]) + MetadataElement(name="file_format", default=None, desc="File format", readonly=True, optional=True, visible=True) + MetadataElement(name="vertex", default=None, desc="Vertex", readonly=True, optional=True, visible=True) + MetadataElement(name="face", default=None, desc="Face", readonly=True, optional=True, visible=True) + MetadataElement( + name="other_elements", + default=[], + desc="Other elements", + readonly=True, + optional=True, + visible=True, + no_value=[], + ) @abc.abstractmethod def __init__(self, **kwd): @@ -52,7 +59,7 @@ class Ply: The structure of a typical PLY file: Header, Vertex List, Face List, (lists of other elements) """ - if not self._is_ply_header(file_prefix.text_io(errors='ignore'), self.subtype): + if not self._is_ply_header(file_prefix.text_io(errors="ignore"), self.subtype): return False return True @@ -61,10 +68,10 @@ class Ply: The header is a series of carriage-return terminated lines of text that describe the remainder of the file. """ - valid_header_items = ['comment', 'obj_info', 'element', 'property'] + valid_header_items = ["comment", "obj_info", "element", "property"] # Line 1: ply line = get_next_line(fh) - if line != 'ply': + if line != "ply": return False # Line 2: format ascii 1.0 line = get_next_line(fh) @@ -74,7 +81,7 @@ class Ply: for line in util.iter_start_of_line(fh, MAX_LINE_LEN): line = line.strip() stop_index += 1 - if line == 'end_header': + if line == "end_header": return True items = line.split() if items[0] not in valid_header_items: @@ -87,22 +94,22 @@ class Ply: def set_meta(self, dataset, **kwd): if dataset.has_data(): - with open(dataset.file_name, errors='ignore') as fh: + with open(dataset.file_name, errors="ignore") as fh: for line in fh: line = line.strip() if not line: continue - if line.startswith('format'): + if line.startswith("format"): items = line.split() dataset.metadata.file_format = items[1] - elif line == 'end_header': + elif line == "end_header": # Metadata is complete. break - elif line.startswith('element'): + elif line.startswith("element"): items = line.split() - if items[1] == 'face': + if items[1] == "face": dataset.metadata.face = int(items[2]) - elif items[1] == 'vertex': + elif items[1] == "vertex": dataset.metadata.vertex = int(items[2]) else: element_tuple = (items[1], int(items[2])) @@ -113,8 +120,8 @@ class Ply: dataset.peek = get_file_peek(dataset.file_name) dataset.blurb = f"Faces: {str(dataset.metadata.face)}, Vertices: {str(dataset.metadata.vertex)}" else: - dataset.peek = 'File does not exist' - dataset.blurb = 'File purged from disc' + dataset.peek = "File does not exist" + dataset.blurb = "File purged from disc" def display_peek(self, dataset): try: @@ -133,8 +140,9 @@ class PlyAscii(Ply, data.Text): # type: ignore[misc] >>> PlyAscii().sniff(fname) False """ + file_ext = "plyascii" - subtype = 'ascii' + subtype = "ascii" def __init__(self, **kwd): data.Text.__init__(self, **kwd) @@ -142,7 +150,7 @@ class PlyAscii(Ply, data.Text): # type: ignore[misc] class PlyBinary(Ply, Binary): # type: ignore[misc] file_ext = "plybinary" - subtype = 'binary' + subtype = "binary" def __init__(self, **kwd): Binary.__init__(self, **kwd) @@ -173,46 +181,46 @@ class Vtk: TODO: only legacy formats are currently supported and support for XML formats should be added. """ - subtype = '' + subtype = "" # Add metadata elements. - MetadataElement(name="vtk_version", default=None, desc="Vtk version", - readonly=True, optional=True, visible=True) - MetadataElement(name="file_format", default=None, desc="File format", - readonly=True, optional=True, visible=True) - MetadataElement(name="dataset_type", default=None, desc="Dataset type", - readonly=True, optional=True, visible=True) + MetadataElement(name="vtk_version", default=None, desc="Vtk version", readonly=True, optional=True, visible=True) + MetadataElement(name="file_format", default=None, desc="File format", readonly=True, optional=True, visible=True) + MetadataElement(name="dataset_type", default=None, desc="Dataset type", readonly=True, optional=True, visible=True) # STRUCTURED_GRID data_type. - MetadataElement(name="dimensions", default=[], desc="Dimensions", - readonly=True, optional=True, visible=True, no_value=[]) - MetadataElement(name="origin", default=[], desc="Origin", - readonly=True, optional=True, visible=True, no_value=[]) - MetadataElement(name="spacing", default=[], desc="Spacing", - readonly=True, optional=True, visible=True, no_value=[]) + MetadataElement( + name="dimensions", default=[], desc="Dimensions", readonly=True, optional=True, visible=True, no_value=[] + ) + MetadataElement(name="origin", default=[], desc="Origin", readonly=True, optional=True, visible=True, no_value=[]) + MetadataElement(name="spacing", default=[], desc="Spacing", readonly=True, optional=True, visible=True, no_value=[]) # POLYDATA data_type (Points element is also a component of UNSTRUCTURED_GRID.. - MetadataElement(name="points", default=None, desc="Points", - readonly=True, optional=True, visible=True) - MetadataElement(name="vertices", default=None, desc="Vertices", - readonly=True, optional=True, visible=True) - MetadataElement(name="lines", default=None, desc="Lines", - readonly=True, optional=True, visible=True) - MetadataElement(name="polygons", default=None, desc="Polygons", - readonly=True, optional=True, visible=True) - MetadataElement(name="triangle_strips", default=None, desc="Triangle strips", - readonly=True, optional=True, visible=True) + MetadataElement(name="points", default=None, desc="Points", readonly=True, optional=True, visible=True) + MetadataElement(name="vertices", default=None, desc="Vertices", readonly=True, optional=True, visible=True) + MetadataElement(name="lines", default=None, desc="Lines", readonly=True, optional=True, visible=True) + MetadataElement(name="polygons", default=None, desc="Polygons", readonly=True, optional=True, visible=True) + MetadataElement( + name="triangle_strips", default=None, desc="Triangle strips", readonly=True, optional=True, visible=True + ) # UNSTRUCTURED_GRID data_type. - MetadataElement(name="cells", default=None, desc="Cells", - readonly=True, optional=True, visible=True) + MetadataElement(name="cells", default=None, desc="Cells", readonly=True, optional=True, visible=True) # Additional elements not categorized by data_type. - MetadataElement(name="field_names", default=[], desc="Field names", - readonly=True, optional=True, visible=True, no_value=[]) + MetadataElement( + name="field_names", default=[], desc="Field names", readonly=True, optional=True, visible=True, no_value=[] + ) # The keys in the field_components map to the list of field_names in the above element # which ensures order for select list options that are built from it. - MetadataElement(name="field_components", default={}, desc="Field names and components", - readonly=True, optional=True, visible=True, no_value={}) + MetadataElement( + name="field_components", + default={}, + desc="Field names and components", + readonly=True, + optional=True, + visible=True, + no_value={}, + ) @abc.abstractmethod def __init__(self, **kwd): @@ -224,7 +232,7 @@ class Vtk: styles of file formats: legacy or XML. We'll assume if the file contains a valid VTK header, then it is a valid VTK file. """ - if self._is_vtk_header(file_prefix.text_io(errors='ignore'), self.subtype): + if self._is_vtk_header(file_prefix.text_io(errors="ignore"), self.subtype): return True return False @@ -236,7 +244,7 @@ class Vtk: data_kind) or the 4th line consists of the data_kind (in which case the 5th line is blank). """ - data_kinds = ['STRUCTURED_GRID', 'POLYDATA', 'UNSTRUCTURED_GRID', 'STRUCTURED_POINTS', 'RECTILINEAR_GRID'] + data_kinds = ["STRUCTURED_GRID", "POLYDATA", "UNSTRUCTURED_GRID", "STRUCTURED_POINTS", "RECTILINEAR_GRID"] def check_data_kind(line): for data_kind in data_kinds: @@ -246,7 +254,7 @@ class Vtk: # Line 1: vtk DataFile Version 3.0 line = get_next_line(fh) - if line.find('vtk') < 0: + if line.find("vtk") < 0: return False # Line 2: can be anything - skip it line = get_next_line(fh) @@ -272,14 +280,14 @@ class Vtk: field_components = {} dataset_structure_complete = False processing_field_section = False - with open(dataset.file_name, errors='ignore') as fh: + with open(dataset.file_name, errors="ignore") as fh: for i, line in enumerate(fh): line = line.strip() if not line: continue if i < 3: dataset = self.set_initial_metadata(i, line, dataset) - elif dataset.metadata.file_format == 'ASCII' or not util.is_binary(line): + elif dataset.metadata.file_format == "ASCII" or not util.is_binary(line): if dataset_structure_complete: """ The final part of legacy VTK files describes the dataset attributes. @@ -300,7 +308,7 @@ class Vtk: reader, then the first data of that type is extracted from the file. """ items = line.split() - if items[0] == 'SCALARS': + if items[0] == "SCALARS": # Example: SCALARS surface_field double 3 # Scalar definition includes specification of a lookup table. The # definition of a lookup table is optional. If not specified, the @@ -317,7 +325,7 @@ class Vtk: num_components = 1 field_component_indexes = [str(i) for i in range(num_components)] field_components[field_name] = field_component_indexes - elif items[0] == 'FIELD': + elif items[0] == "FIELD": # The dataset consists of CELL_DATA. # FIELD FieldData 2 processing_field_section = True @@ -340,11 +348,11 @@ class Vtk: field_component_indexes = [str(i) for i in range(num_components)] field_components[field_name] = field_component_indexes fields_processed.append(field_name) - elif line.startswith('CELL_DATA'): + elif line.startswith("CELL_DATA"): # CELL_DATA 3188 dataset_structure_complete = True dataset.metadata.cells = int(line.split()[1]) - elif line.startswith('POINT_DATA'): + elif line.startswith("POINT_DATA"): # POINT_DATA 1876 dataset_structure_complete = True dataset.metadata.points = int(line.split()[1]) @@ -358,7 +366,7 @@ class Vtk: # The first part of legacy VTK files is the file version and # identifier. This part contains the single line: # # vtk DataFile Version X.Y - dataset.metadata.vtk_version = line.lower().split('version')[1] + dataset.metadata.vtk_version = line.lower().split("version")[1] # The second part of legacy VTK files is the header. The header # consists of a character string terminated by end-of-line # character \n. The header is 256 characters maximum. The header @@ -380,51 +388,51 @@ class Vtk: the type of dataset, other keyword/ data combinations define the actual data. """ - if dataset_type is None and line.startswith('DATASET'): + if dataset_type is None and line.startswith("DATASET"): dataset_type = line.split()[1] dataset.metadata.dataset_type = dataset_type - if dataset_type == 'STRUCTURED_GRID': + if dataset_type == "STRUCTURED_GRID": # The STRUCTURED_GRID format supports 1D, 2D, and 3D structured # grid datasets. The dimensions nx, ny, nz must be greater # than or equal to 1. The point coordinates are defined by the # data in the POINTS section. This consists of x-y-z data values # for each point. - if line.startswith('DIMENSIONS'): + if line.startswith("DIMENSIONS"): # DIMENSIONS 10 5 1 dataset.metadata.dimensions = [line.split()[1:]] - elif line.startswith('ORIGIN'): + elif line.startswith("ORIGIN"): # ORIGIN 0 0 0 dataset.metadata.origin = [line.split()[1:]] - elif line.startswith('SPACING'): + elif line.startswith("SPACING"): # SPACING 1 1 1 dataset.metadata.spacing = [line.split()[1:]] - elif dataset_type == 'POLYDATA': + elif dataset_type == "POLYDATA": # The polygonal dataset consists of arbitrary combinations # of surface graphics primitives vertices, lines, polygons # and triangle strips. Polygonal data is defined by the POINTS, # VERTICES, LINES, POLYGONS, or TRIANGLE_STRIPS sections. - if line.startswith('POINTS'): + if line.startswith("POINTS"): # POINTS 18 float dataset.metadata.points = int(line.split()[1]) - elif line.startswith('VERTICES'): + elif line.startswith("VERTICES"): dataset.metadata.vertices = int(line.split()[1]) - elif line.startswith('LINES'): + elif line.startswith("LINES"): # LINES 5 17 dataset.metadata.lines = int(line.split()[1]) - elif line.startswith('POLYGONS'): + elif line.startswith("POLYGONS"): # POLYGONS 6 30 dataset.metadata.polygons = int(line.split()[1]) - elif line.startswith('TRIANGLE_STRIPS'): + elif line.startswith("TRIANGLE_STRIPS"): # TRIANGLE_STRIPS 2212 16158 dataset.metadata.triangle_strips = int(line.split()[1]) - elif dataset_type == 'UNSTRUCTURED_GRID': + elif dataset_type == "UNSTRUCTURED_GRID": # The unstructured grid dataset consists of arbitrary combinations # of any possible cell type. Unstructured grids are defined by points, # cells, and cell types. - if line.startswith('POINTS'): + if line.startswith("POINTS"): # POINTS 18 float dataset.metadata.points = int(line.split()[1]) - if line.startswith('CELLS'): + if line.startswith("CELLS"): # CELLS 756 3024 dataset.metadata.cells = int(line.split()[1]) return dataset, dataset_type @@ -432,20 +440,20 @@ class Vtk: def get_blurb(self, dataset): blurb = "" if dataset.metadata.vtk_version is not None: - blurb += f'VTK Version {str(dataset.metadata.vtk_version)}' + blurb += f"VTK Version {str(dataset.metadata.vtk_version)}" if dataset.metadata.dataset_type is not None: if blurb: - blurb += ' ' + blurb += " " blurb += str(dataset.metadata.dataset_type) - return blurb or 'VTK data' + return blurb or "VTK data" def set_peek(self, dataset): if not dataset.dataset.purged: dataset.peek = get_file_peek(dataset.file_name) dataset.blurb = self.get_blurb(dataset) else: - dataset.peek = 'File does not exist' - dataset.blurb = 'File purged from disc' + dataset.peek = "File does not exist" + dataset.blurb = "File purged from disc" def display_peek(self, dataset): try: @@ -464,14 +472,15 @@ class VtkAscii(Vtk, data.Text): # type: ignore[misc] >>> VtkAscii().sniff(fname) False """ + file_ext = "vtkascii" - subtype = 'ASCII' + subtype = "ASCII" def __init__(self, **kwd): data.Text.__init__(self, **kwd) -class VtkBinary(Vtk, Binary): # type: ignore[misc] +class VtkBinary(Vtk, Binary): # type: ignore[misc] """ >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname('test.vtkbinary') @@ -483,7 +492,7 @@ class VtkBinary(Vtk, Binary): # type: ignore[misc] """ file_ext = "vtkbinary" - subtype = 'BINARY' + subtype = "BINARY" def __init__(self, **kwd): Binary.__init__(self, **kwd) @@ -505,6 +514,7 @@ class NeperTess(data.Text): **cell number_of_cells """ + file_ext = "neper.tess" MetadataElement(name="format", default=None, desc="format", readonly=True, visible=True) MetadataElement(name="dimension", default=None, desc="dimension", readonly=True, visible=True) @@ -524,16 +534,16 @@ class NeperTess(data.Text): >>> NeperTess().sniff(fname) False """ - return file_prefix.text_io(errors='ignore').readline(10).startswith('***tess') + return file_prefix.text_io(errors="ignore").readline(10).startswith("***tess") def set_meta(self, dataset, **kwd): if dataset.has_data(): - with open(dataset.file_name, errors='ignore') as fh: + with open(dataset.file_name, errors="ignore") as fh: for i, line in enumerate(fh): line = line.strip() if not line or i > 6: break - if i == 0 and not line.startswith('***tess'): + if i == 0 and not line.startswith("***tess"): break if i == 2: dataset.metadata.format = line @@ -545,10 +555,10 @@ class NeperTess(data.Text): def set_peek(self, dataset): if not dataset.dataset.purged: dataset.peek = get_file_peek(dataset.file_name, LINE_COUNT=7) - dataset.blurb = f'format: {str(dataset.metadata.format)} dim: {str(dataset.metadata.dimension)} cells: {str(dataset.metadata.cells)}' + dataset.blurb = f"format: {str(dataset.metadata.format)} dim: {str(dataset.metadata.dimension)} cells: {str(dataset.metadata.cells)}" else: - dataset.peek = 'File does not exist' - dataset.blurb = 'File purged from disc' + dataset.peek = "File does not exist" + dataset.blurb = "File purged from disc" @build_sniff_from_prefix @@ -568,6 +578,7 @@ class NeperTesr(Binary): [**cell number_of_cells """ + file_ext = "neper.tesr" MetadataElement(name="format", default=None, desc="format", readonly=True, visible=True) MetadataElement(name="dimension", default=None, desc="dimension", readonly=True, visible=True) @@ -590,19 +601,19 @@ class NeperTesr(Binary): >>> NeperTesr().sniff(fname) False """ - return file_prefix.text_io(errors='ignore').readline(10).startswith('***tesr') + return file_prefix.text_io(errors="ignore").readline(10).startswith("***tesr") def set_meta(self, dataset, **kwd): if dataset.has_data(): - with open(dataset.file_name, errors='ignore') as fh: - field = '' + with open(dataset.file_name, errors="ignore") as fh: + field = "" for i, line in enumerate(fh): line = line.strip() if not line or i > 12: break - if i == 0 and not line.startswith('***tesr'): + if i == 0 and not line.startswith("***tesr"): break - if line.startswith('*'): + if line.startswith("*"): field = line continue if i == 2: @@ -617,20 +628,20 @@ class NeperTesr(Binary): if i == 6: dataset.metadata.voxsize = line.split() continue - if field.startswith('*origin'): + if field.startswith("*origin"): dataset.metadata.origin = line.split() continue - if field.startswith('**cell'): + if field.startswith("**cell"): dataset.metadata.cells = int(line) break def set_peek(self, dataset): if not dataset.dataset.purged: dataset.peek = get_file_peek(dataset.file_name, LINE_COUNT=9) - dataset.blurb = f'format: {str(dataset.metadata.format)} dim: {str(dataset.metadata.dimension)} cells: {str(dataset.metadata.cells)}' + dataset.blurb = f"format: {str(dataset.metadata.format)} dim: {str(dataset.metadata.dimension)} cells: {str(dataset.metadata.cells)}" else: - dataset.peek = 'File does not exist' - dataset.blurb = 'File purged from disc' + dataset.peek = "File does not exist" + dataset.blurb = "File purged from disc" class NeperPoints(data.Text): @@ -638,6 +649,7 @@ class NeperPoints(data.Text): Neper Position File Neper position format has 1 - 3 floats per line separated by white space. """ + file_ext = "neper.points" MetadataElement(name="dimension", default=None, desc="dimension", readonly=True, visible=True) @@ -647,7 +659,7 @@ class NeperPoints(data.Text): def set_meta(self, dataset, **kwd): data.Text.set_meta(self, dataset, **kwd) if dataset.has_data(): - with open(dataset.file_name, errors='ignore') as fh: + with open(dataset.file_name, errors="ignore") as fh: dataset.metadata.dimension = self._get_dimension(fh) def _get_dimension(self, fh, maxlines=100, sep=None): @@ -672,7 +684,7 @@ class NeperPoints(data.Text): def set_peek(self, dataset): data.Text.set_peek(self, dataset) if not dataset.dataset.purged: - dataset.blurb += f' dim: {str(dataset.metadata.dimension)}' + dataset.blurb += f" dim: {str(dataset.metadata.dimension)}" class NeperPointsTabular(NeperPoints, Tabular): @@ -680,6 +692,7 @@ class NeperPointsTabular(NeperPoints, Tabular): Neper Position File Neper position format has 1 - 3 floats per line separated by TABs. """ + file_ext = "neper.points.tsv" def __init__(self, **kwd): @@ -688,25 +701,27 @@ class NeperPointsTabular(NeperPoints, Tabular): def set_meta(self, dataset, **kwd): Tabular.set_meta(self, dataset, **kwd) if dataset.has_data(): - with open(dataset.file_name, errors='ignore') as fh: + with open(dataset.file_name, errors="ignore") as fh: dataset.metadata.dimension = self._get_dimension(fh) def set_peek(self, dataset): Tabular.set_peek(self, dataset) if not dataset.dataset.purged: - dataset.blurb += f' dim: {str(dataset.metadata.dimension)}' + dataset.blurb += f" dim: {str(dataset.metadata.dimension)}" class NeperMultiScaleCell(data.Text): """ Neper Multiscale Cell File """ + file_ext = "neper.mscell" @build_sniff_from_prefix class GmshMsh(Binary): """Gmsh Mesh File""" + file_ext = "gmsh.msh" MetadataElement(name="version", default=None, desc="version", readonly=True, visible=True) MetadataElement(name="format", default=None, desc="format", readonly=True, visible=True) @@ -725,35 +740,36 @@ class GmshMsh(Binary): >>> GmshMsh().sniff(fname) False """ - return file_prefix.text_io(errors='ignore').readline().startswith('$MeshFormat') + return file_prefix.text_io(errors="ignore").readline().startswith("$MeshFormat") def set_meta(self, dataset, **kwd): if dataset.has_data(): - with open(dataset.file_name, errors='ignore') as fh: + with open(dataset.file_name, errors="ignore") as fh: for i, line in enumerate(fh): line = line.strip() if not line or i > 1: break - if i == 0 and not line.startswith('$MeshFormat'): + if i == 0 and not line.startswith("$MeshFormat"): break if i == 1: fields = line.split() if len(fields) > 0: dataset.metadata.version = fields[0] if len(fields) > 1: - dataset.metadata.format = 'ASCII' if fields[1] == '0' else 'binary' + dataset.metadata.format = "ASCII" if fields[1] == "0" else "binary" def set_peek(self, dataset): if not dataset.dataset.purged: dataset.peek = get_file_peek(dataset.file_name, LINE_COUNT=3) - dataset.blurb = f'Gmsh verion: {str(dataset.metadata.version)} {str(dataset.metadata.format)}' + dataset.blurb = f"Gmsh verion: {str(dataset.metadata.version)} {str(dataset.metadata.format)}" else: - dataset.peek = 'File does not exist' - dataset.blurb = 'File purged from disc' + dataset.peek = "File does not exist" + dataset.blurb = "File purged from disc" class GmshGeo(data.Text): """Gmsh geometry File""" + file_ext = "gmsh.geo" @@ -761,6 +777,7 @@ class ZsetGeof(data.Text): """ Z-set geof File """ + file_ext = "zset.geof" diff --git a/lib/galaxy/datatypes/converters/bed_to_gff_converter.py b/lib/galaxy/datatypes/converters/bed_to_gff_converter.py index 8e72f45d9b8..eee7add9a8d 100644 --- a/lib/galaxy/datatypes/converters/bed_to_gff_converter.py +++ b/lib/galaxy/datatypes/converters/bed_to_gff_converter.py @@ -12,15 +12,15 @@ def __main__(): skipped_lines = 0 first_skipped_line = 0 i = 0 - with open(input_name) as fh, open(output_name, 'w') as out: + with open(input_name) as fh, open(output_name, "w") as out: out.write("##gff-version 2\n") out.write("##bed_to_gff_converter.py\n\n") for i, line in enumerate(fh): complete_bed = False - line = line.rstrip('\r\n') - if line and not line.startswith('#') and not line.startswith('track') and not line.startswith('browser'): + line = line.rstrip("\r\n") + if line and not line.startswith("#") and not line.startswith("track") and not line.startswith("browser"): try: - elems = line.split('\t') + elems = line.split("\t") if len(elems) == 12: complete_bed = True chrom = elems[0] @@ -30,34 +30,43 @@ def __main__(): try: feature = elems[3] except Exception: - feature = 'feature%d' % (i + 1) + feature = "feature%d" % (i + 1) start = int(elems[1]) + 1 end = int(elems[2]) try: score = elems[4] except Exception: - score = '0' + score = "0" try: strand = elems[5] except Exception: - strand = '+' + strand = "+" try: group = elems[3] except Exception: - group = 'group%d' % (i + 1) + group = "group%d" % (i + 1) if complete_bed: - out.write('%s\tbed2gff\t%s\t%d\t%d\t%s\t%s\t.\t%s %s;\n' % (chrom, feature, start, end, score, strand, feature, group)) + out.write( + "%s\tbed2gff\t%s\t%d\t%d\t%s\t%s\t.\t%s %s;\n" + % (chrom, feature, start, end, score, strand, feature, group) + ) else: - out.write('%s\tbed2gff\t%s\t%d\t%d\t%s\t%s\t.\t%s;\n' % (chrom, feature, start, end, score, strand, group)) + out.write( + "%s\tbed2gff\t%s\t%d\t%d\t%s\t%s\t.\t%s;\n" + % (chrom, feature, start, end, score, strand, group) + ) if complete_bed: # We have all the info necessary to annotate exons for genes and mRNAs block_count = int(elems[9]) - block_sizes = elems[10].split(',') - block_starts = elems[11].split(',') + block_sizes = elems[10].split(",") + block_starts = elems[11].split(",") for j in range(block_count): exon_start = int(start) + int(block_starts[j]) exon_end = exon_start + int(block_sizes[j]) - 1 - out.write('%s\tbed2gff\texon\t%d\t%d\t%s\t%s\t.\texon %s;\n' % (chrom, exon_start, exon_end, score, strand, group)) + out.write( + "%s\tbed2gff\texon\t%d\t%d\t%s\t%s\t.\texon %s;\n" + % (chrom, exon_start, exon_end, score, strand, group) + ) except Exception: skipped_lines += 1 if not first_skipped_line: @@ -68,7 +77,10 @@ def __main__(): first_skipped_line = i + 1 info_msg = "%i lines converted to GFF version 2. " % (i + 1 - skipped_lines) if skipped_lines > 0: - info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % (skipped_lines, first_skipped_line) + info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % ( + skipped_lines, + first_skipped_line, + ) print(info_msg) diff --git a/lib/galaxy/datatypes/converters/bgzip.py b/lib/galaxy/datatypes/converters/bgzip.py index 4aa23ae2382..3e996c5442a 100644 --- a/lib/galaxy/datatypes/converters/bgzip.py +++ b/lib/galaxy/datatypes/converters/bgzip.py @@ -15,10 +15,10 @@ import pysam def main(): # Read options, args. parser = optparse.OptionParser() - parser.add_option('-c', '--chr-col', type='int', dest='chrom_col') - parser.add_option('-s', '--start-col', type='int', dest='start_col') - parser.add_option('-e', '--end-col', type='int', dest='end_col') - parser.add_option('-P', '--preset', dest='preset') + parser.add_option("-c", "--chr-col", type="int", dest="chrom_col") + parser.add_option("-s", "--start-col", type="int", dest="start_col") + parser.add_option("-e", "--end-col", type="int", dest="end_col") + parser.add_option("-P", "--preset", dest="preset") (options, args) = parser.parse_args() input_fname, output_fname = args @@ -29,8 +29,8 @@ def main(): sort_params = [ "sort", "-k{i},{i}".format(i=options.chrom_col), - "-k%(i)i,%(i)in" % {'i': options.start_col}, - "-k%(i)i,%(i)in" % {'i': options.end_col} + "-k%(i)i,%(i)in" % {"i": options.start_col}, + "-k%(i)i,%(i)in" % {"i": options.end_col}, ] elif options.preset == "bed": sort_params = ["sort", "-k1,1", "-k2,2n", "-k3,3n"] @@ -39,7 +39,9 @@ def main(): elif options.preset == "gff": sort_params = ["sort", "-s", "-k1,1", "-k4,4n"] # stable sort on start column # Skip any lines starting with "#" and "track" - grepped = subprocess.Popen(["grep", "-e", "^\"#\"", "-e", "^track", "-v", input_fname], stderr=subprocess.PIPE, stdout=subprocess.PIPE) + grepped = subprocess.Popen( + ["grep", "-e", '^"#"', "-e", "^track", "-v", input_fname], stderr=subprocess.PIPE, stdout=subprocess.PIPE + ) after_sort = subprocess.Popen(sort_params, stdin=grepped.stdout, stderr=subprocess.PIPE, stdout=tmpfile) grepped.stdout.close() output, err = after_sort.communicate() diff --git a/lib/galaxy/datatypes/converters/cram_to_bam.py b/lib/galaxy/datatypes/converters/cram_to_bam.py index f139442835f..673abff4f80 100644 --- a/lib/galaxy/datatypes/converters/cram_to_bam.py +++ b/lib/galaxy/datatypes/converters/cram_to_bam.py @@ -15,8 +15,8 @@ def main(): parser = optparse.OptionParser() (options, args) = parser.parse_args() input_fname, output_fname = args - slots = os.getenv('GALAXY_SLOTS', 1) - pysam.sort(f"-@{slots}", '-o', output_fname, '-O', 'bam', '-T', '.', input_fname) + slots = os.getenv("GALAXY_SLOTS", 1) + pysam.sort(f"-@{slots}", "-o", output_fname, "-O", "bam", "-T", ".", input_fname) if __name__ == "__main__": diff --git a/lib/galaxy/datatypes/converters/fasta_to_len.py b/lib/galaxy/datatypes/converters/fasta_to_len.py index 3447aba63c0..cec4bf35d68 100644 --- a/lib/galaxy/datatypes/converters/fasta_to_len.py +++ b/lib/galaxy/datatypes/converters/fasta_to_len.py @@ -14,7 +14,7 @@ def compute_fasta_length(fasta_file, out_file, keep_first_char, keep_first_word= infile = fasta_file keep_first_char = int(keep_first_char) - fasta_title = '' + fasta_title = "" seq_len = 0 # number of char to keep in the title @@ -25,13 +25,13 @@ def compute_fasta_length(fasta_file, out_file, keep_first_char, keep_first_word= first_entry = True - with open(out_file, 'w') as out: + with open(out_file, "w") as out: with open(infile) as fh: for line in fh: line = line.strip() - if not line or line.startswith('#'): + if not line or line.startswith("#"): continue - if line[0] == '>': + if line[0] == ">": if first_entry is False: if keep_first_word: fasta_title = fasta_title.split()[0] diff --git a/lib/galaxy/datatypes/converters/fasta_to_tabular_converter.py b/lib/galaxy/datatypes/converters/fasta_to_tabular_converter.py index 18d5b18789f..45ba059dd35 100644 --- a/lib/galaxy/datatypes/converters/fasta_to_tabular_converter.py +++ b/lib/galaxy/datatypes/converters/fasta_to_tabular_converter.py @@ -23,26 +23,26 @@ def __main__(): sys.exit(1) with open(infile) as inp: - with open(outfile, 'w') as out: - sequence = '' + with open(outfile, "w") as out: + sequence = "" for line in inp: - line = line.rstrip('\r\n') - if line.startswith('>'): + line = line.rstrip("\r\n") + if line.startswith(">"): if sequence: # Flush sequence from previous FASTA record, # removing any white space - out.write("".join(sequence.split()) + '\n') - sequence = '' + out.write("".join(sequence.split()) + "\n") + sequence = "" # Strip off the leading '>' and remove any pre-existing # tabs which would trigger extra columns; write with # tab to separate this from the sequence column: - out.write(line[1:].replace('\t', ' ') + '\t') + out.write(line[1:].replace("\t", " ") + "\t") else: # Continuing sequence, sequence += line # End of FASTA file, flush last sequence if sequence: - out.write("".join(sequence.split()) + '\n') + out.write("".join(sequence.split()) + "\n") if __name__ == "__main__": diff --git a/lib/galaxy/datatypes/converters/fastq_to_fqtoc.py b/lib/galaxy/datatypes/converters/fastq_to_fqtoc.py index cb0fb34b0e9..1a15c47d65d 100644 --- a/lib/galaxy/datatypes/converters/fastq_to_fqtoc.py +++ b/lib/galaxy/datatypes/converters/fastq_to_fqtoc.py @@ -19,17 +19,17 @@ def main(): """ input_fname = sys.argv[1] if is_gzip(input_fname): - sys.exit('Conversion is only possible for uncompressed files') + sys.exit("Conversion is only possible for uncompressed files") current_line = 0 sequences = 1000000 lines_per_chunk = 4 * sequences chunk_begin = 0 - with open(input_fname) as in_file, open(sys.argv[2], 'w') as out_file: + with open(input_fname) as in_file, open(sys.argv[2], "w") as out_file: out_file.write('{"sections" : [') - for _ in iter(in_file.readline, ''): + for _ in iter(in_file.readline, ""): current_line += 1 if 0 == current_line % lines_per_chunk: chunk_end = in_file.tell() @@ -37,8 +37,10 @@ def main(): chunk_begin = chunk_end chunk_end = in_file.tell() - out_file.write(f'{{"start":"{chunk_begin}","end":"{chunk_end}","sequences":"{current_line % lines_per_chunk / 4}"}}') - out_file.write(']}\n') + out_file.write( + f'{{"start":"{chunk_begin}","end":"{chunk_end}","sequences":"{current_line % lines_per_chunk / 4}"}}' + ) + out_file.write("]}\n") if __name__ == "__main__": diff --git a/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.py b/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.py index 18428f6c368..b7557de1481 100644 --- a/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.py +++ b/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.py @@ -25,12 +25,12 @@ def stop_err(msg): def __main__(): infile_name = sys.argv[1] fastq_block_lines = 0 - seq_title_startswith = '' + seq_title_startswith = "" - with open(infile_name) as fh, open(sys.argv[2], 'w') as outfile: + with open(infile_name) as fh, open(sys.argv[2], "w") as outfile: for i, line in enumerate(fh): line = line.rstrip() # eliminate trailing space and new line characters - if not line or line.startswith('#'): + if not line or line.startswith("#"): continue fastq_block_lines = (fastq_block_lines + 1) % 4 line_startswith = line[0:1] @@ -39,11 +39,11 @@ def __main__(): if not seq_title_startswith: seq_title_startswith = line_startswith if seq_title_startswith != line_startswith: - stop_err('Invalid fastqsolexa format at line %d: %s.' % (i + 1, line)) - outfile.write(f'>{line[1:]}\n') + stop_err("Invalid fastqsolexa format at line %d: %s." % (i + 1, line)) + outfile.write(f">{line[1:]}\n") elif fastq_block_lines == 2: # line 2 is nucleotides - outfile.write(f'{line}\n') + outfile.write(f"{line}\n") else: pass diff --git a/lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.py b/lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.py index c48283ba535..c2f51cde20c 100644 --- a/lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.py +++ b/lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.py @@ -25,15 +25,15 @@ def stop_err(msg): def __main__(): infile_name = sys.argv[1] # datatype = sys.argv[3] - qual_title_startswith = '' - seq_title_startswith = '' + qual_title_startswith = "" + seq_title_startswith = "" default_coding_value = 64 fastq_block_lines = 0 - with open(infile_name) as fh, open(sys.argv[2], 'w') as outfile_score: + with open(infile_name) as fh, open(sys.argv[2], "w") as outfile_score: for i, line in enumerate(fh): line = line.rstrip() - if not line or line.startswith('#'): + if not line or line.startswith("#"): continue fastq_block_lines = (fastq_block_lines + 1) % 4 line_startswith = line[0:1] @@ -42,7 +42,7 @@ def __main__(): if not seq_title_startswith: seq_title_startswith = line_startswith if line_startswith != seq_title_startswith: - stop_err('Invalid fastqsolexa format at line %d: %s.' % (i + 1, line)) + stop_err("Invalid fastqsolexa format at line %d: %s." % (i + 1, line)) read_title = line[1:] elif fastq_block_lines == 2: # second line is nucleotides @@ -52,17 +52,20 @@ def __main__(): if not qual_title_startswith: qual_title_startswith = line_startswith if line_startswith != qual_title_startswith: - stop_err('Invalid fastqsolexa format at line %d: %s.' % (i + 1, line)) + stop_err("Invalid fastqsolexa format at line %d: %s." % (i + 1, line)) quality_title = line[1:] if quality_title and read_title != quality_title: - stop_err('Invalid fastqsolexa format at line %d: sequence title "%s" differes from score title "%s".' % (i + 1, read_title, quality_title)) + stop_err( + 'Invalid fastqsolexa format at line %d: sequence title "%s" differes from score title "%s".' + % (i + 1, read_title, quality_title) + ) if not quality_title: - outfile_score.write(f'>{read_title}\n') + outfile_score.write(f">{read_title}\n") else: - outfile_score.write(f'>{line[1:]}\n') + outfile_score.write(f">{line[1:]}\n") else: # fourth line is quality scores - qual = '' + qual = "" fastq_integer = True # peek: ascii or digits? val = line.split()[0] @@ -84,11 +87,14 @@ def __main__(): elif quality_score_length == read_length: quality_score_startswith = default_coding_value else: - stop_err('Invalid fastqsolexa format at line %d: the number of quality scores ( %d ) is not the same as bases ( %d ).' % (i + 1, quality_score_length, read_length)) + stop_err( + "Invalid fastqsolexa format at line %d: the number of quality scores ( %d ) is not the same as bases ( %d )." + % (i + 1, quality_score_length, read_length) + ) for char in line: - score = ord(char) - quality_score_startswith # 64 + score = ord(char) - quality_score_startswith # 64 qual = f"{qual}{str(score)} " - outfile_score.write(f'{qual}\n') + outfile_score.write(f"{qual}\n") if __name__ == "__main__": diff --git a/lib/galaxy/datatypes/converters/gff_to_bed_converter.py b/lib/galaxy/datatypes/converters/gff_to_bed_converter.py index 1fe4ed4ef3b..379b3f3393c 100644 --- a/lib/galaxy/datatypes/converters/gff_to_bed_converter.py +++ b/lib/galaxy/datatypes/converters/gff_to_bed_converter.py @@ -11,16 +11,16 @@ def __main__(): skipped_lines = 0 first_skipped_line = 0 i = 0 - with open(input_name) as fh, open(output_name, 'w') as out: + with open(input_name) as fh, open(output_name, "w") as out: for i, line in enumerate(fh): - line = line.rstrip('\r\n') - if line and not line.startswith('#'): + line = line.rstrip("\r\n") + if line and not line.startswith("#"): try: - elems = line.split('\t') + elems = line.split("\t") start = str(int(elems[3]) - 1) strand = elems[6] - if strand not in ['+', '-']: - strand = '+' + if strand not in ["+", "-"]: + strand = "+" # GFF format: chrom source, name, chromStart, chromEnd, score, strand # Bed format: chrom, chromStart, chromEnd, name, score, strand # @@ -37,7 +37,10 @@ def __main__(): first_skipped_line = i + 1 info_msg = "%i lines converted to BED. " % (i + 1 - skipped_lines) if skipped_lines > 0: - info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % (skipped_lines, first_skipped_line) + info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % ( + skipped_lines, + first_skipped_line, + ) print(info_msg) diff --git a/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py b/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py index 1f7d7670e2c..70ea0aa2f01 100644 --- a/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py +++ b/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py @@ -12,7 +12,11 @@ import sys from bx.interval_index_file import Indexes -from galaxy.datatypes.util.gff_util import convert_gff_coords_to_bed, GenomicInterval, GFFReaderWrapper +from galaxy.datatypes.util.gff_util import ( + convert_gff_coords_to_bed, + GenomicInterval, + GFFReaderWrapper, +) def main(): diff --git a/lib/galaxy/datatypes/converters/interval_to_bed_converter.py b/lib/galaxy/datatypes/converters/interval_to_bed_converter.py index 4263962ebab..b5d5cddb423 100644 --- a/lib/galaxy/datatypes/converters/interval_to_bed_converter.py +++ b/lib/galaxy/datatypes/converters/interval_to_bed_converter.py @@ -18,15 +18,21 @@ def __main__(): try: chromCol = int(sys.argv[3]) - 1 except Exception: - stop_err(f"'{str(sys.argv[3])}' is an invalid chrom column, correct the column settings before attempting to convert the data format.") + stop_err( + f"'{str(sys.argv[3])}' is an invalid chrom column, correct the column settings before attempting to convert the data format." + ) try: startCol = int(sys.argv[4]) - 1 except Exception: - stop_err(f"'{str(sys.argv[4])}' is an invalid start column, correct the column settings before attempting to convert the data format.") + stop_err( + f"'{str(sys.argv[4])}' is an invalid start column, correct the column settings before attempting to convert the data format." + ) try: endCol = int(sys.argv[5]) - 1 except Exception: - stop_err(f"'{str(sys.argv[5])}' is an invalid end column, correct the column settings before attempting to convert the data format.") + stop_err( + f"'{str(sys.argv[5])}' is an invalid end column, correct the column settings before attempting to convert the data format." + ) try: strandCol = int(sys.argv[6]) - 1 except Exception: @@ -38,8 +44,19 @@ def __main__(): skipped_lines = 0 first_skipped_line = 0 count = 0 - with open(input_name) as fh, open(output_name, 'w') as out: - for count, region in enumerate(bx.intervals.io.NiceReaderWrapper(fh, chrom_col=chromCol, start_col=startCol, end_col=endCol, strand_col=strandCol, fix_strand=True, return_header=False, return_comments=False)): + with open(input_name) as fh, open(output_name, "w") as out: + for count, region in enumerate( + bx.intervals.io.NiceReaderWrapper( + fh, + chrom_col=chromCol, + start_col=startCol, + end_col=endCol, + strand_col=strandCol, + fix_strand=True, + return_header=False, + return_comments=False, + ) + ): try: if nameCol >= 0: name = region.fields[nameCol] diff --git a/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py b/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py index 70f34cc7d82..ea2d1b01313 100644 --- a/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py +++ b/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py @@ -14,23 +14,23 @@ def stop_err(msg): def force_bed_field_count(fields, region_count, force_num_columns): if force_num_columns >= 4 and len(fields) < 4: - fields.append('region_%i' % (region_count)) + fields.append("region_%i" % (region_count)) if force_num_columns >= 5 and len(fields) < 5: - fields.append('0') + fields.append("0") if force_num_columns >= 6 and len(fields) < 6: - fields.append('+') + fields.append("+") if force_num_columns >= 7 and len(fields) < 7: fields.append(fields[1]) if force_num_columns >= 8 and len(fields) < 8: fields.append(fields[2]) if force_num_columns >= 9 and len(fields) < 9: - fields.append('0') + fields.append("0") if force_num_columns >= 10 and len(fields) < 10: - fields.append('0') + fields.append("0") if force_num_columns >= 11 and len(fields) < 11: - fields.append(',') + fields.append(",") if force_num_columns >= 12 and len(fields) < 12: - fields.append(',') + fields.append(",") return fields[:force_num_columns] @@ -40,15 +40,21 @@ def __main__(): try: chromCol = int(sys.argv[3]) - 1 except Exception: - stop_err(f"'{str(sys.argv[3])}' is an invalid chrom column, correct the column settings before attempting to convert the data format.") + stop_err( + f"'{str(sys.argv[3])}' is an invalid chrom column, correct the column settings before attempting to convert the data format." + ) try: startCol = int(sys.argv[4]) - 1 except Exception: - stop_err(f"'{str(sys.argv[4])}' is an invalid start column, correct the column settings before attempting to convert the data format.") + stop_err( + f"'{str(sys.argv[4])}' is an invalid start column, correct the column settings before attempting to convert the data format." + ) try: endCol = int(sys.argv[5]) - 1 except Exception: - stop_err(f"'{str(sys.argv[5])}' is an invalid end column, correct the column settings before attempting to convert the data format.") + stop_err( + f"'{str(sys.argv[5])}' is an invalid end column, correct the column settings before attempting to convert the data format." + ) try: strandCol = int(sys.argv[6]) - 1 except Exception: @@ -60,7 +66,7 @@ def __main__(): try: extension = sys.argv[8] except IndexError: - extension = 'interval' # default extension + extension = "interval" # default extension try: force_num_columns = int(sys.argv[9]) except Exception: @@ -72,53 +78,77 @@ def __main__(): # does file already conform to bed strict? # if so, we want to keep extended columns, otherwise we'll create a generic 6 column bed file strict_bed = True - if extension in ['bed', 'bedstrict', 'bed6', 'bed12'] and (chromCol, startCol, endCol) == (0, 1, 2) and (nameCol < 0 or nameCol == 3) and (strandCol < 0 or strandCol == 5): - with open(input_name) as fh, open(output_name, 'w') as out: + if ( + extension in ["bed", "bedstrict", "bed6", "bed12"] + and (chromCol, startCol, endCol) == (0, 1, 2) + and (nameCol < 0 or nameCol == 3) + and (strandCol < 0 or strandCol == 5) + ): + with open(input_name) as fh, open(output_name, "w") as out: for count, line in enumerate(fh): - line = line.rstrip('\n\r') + line = line.rstrip("\n\r") if line == "" or line.startswith("#"): skipped_lines += 1 if first_skipped_line is None: first_skipped_line = count + 1 continue - fields = line.split('\t') + fields = line.split("\t") try: - assert len(fields) >= 3, 'A BED file requires at least 3 columns' # we can't fix this + assert len(fields) >= 3, "A BED file requires at least 3 columns" # we can't fix this if len(fields) > 12: strict_bed = False break # name (fields[3]) can be anything, no verification needed if len(fields) > 4: - float(fields[4]) # score - A score between 0 and 1000. If the track line useScore attribute is set to 1 for this annotation data set, the score value will determine the level of gray in which this feature is displayed (higher numbers = darker gray). + float( + fields[4] + ) # score - A score between 0 and 1000. If the track line useScore attribute is set to 1 for this annotation data set, the score value will determine the level of gray in which this feature is displayed (higher numbers = darker gray). if len(fields) > 5: - assert fields[5] in ['+', '-'], 'Invalid strand' # strand - Defines the strand - either '+' or '-'. + assert fields[5] in [ + "+", + "-", + ], "Invalid strand" # strand - Defines the strand - either '+' or '-'. if len(fields) > 6: - int(fields[6]) # thickStart - The starting position at which the feature is drawn thickly (for example, the start codon in gene displays). + int( + fields[6] + ) # thickStart - The starting position at which the feature is drawn thickly (for example, the start codon in gene displays). if len(fields) > 7: - int(fields[7]) # thickEnd - The ending position at which the feature is drawn thickly (for example, the stop codon in gene displays). + int( + fields[7] + ) # thickEnd - The ending position at which the feature is drawn thickly (for example, the stop codon in gene displays). if len(fields) > 8: - if fields[8] != '0': # itemRgb - An RGB value of the form R,G,B (e.g. 255,0,0). If the track line itemRgb attribute is set to "On", this RBG value will determine the display color of the data contained in this BED line. NOTE: It is recommended that a simple color scheme (eight colors or less) be used with this attribute to avoid overwhelming the color resources of the Genome Browser and your Internet browser. - fields2 = fields[8].split(',') - assert len(fields2) == 3, 'RGB value must be 0 or have length of 3' + if ( + fields[8] != "0" + ): # itemRgb - An RGB value of the form R,G,B (e.g. 255,0,0). If the track line itemRgb attribute is set to "On", this RBG value will determine the display color of the data contained in this BED line. NOTE: It is recommended that a simple color scheme (eight colors or less) be used with this attribute to avoid overwhelming the color resources of the Genome Browser and your Internet browser. + fields2 = fields[8].split(",") + assert len(fields2) == 3, "RGB value must be 0 or have length of 3" for field in fields2: int(field) # rgb values are integers if len(fields) > 9: int(fields[9]) # blockCount - The number of blocks (exons) in the BED line. if len(fields) > 10: - if fields[10] != ',': # blockSizes - A comma-separated list of the block sizes. The number of items in this list should correspond to blockCount. - fields2 = fields[10].rstrip(",").split(",") # remove trailing comma and split on comma + if ( + fields[10] != "," + ): # blockSizes - A comma-separated list of the block sizes. The number of items in this list should correspond to blockCount. + fields2 = ( + fields[10].rstrip(",").split(",") + ) # remove trailing comma and split on comma for field in fields2: int(field) if len(fields) > 11: - if fields[11] != ',': # blockStarts - A comma-separated list of block starts. All of the blockStart positions should be calculated relative to chromStart. The number of items in this list should correspond to blockCount. - fields2 = fields[11].rstrip(",").split(",") # remove trailing comma and split on comma + if ( + fields[11] != "," + ): # blockStarts - A comma-separated list of block starts. All of the blockStart positions should be calculated relative to chromStart. The number of items in this list should correspond to blockCount. + fields2 = ( + fields[11].rstrip(",").split(",") + ) # remove trailing comma and split on comma for field in fields2: int(field) except Exception: strict_bed = False break if force_num_columns is not None and len(fields) != force_num_columns: - line = '\t'.join(force_bed_field_count(fields, count, force_num_columns)) + line = "\t".join(force_bed_field_count(fields, count, force_num_columns)) out.write(f"{line}\n") else: strict_bed = False @@ -127,8 +157,19 @@ def __main__(): skipped_lines = 0 first_skipped_line = None count = 0 - with open(input_name) as fh, open(output_name, 'w') as out: - for count, region in enumerate(bx.intervals.io.NiceReaderWrapper(fh, chrom_col=chromCol, start_col=startCol, end_col=endCol, strand_col=strandCol, fix_strand=True, return_header=False, return_comments=False)): + with open(input_name) as fh, open(output_name, "w") as out: + for count, region in enumerate( + bx.intervals.io.NiceReaderWrapper( + fh, + chrom_col=chromCol, + start_col=startCol, + end_col=endCol, + strand_col=strandCol, + fix_strand=True, + return_header=False, + return_comments=False, + ) + ): try: if nameCol >= 0: name = region.fields[nameCol] @@ -140,7 +181,7 @@ def __main__(): fields = [str(item) for item in (region.chrom, region.start, region.end, name, 0, region.strand)] if force_num_columns is not None and len(fields) != force_num_columns: fields = force_bed_field_count(fields, count, force_num_columns) - out.write("%s\n" % '\t'.join(fields)) + out.write("%s\n" % "\t".join(fields)) except Exception: skipped_lines += 1 if first_skipped_line is None: diff --git a/lib/galaxy/datatypes/converters/interval_to_fli.py b/lib/galaxy/datatypes/converters/interval_to_fli.py index 16dcf4f88d3..fb2d1351460 100644 --- a/lib/galaxy/datatypes/converters/interval_to_fli.py +++ b/lib/galaxy/datatypes/converters/interval_to_fli.py @@ -1,4 +1,4 @@ -''' +""" Creates a feature location index (FLI) for a given BED/GFF file. FLI index has the form:: @@ -12,29 +12,36 @@ where location is formatted as: contig:start-end and symbols are sorted in lexigraphical order. -''' +""" import optparse -from bx.tabular.io import Comment, Header +from bx.tabular.io import ( + Comment, + Header, +) -from galaxy.datatypes.util.gff_util import convert_gff_coords_to_bed, GFFReaderWrapper, read_unordered_gtf +from galaxy.datatypes.util.gff_util import ( + convert_gff_coords_to_bed, + GFFReaderWrapper, + read_unordered_gtf, +) def main(): # Process arguments. parser = optparse.OptionParser() - parser.add_option('-F', '--format', dest="input_format") + parser.add_option("-F", "--format", dest="input_format") (options, args) = parser.parse_args() in_fname, out_fname = args input_format = options.input_format.lower() # Create dict of name-location pairings. name_loc_dict = {} - if input_format in ['gff', 'gtf']: + if input_format in ["gff", "gtf"]: # GTF/GFF format # Create reader. - if input_format == 'gff': + if input_format == "gff": in_reader = GFFReaderWrapper(open(in_fname)) else: # input_format == 'gtf' in_reader = read_unordered_gtf(open(in_fname)) @@ -53,19 +60,15 @@ def main(): # Value is not a number, so it can be indexed. if val not in name_loc_dict: # Value is not in dictionary. - name_loc_dict[val] = { - 'contig': feature.chrom, - 'start': feature.start, - 'end': feature.end - } + name_loc_dict[val] = {"contig": feature.chrom, "start": feature.start, "end": feature.end} else: # Value already in dictionary, so update dictionary. loc = name_loc_dict[val] - if feature.start < loc['start']: - loc['start'] = feature.start - if feature.end > loc['end']: - loc['end'] = feature.end - elif input_format == 'bed': + if feature.start < loc["start"]: + loc["start"] = feature.start + if feature.end > loc["end"]: + loc["end"] = feature.end + elif input_format == "bed": # BED format. for line in open(in_fname): # Ignore track lines. @@ -79,28 +82,24 @@ def main(): continue # Process line - name_loc_dict[fields[3]] = { - 'contig': fields[0], - 'start': int(fields[1]), - 'end': int(fields[2]) - } + name_loc_dict[fields[3]] = {"contig": fields[0], "start": int(fields[1]), "end": int(fields[2])} # Create sorted list of entries. max_len = 0 entries = [] for name in sorted(name_loc_dict.keys()): loc = name_loc_dict[name] - entry = '{}\t{}\t{}'.format(name.lower(), name, '%s:%i-%i' % (loc['contig'], loc['start'], loc['end'])) + entry = "{}\t{}\t{}".format(name.lower(), name, "%s:%i-%i" % (loc["contig"], loc["start"], loc["end"])) if len(entry) > max_len: max_len = len(entry) entries.append(entry) # Write padded entries. - with open(out_fname, 'w') as out: + with open(out_fname, "w") as out: out.write(f"{str(max_len + 1).ljust(max_len)}\n") for entry in entries: out.write(f"{entry.ljust(max_len)}\n") -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.py b/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.py index 68ff09f2bc1..60a9146fe20 100644 --- a/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.py +++ b/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.py @@ -18,9 +18,9 @@ def main(): # Read options, args. parser = optparse.OptionParser() - parser.add_option('-c', '--chr-col', type='int', dest='chrom_col', default=1) - parser.add_option('-s', '--start-col', type='int', dest='start_col', default=2) - parser.add_option('-e', '--end-col', type='int', dest='end_col', default=3) + parser.add_option("-c", "--chr-col", type="int", dest="chrom_col", default=1) + parser.add_option("-s", "--start-col", type="int", dest="start_col", default=2) + parser.add_option("-e", "--end-col", type="int", dest="end_col", default=3) (options, args) = parser.parse_args() input_fname, output_fname = args @@ -44,7 +44,7 @@ def main(): index.add(chrom, chrom_start, chrom_end, offset) offset += len(line) - with open(output_fname, 'wb') as out: + with open(output_fname, "wb") as out: index.write(out) diff --git a/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py b/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py index a4fec2bef4d..29e66b518a0 100644 --- a/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py +++ b/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py @@ -16,31 +16,40 @@ import pysam def main(): # Read options, args. parser = optparse.OptionParser() - parser.add_option('-c', '--chr-col', type='int', dest='chrom_col') - parser.add_option('-s', '--start-col', type='int', dest='start_col') - parser.add_option('-e', '--end-col', type='int', dest='end_col') - parser.add_option('-P', '--preset', dest='preset') + parser.add_option("-c", "--chr-col", type="int", dest="chrom_col") + parser.add_option("-s", "--start-col", type="int", dest="start_col") + parser.add_option("-e", "--end-col", type="int", dest="end_col") + parser.add_option("-P", "--preset", dest="preset") (options, args) = parser.parse_args() _, bgzip_fname, out_fname = args - to_tabix(bgzip_fname=bgzip_fname, - out_fname=out_fname, - preset=options.preset, - chrom_col=options.chrom_col, - start_col=options.start_col, - end_col=options.end_col) + to_tabix( + bgzip_fname=bgzip_fname, + out_fname=out_fname, + preset=options.preset, + chrom_col=options.chrom_col, + start_col=options.start_col, + end_col=options.end_col, + ) def to_tabix(bgzip_fname, out_fname, preset=None, chrom_col=None, start_col=None, end_col=None): # Create index. if preset: # Preset type. - bgzip_fname = pysam.tabix_index(filename=bgzip_fname, preset=preset, keep_original=True, - index=out_fname, force=True) + bgzip_fname = pysam.tabix_index( + filename=bgzip_fname, preset=preset, keep_original=True, index=out_fname, force=True + ) else: # For interval files; column indices are 0-based. - bgzip_fname = pysam.tabix_index(filename=bgzip_fname, seq_col=(chrom_col - 1), - start_col=(start_col - 1), end_col=(end_col - 1), - keep_original=True, index=out_fname, force=True) + bgzip_fname = pysam.tabix_index( + filename=bgzip_fname, + seq_col=(chrom_col - 1), + start_col=(start_col - 1), + end_col=(end_col - 1), + keep_original=True, + index=out_fname, + force=True, + ) if os.path.getsize(out_fname) == 0: sys.exit("The converted tabix index file is empty, meaning the input data is invalid.") return bgzip_fname diff --git a/lib/galaxy/datatypes/converters/lped_to_fped_converter.py b/lib/galaxy/datatypes/converters/lped_to_fped_converter.py index 9ba6c738e1c..e1484529278 100644 --- a/lib/galaxy/datatypes/converters/lped_to_fped_converter.py +++ b/lib/galaxy/datatypes/converters/lped_to_fped_converter.py @@ -8,7 +8,7 @@ import sys import time prog = os.path.split(sys.argv[0])[-1] -myversion = 'Oct 10 2009' +myversion = "Oct 10 2009" galhtmlprefix = """ @@ -25,36 +25,35 @@ galhtmlprefix = """ def timenow(): - """return current time as a string - """ - return time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(time.time())) + """return current time as a string""" + return time.strftime("%d/%m/%Y %H:%M:%S", time.localtime(time.time())) def rgConv(inpedfilepath, outhtmlname, outfilepath): """convert linkage ped/map to fbat""" - recode = {'A': '1', 'C': '2', 'G': '3', 'T': '4', 'N': '0', '0': '0', '1': '1', '2': '2', '3': '3', '4': '4'} + recode = {"A": "1", "C": "2", "G": "3", "T": "4", "N": "0", "0": "0", "1": "1", "2": "2", "3": "3", "4": "4"} basename = os.path.split(inpedfilepath)[-1] # get basename - inmap = f'{inpedfilepath}.map' - inped = f'{inpedfilepath}.ped' - outf = f'{basename}.ped' # note the fbat exe insists that this is the extension for the ped data + inmap = f"{inpedfilepath}.map" + inped = f"{inpedfilepath}.ped" + outf = f"{basename}.ped" # note the fbat exe insists that this is the extension for the ped data outfpath = os.path.join(outfilepath, outf) # where to write the fbat format file to try: mf = open(inmap) except Exception: - sys.exit(f'{prog} cannot open inmap file {inmap} - do you have permission?\n') + sys.exit(f"{prog} cannot open inmap file {inmap} - do you have permission?\n") try: rsl = [x.split()[1] for x in mf] except Exception: - sys.exit(f'## cannot parse {inmap}') + sys.exit(f"## cannot parse {inmap}") try: os.makedirs(outfilepath) except Exception: pass # already exists - head = ' '.join(rsl) # list of rs numbers + head = " ".join(rsl) # list of rs numbers # TODO add anno to rs but fbat will prolly barf? - with open(inped) as pedf, open(outfpath, 'w', 2 ** 20) as o: + with open(inped) as pedf, open(outfpath, "w", 2**20) as o: o.write(head) - o.write('\n') + o.write("\n") for i, row in enumerate(pedf): if i == 0: lrow = row.split() @@ -66,7 +65,7 @@ def rgConv(inpedfilepath, outhtmlname, outfilepath): lrow = row.strip().split() p = lrow[:6] g = lrow[6:] - gc = [recode.get(z, '0') for z in g] + gc = [recode.get(z, "0") for z in g] lrow = p + gc row = f"{' '.join(lrow)}\n" o.write(row) @@ -82,7 +81,7 @@ def main(): """ nparm = 3 if len(sys.argv) < nparm: - sys.exit('## %s called with %s - needs %d parameters \n' % (prog, sys.argv, nparm)) + sys.exit("## %s called with %s - needs %d parameters \n" % (prog, sys.argv, nparm)) inpedfilepath = sys.argv[1] outhtmlname = sys.argv[2] outfilepath = sys.argv[3] @@ -92,10 +91,10 @@ def main(): pass rgConv(inpedfilepath, outhtmlname, outfilepath) flist = os.listdir(outfilepath) - with open(outhtmlname, 'w') as f: + with open(outhtmlname, "w") as f: f.write(galhtmlprefix % prog) - print(f'## Rgenetics: http://rgenetics.org Galaxy Tools {prog} {timenow()}') # becomes info - f.write(f'
    ## Rgenetics: http://rgenetics.org Galaxy Tools {prog} {timenow()}\n
      ') + print(f"## Rgenetics: http://rgenetics.org Galaxy Tools {prog} {timenow()}") # becomes info + f.write(f"
      ## Rgenetics: http://rgenetics.org Galaxy Tools {prog} {timenow()}\n
        ") for data in flist: f.write(f'
      1. {os.path.split(data)[-1]}
      2. \n') f.write("
      ") diff --git a/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py b/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py index 90bc0586926..c9258e22375 100644 --- a/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py +++ b/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py @@ -11,7 +11,7 @@ import sys import time prog = os.path.split(sys.argv[0])[-1] -myversion = 'Oct 10 2009' +myversion = "Oct 10 2009" galhtmlprefix = """ @@ -28,17 +28,16 @@ galhtmlprefix = """ def timenow(): - """return current time as a string - """ - return time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(time.time())) + """return current time as a string""" + return time.strftime("%d/%m/%Y %H:%M:%S", time.localtime(time.time())) -def getMissval(inped=''): +def getMissval(inped=""): """ read some lines...ugly hack - try to guess missing value should be N or 0 but might be . or - """ - commonmissvals = {'N': 'N', '0': '0', 'n': 'n', '9': '9', '-': '-', '.': '.'} + commonmissvals = {"N": "N", "0": "0", "n": "n", "9": "9", "-": "-", ".": "."} try: f = open(inped) except Exception: @@ -56,24 +55,24 @@ def getMissval(inped=''): f.close() return missval if not missval: - missval = 'N' # punt + missval = "N" # punt f.close() return missval def rgConv(inpedfilepath, outhtmlname, outfilepath, plink): - """ - """ - pedf = f'{inpedfilepath}.ped' + """ """ + pedf = f"{inpedfilepath}.ped" basename = os.path.split(inpedfilepath)[-1] # get basename outroot = os.path.join(outfilepath, basename) missval = getMissval(inped=pedf) if not missval: - print(f'### lped_to_pbed_converter.py cannot identify missing value in {pedf}') - missval = '0' - subprocess.check_call([plink, '--noweb', '--file', inpedfilepath, - '--make-bed', '--out', outroot, - '--missing-genotype', missval], cwd=outfilepath) + print(f"### lped_to_pbed_converter.py cannot identify missing value in {pedf}") + missval = "0" + subprocess.check_call( + [plink, "--noweb", "--file", inpedfilepath, "--make-bed", "--out", outroot, "--missing-genotype", missval], + cwd=outfilepath, + ) def main(): @@ -86,7 +85,7 @@ def main(): """ nparm = 4 if len(sys.argv) < nparm: - sys.exit('## %s called with %s - needs %d parameters \n' % (prog, sys.argv, nparm)) + sys.exit("## %s called with %s - needs %d parameters \n" % (prog, sys.argv, nparm)) inpedfilepath = sys.argv[1] outhtmlname = sys.argv[2] outfilepath = sys.argv[3] @@ -97,11 +96,11 @@ def main(): plink = sys.argv[4] rgConv(inpedfilepath, outhtmlname, outfilepath, plink) flist = os.listdir(outfilepath) - with open(outhtmlname, 'w') as f: + with open(outhtmlname, "w") as f: f.write(galhtmlprefix % prog) - s = f'## Rgenetics: http://rgenetics.org Galaxy Tools {prog} {timenow()}' # becomes info + s = f"## Rgenetics: http://rgenetics.org Galaxy Tools {prog} {timenow()}" # becomes info print(s) - f.write(f'
      {s}\n
        ') + f.write(f"
        {s}\n
          ") for data in flist: f.write(f'
        1. {os.path.split(data)[-1]}
        2. \n') f.write("
      ") diff --git a/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py b/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py index 78906386ab4..0b968f3c753 100644 --- a/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py +++ b/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py @@ -14,7 +14,7 @@ def __main__(): output_name = sys.argv.pop(1) input_name = sys.argv.pop(1) count = 0 - with open(output_name, 'w') as out, open(input_name) as infile: + with open(output_name, "w") as out, open(input_name) as infile: for count, block in enumerate(bx.align.maf.Reader(infile)): spec_counts = {} for c in block.components: @@ -23,7 +23,14 @@ def __main__(): spec_counts[spec] = 0 else: spec_counts[spec] += 1 - out.write("%s\n" % maf_utilities.get_fasta_header(c, {'block_index': count, 'species': spec, 'sequence_index': spec_counts[spec]}, suffix="%s_%i_%i" % (spec, count, spec_counts[spec]))) + out.write( + "%s\n" + % maf_utilities.get_fasta_header( + c, + {"block_index": count, "species": spec, "sequence_index": spec_counts[spec]}, + suffix="%s_%i_%i" % (spec, count, spec_counts[spec]), + ) + ) out.write(f"{c.text}\n") out.write("\n") print("%i MAF blocks converted to FASTA." % (count)) diff --git a/lib/galaxy/datatypes/converters/maf_to_interval_converter.py b/lib/galaxy/datatypes/converters/maf_to_interval_converter.py index 92bc7c2b4e8..05c9c4b1a73 100644 --- a/lib/galaxy/datatypes/converters/maf_to_interval_converter.py +++ b/lib/galaxy/datatypes/converters/maf_to_interval_converter.py @@ -15,7 +15,7 @@ def __main__(): input_name = sys.argv.pop(1) species = sys.argv.pop(1) count = 0 - with open(output_name, 'w') as out: + with open(output_name, "w") as out: # write interval header line out.write("#chrom\tstart\tend\tstrand\n") try: @@ -23,7 +23,15 @@ def __main__(): for block in bx.align.maf.Reader(fh): for c in maf_utilities.iter_components_by_src_start(block, species): if c is not None: - out.write("%s\t%i\t%i\t%s\n" % (maf_utilities.src_split(c.src)[-1], c.get_forward_strand_start(), c.get_forward_strand_end(), c.strand)) + out.write( + "%s\t%i\t%i\t%s\n" + % ( + maf_utilities.src_split(c.src)[-1], + c.get_forward_strand_start(), + c.get_forward_strand_end(), + c.strand, + ) + ) count += 1 except Exception as e: print(f"There was a problem processing your input: {e}", file=sys.stderr) diff --git a/lib/galaxy/datatypes/converters/parquet_to_csv_converter.py b/lib/galaxy/datatypes/converters/parquet_to_csv_converter.py index d4e6969d279..6646e7a9533 100644 --- a/lib/galaxy/datatypes/converters/parquet_to_csv_converter.py +++ b/lib/galaxy/datatypes/converters/parquet_to_csv_converter.py @@ -5,6 +5,7 @@ Output: csv """ import os import sys + try: import pyarrow.csv import pyarrow.parquet diff --git a/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py b/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py index 7835d7ad96a..cffebc2d8d6 100644 --- a/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py +++ b/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py @@ -22,46 +22,56 @@ galhtmlprefix = """
      """ -plinke = 'plink' +plinke = "plink" def timenow(): - """return current time as a string - """ - return time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(time.time())) + """return current time as a string""" + return time.strftime("%d/%m/%Y %H:%M:%S", time.localtime(time.time())) -def pruneLD(plinktasks=None, cd='./', vclbase=None): - """ - """ +def pruneLD(plinktasks=None, cd="./", vclbase=None): + """ """ plinktasks = plinktasks or [] vclbase = vclbase or [] - alog = ['## Rgenetics: http://rgenetics.org Galaxy Tools rgQC.py Plink pruneLD runner\n'] - with tempfile.NamedTemporaryFile(mode='r+') as plog: + alog = ["## Rgenetics: http://rgenetics.org Galaxy Tools rgQC.py Plink pruneLD runner\n"] + with tempfile.NamedTemporaryFile(mode="r+") as plog: for task in plinktasks: # each is a list vcl = vclbase + task subprocess.check_call(vcl, stdout=plog, stderr=plog, cwd=cd) try: plog.seek(0) - lplog = [elem for elem in plog.readlines() if elem.find('Pruning SNP') == -1] + lplog = [elem for elem in plog.readlines() if elem.find("Pruning SNP") == -1] alog += lplog - alog.append('\n') + alog.append("\n") except Exception: - alog.append(f"### {timenow()} Strange - no std out from plink when running command line\n{' '.join(vcl)}\n") + alog.append( + f"### {timenow()} Strange - no std out from plink when running command line\n{' '.join(vcl)}\n" + ) return alog -def makeLDreduced(basename, infpath=None, outfpath=None, plinke='plink', forcerebuild=False, returnFname=False, - winsize="60", winmove="40", r2thresh="0.1"): - """ not there so make and leave in output dir for post job hook to copy back into input extra files path for next time - """ +def makeLDreduced( + basename, + infpath=None, + outfpath=None, + plinke="plink", + forcerebuild=False, + returnFname=False, + winsize="60", + winmove="40", + r2thresh="0.1", +): + """not there so make and leave in output dir for post job hook to copy back into input extra files path for next time""" outbase = os.path.join(outfpath, basename) inbase = os.path.join(infpath) plinktasks = [] - vclbase = [plinke, '--noweb'] - plinktasks += [['--bfile', inbase, f'--indep-pairwise {winsize} {winmove} {r2thresh}', f'--out {outbase}'], - ['--bfile', inbase, f'--extract {outbase}.prune.in --make-bed --out {outbase}']] - vclbase = [plinke, '--noweb'] + vclbase = [plinke, "--noweb"] + plinktasks += [ + ["--bfile", inbase, f"--indep-pairwise {winsize} {winmove} {r2thresh}", f"--out {outbase}"], + ["--bfile", inbase, f"--extract {outbase}.prune.in --make-bed --out {outbase}"], + ] + vclbase = [plinke, "--noweb"] pruneLD(plinktasks=plinktasks, cd=outfpath, vclbase=vclbase) @@ -79,7 +89,7 @@ def main(): """ nparm = 7 if len(sys.argv) < nparm: - sys.stderr.write('## %s called with %s - needs %d parameters \n' % (prog, sys.argv, nparm)) + sys.stderr.write("## %s called with %s - needs %d parameters \n" % (prog, sys.argv, nparm)) sys.exit(1) inpedfilepath = sys.argv[1] base_name = os.path.split(inpedfilepath)[-1] @@ -93,15 +103,24 @@ def main(): except Exception: pass plink = sys.argv[7] - makeLDreduced(base_name, infpath=inpedfilepath, outfpath=outfilepath, plinke=plink, forcerebuild=False, returnFname=False, - winsize=winsize, winmove=winmove, r2thresh=r2thresh) + makeLDreduced( + base_name, + infpath=inpedfilepath, + outfpath=outfilepath, + plinke=plink, + forcerebuild=False, + returnFname=False, + winsize=winsize, + winmove=winmove, + r2thresh=r2thresh, + ) flist = os.listdir(outfilepath) - with open(outhtmlname, 'w') as f: + with open(outhtmlname, "w") as f: f.write(galhtmlprefix % prog) - s1 = f'## Rgenetics: http://rgenetics.org Galaxy Tools {prog} {timenow()}' # becomes info - s2 = f'Input {base_name}, winsize={winsize}, winmove={winmove}, r2thresh={r2thresh}' - print(f'{s1} {s2}') - f.write(f'
      {s1}\n{s2}\n
        ') + s1 = f"## Rgenetics: http://rgenetics.org Galaxy Tools {prog} {timenow()}" # becomes info + s2 = f"Input {base_name}, winsize={winsize}, winmove={winmove}, r2thresh={r2thresh}" + print(f"{s1} {s2}") + f.write(f"
        {s1}\n{s2}\n
          ") for data in flist: f.write(f'
        1. {os.path.split(data)[-1]}
        2. \n') f.write("
        ") diff --git a/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py b/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py index d967206bff5..eee21aa85da 100644 --- a/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py +++ b/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py @@ -10,9 +10,8 @@ import subprocess import sys import time - prog = os.path.split(sys.argv[0])[-1] -myversion = 'Oct 10 2009' +myversion = "Oct 10 2009" galhtmlprefix = """ @@ -29,17 +28,15 @@ galhtmlprefix = """ def timenow(): - """return current time as a string - """ - return time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(time.time())) + """return current time as a string""" + return time.strftime("%d/%m/%Y %H:%M:%S", time.localtime(time.time())) def rgConv(inpedfilepath, outhtmlname, outfilepath, plink): - """ - """ + """ """ basename = os.path.split(inpedfilepath)[-1] # get basename outroot = os.path.join(outfilepath, basename) - subprocess.check_call([plink, '--noweb', '--bfile', inpedfilepath, '--recode', '--out', outroot], cwd=outfilepath) + subprocess.check_call([plink, "--noweb", "--bfile", inpedfilepath, "--recode", "--out", outroot], cwd=outfilepath) def main(): @@ -52,7 +49,7 @@ def main(): """ nparm = 4 if len(sys.argv) < nparm: - sys.exit('PBED to LPED converter called with %s - needs %d parameters \n' % (sys.argv, nparm)) + sys.exit("PBED to LPED converter called with %s - needs %d parameters \n" % (sys.argv, nparm)) inpedfilepath = sys.argv[1] outhtmlname = sys.argv[2] outfilepath = sys.argv[3] @@ -63,11 +60,11 @@ def main(): plink = sys.argv[4] rgConv(inpedfilepath, outhtmlname, outfilepath, plink) flist = os.listdir(outfilepath) - with open(outhtmlname, 'w') as f: + with open(outhtmlname, "w") as f: f.write(galhtmlprefix % prog) - s = f'## Rgenetics: http://bitbucket.org/rgalaxy Galaxy Tools {prog} {timenow()}' # becomes info + s = f"## Rgenetics: http://bitbucket.org/rgalaxy Galaxy Tools {prog} {timenow()}" # becomes info print(s) - f.write(f'
        {s}\n
          ') + f.write(f"
          {s}\n
            ") for data in flist: f.write(f'
          1. {os.path.split(data)[-1]}
          2. \n') f.write("
        ") diff --git a/lib/galaxy/datatypes/converters/picard_interval_list_to_bed6_converter.py b/lib/galaxy/datatypes/converters/picard_interval_list_to_bed6_converter.py index 52a16deefff..dde54d6d6ad 100644 --- a/lib/galaxy/datatypes/converters/picard_interval_list_to_bed6_converter.py +++ b/lib/galaxy/datatypes/converters/picard_interval_list_to_bed6_converter.py @@ -4,7 +4,7 @@ import sys assert sys.version_info[:2] >= (2, 6) -HEADER_STARTS_WITH = ('@') +HEADER_STARTS_WITH = "@" def __main__(): @@ -14,16 +14,16 @@ def __main__(): first_skipped_line = 0 header_lines = 0 i = 0 - with open(input_name) as fh, open(output_name, 'w') as out: + with open(input_name) as fh, open(output_name, "w") as out: for i, line in enumerate(fh): - line = line.rstrip('\r\n') + line = line.rstrip("\r\n") if line: if line.startswith(HEADER_STARTS_WITH): header_lines += 1 else: try: - elems = line.split('\t') - out.write(f'{elems[0]}\t{int(elems[1]) - 1}\t{elems[2]}\t{elems[4]}\t0\t{elems[3]}\n') + elems = line.split("\t") + out.write(f"{elems[0]}\t{int(elems[1]) - 1}\t{elems[2]}\t{elems[4]}\t0\t{elems[3]}\n") except Exception as e: print(e) skipped_lines += 1 @@ -35,7 +35,10 @@ def __main__(): first_skipped_line = i + 1 info_msg = "%i lines converted to BED. " % (i + 1 - skipped_lines) if skipped_lines > 0: - info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % (skipped_lines, first_skipped_line) + info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % ( + skipped_lines, + first_skipped_line, + ) print(info_msg) diff --git a/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py b/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py index 9847a909fa5..1366d5d15d2 100644 --- a/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py +++ b/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py @@ -29,7 +29,7 @@ def main(): index.add(chrom, start, start + 1, offset) offset += len(line) - with open(output_fname, 'wb') as out: + with open(output_fname, "wb") as out: index.write(out) diff --git a/lib/galaxy/datatypes/converters/ref_to_seq_taxonomy_converter.py b/lib/galaxy/datatypes/converters/ref_to_seq_taxonomy_converter.py index c5c978a2da3..097459048de 100644 --- a/lib/galaxy/datatypes/converters/ref_to_seq_taxonomy_converter.py +++ b/lib/galaxy/datatypes/converters/ref_to_seq_taxonomy_converter.py @@ -11,11 +11,11 @@ assert sys.version_info[:2] >= (2, 4) def __main__(): - with open(sys.argv[1]) as infile, open(sys.argv[2], 'w') as outfile: + with open(sys.argv[1]) as infile, open(sys.argv[2], "w") as outfile: for line in infile: line = line.rstrip() - if line and not line.startswith('#'): - fields = line.split('\t') + if line and not line.startswith("#"): + fields = line.split("\t") # make sure the 2nd field (taxonomy) ends with a ; outfile.write(f"{fields[0]}\t{re.sub(';$', '', fields[1])};\n") diff --git a/lib/galaxy/datatypes/converters/tabular_csv.py b/lib/galaxy/datatypes/converters/tabular_csv.py index a323385037a..4170613b879 100644 --- a/lib/galaxy/datatypes/converters/tabular_csv.py +++ b/lib/galaxy/datatypes/converters/tabular_csv.py @@ -12,9 +12,9 @@ import csv def main(): usage = "Usage: %prog [options]" parser = argparse.ArgumentParser(usage=usage) - parser.add_argument('-f', '--from-tabular', action='store_true', default=False, dest='fromtabular') - parser.add_argument('-i', '--input', type=str, dest='input') - parser.add_argument('-o', '--output', type=str, dest='output') + parser.add_argument("-f", "--from-tabular", action="store_true", default=False, dest="fromtabular") + parser.add_argument("-i", "--input", type=str, dest="input") + parser.add_argument("-o", "--output", type=str, dest="output") args = parser.parse_args() input_fname = args.input output_fname = args.output @@ -25,17 +25,17 @@ def main(): def convert_to_tsv(input_fname, output_fname): - with open(input_fname, newline="") as csvfile, open(output_fname, 'w') as ofh: + with open(input_fname, newline="") as csvfile, open(output_fname, "w") as ofh: reader = csv.reader(csvfile) for line in reader: - ofh.write('\t'.join(line) + '\n') + ofh.write("\t".join(line) + "\n") def convert_to_csv(input_fname, output_fname): - with open(input_fname) as tabfile, open(output_fname, 'w', newline='') as ofh: - writer = csv.writer(ofh, delimiter=',') + with open(input_fname) as tabfile, open(output_fname, "w", newline="") as ofh: + writer = csv.writer(ofh, delimiter=",") for line in tabfile.readlines(): - writer.writerow(line.strip().split('\t')) + writer.writerow(line.strip().split("\t")) if __name__ == "__main__": diff --git a/lib/galaxy/datatypes/converters/tabular_to_dbnsfp.py b/lib/galaxy/datatypes/converters/tabular_to_dbnsfp.py index e6545b9d0de..a8c7c06ee5a 100644 --- a/lib/galaxy/datatypes/converters/tabular_to_dbnsfp.py +++ b/lib/galaxy/datatypes/converters/tabular_to_dbnsfp.py @@ -15,9 +15,9 @@ def main(): # Read options, args. usage = "Usage: %prog [options] tabular_input_file bgzip_output_file" parser = optparse.OptionParser(usage=usage) - parser.add_option('-c', '--chr-col', type='int', default=0, dest='chrom_col') - parser.add_option('-s', '--start-col', type='int', default=1, dest='start_col') - parser.add_option('-e', '--end-col', type='int', default=1, dest='end_col') + parser.add_option("-c", "--chr-col", type="int", default=0, dest="chrom_col") + parser.add_option("-s", "--start-col", type="int", default=1, dest="start_col") + parser.add_option("-e", "--end-col", type="int", default=1, dest="end_col") (options, args) = parser.parse_args() if len(args) != 2: parser.print_usage() diff --git a/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py b/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py index 30b6330b010..853d57e0172 100644 --- a/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py +++ b/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py @@ -12,7 +12,7 @@ import bx.wiggle from galaxy.util.ucsc import ( UCSCLimitException, - UCSCOutWrapper + UCSCOutWrapper, ) @@ -23,7 +23,9 @@ def main(): out_file.write("%s\n" % "\t".join(map(str, fields))) except UCSCLimitException: # Wiggle data was truncated, at the very least need to warn the user. - sys.stderr.write('Encountered message from UCSC: "Reached output limit of 100000 data values", so be aware your data was truncated.') + sys.stderr.write( + 'Encountered message from UCSC: "Reached output limit of 100000 data values", so be aware your data was truncated.' + ) if __name__ == "__main__": diff --git a/lib/galaxy/datatypes/coverage.py b/lib/galaxy/datatypes/coverage.py index 50f7895bbcb..1d489c1fc48 100644 --- a/lib/galaxy/datatypes/coverage.py +++ b/lib/galaxy/datatypes/coverage.py @@ -18,8 +18,16 @@ class LastzCoverage(Tabular): MetadataElement(name="chromCol", default=1, desc="Chrom column", param=metadata.ColumnParameter) MetadataElement(name="positionCol", default=2, desc="Position column", param=metadata.ColumnParameter) - MetadataElement(name="forwardCol", default=3, desc="Forward or aggregate read column", param=metadata.ColumnParameter) - MetadataElement(name="reverseCol", desc="Optional reverse read column", param=metadata.ColumnParameter, optional=True, no_value=0) + MetadataElement( + name="forwardCol", default=3, desc="Forward or aggregate read column", param=metadata.ColumnParameter + ) + MetadataElement( + name="reverseCol", + desc="Optional reverse read column", + param=metadata.ColumnParameter, + optional=True, + no_value=0, + ) MetadataElement(name="columns", default=3, desc="Number of columns", readonly=True, visible=False) def get_track_resolution(self, dataset, start, end): diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index bd42a995b94..98146ac554c 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -45,16 +45,16 @@ if TYPE_CHECKING: from galaxy.model import DatasetInstance XSS_VULNERABLE_MIME_TYPES = [ - 'image/svg+xml', # Unfiltered by Galaxy and may contain JS that would be executed by some browsers. - 'application/xml', # Some browsers will evalute SVG embedded JS in such XML documents. + "image/svg+xml", # Unfiltered by Galaxy and may contain JS that would be executed by some browsers. + "application/xml", # Some browsers will evalute SVG embedded JS in such XML documents. ] -DEFAULT_MIME_TYPE = 'text/plain' # Vulnerable mime types will be replaced with this. +DEFAULT_MIME_TYPE = "text/plain" # Vulnerable mime types will be replaced with this. log = logging.getLogger(__name__) # Valid first column and strand column values vor bed, other formats -col1_startswith = ['chr', 'chl', 'groupun', 'reftig_', 'scaffold', 'super_', 'vcho'] -valid_strand = ['+', '-', '.'] +col1_startswith = ["chr", "chl", "groupun", "reftig_", "scaffold", "super_", "vcho"] +valid_strand = ["+", "-", "."] DOWNLOAD_FILENAME_PATTERN_DATASET = "Galaxy${hid}-[${name}].${ext}" DOWNLOAD_FILENAME_PATTERN_COLLECTION_ELEMENT = "Galaxy${hdca_hid}-[${hdca_name}__${element_identifier}].${ext}" @@ -68,7 +68,6 @@ class DatatypeConverterNotFoundException(Exception): class DatatypeValidation: - def __init__(self, state, message): self.state = state self.message = message @@ -101,17 +100,17 @@ def get_params_and_input_name(converter, deps, target_context=None): # Generate parameter dictionary params = {} # determine input parameter name and add to params - input_name = 'input1' + input_name = "input1" for key, value in converter.inputs.items(): if deps and value.name in deps: params[value.name] = deps[value.name] - elif value.type == 'data': + elif value.type == "data": input_name = key # add potentially required/common internal tool parameters e.g. '__job_resource' if target_context: for key, value in target_context.items(): - if key.startswith('__'): + if key.startswith("__"): params[key] = value return params, input_name @@ -120,6 +119,7 @@ class DataMeta(abc.ABCMeta): """ Metaclass for Data class. Sets up metadata spec. """ + def __init__(cls, name, bases, dict_): cls.metadata_spec = metadata.MetadataSpecCollection() for base in bases: # loop through bases (class/types) of cls @@ -144,9 +144,10 @@ class Data(metaclass=DataMeta): >>> type( DataTest.metadata_spec.test.param ) """ + edam_data = "data_0006" edam_format = "format_1915" - file_ext = 'data' + file_ext = "data" # Data is not chunkable by default. CHUNKABLE = False @@ -154,7 +155,15 @@ class Data(metaclass=DataMeta): metadata_spec: metadata.MetadataSpecCollection # Add metadata elements - MetadataElement(name="dbkey", desc="Database/Build", default="?", param=metadata.DBKeyParameter, multiple=False, optional=True, no_value="?") + MetadataElement( + name="dbkey", + desc="Database/Build", + default="?", + param=metadata.DBKeyParameter, + multiple=False, + optional=True, + no_value="?", + ) # Stores the set of display applications, and viewing methods, supported by this datatype supported_display_apps: Dict[str, Any] = {} # If False, the peek is regenerated whenever a dataset of this type is copied @@ -165,7 +174,7 @@ class Data(metaclass=DataMeta): # Composite datatypes composite_type: Optional[str] = None composite_files: Dict[str, Any] = {} - primary_file_name = 'index' + primary_file_name = "index" # Allow user to change between this datatype and others. If left to None, # datatype change is allowed if the datatype is not composite. allow_datatype_change: Optional[bool] = None @@ -199,10 +208,10 @@ class Data(metaclass=DataMeta): def get_raw_data(self, dataset): """Returns the full data. To stream it open the file_name and read/write as needed""" try: - return open(dataset.file_name, 'rb').read(-1) + return open(dataset.file_name, "rb").read(-1) except OSError: - log.exception('%s reading a file that does not exist %s', self.__class__.__name__, dataset.file_name) - return '' + log.exception("%s reading a file that does not exist %s", self.__class__.__name__, dataset.file_name) + return "" def dataset_content_needs_grooming(self, file_name): """This function is called on an output dataset file after the content is initially generated.""" @@ -267,11 +276,11 @@ class Data(metaclass=DataMeta): Set the peek and blurb text """ if not dataset.dataset.purged: - dataset.peek = '' - dataset.blurb = 'data' + dataset.peek = "" + dataset.blurb = "data" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): """Create HTML table, used for displaying peek""" @@ -286,7 +295,7 @@ class Data(metaclass=DataMeta): if not line: continue out.append(f"{escape(unicodify(line, 'utf-8'))}") - out.append('') + out.append("") return "".join(out) except Exception as exc: return f"Can't create peek: {unicodify(exc)}" @@ -301,7 +310,7 @@ class Data(metaclass=DataMeta): Returns a tuple of boolean, string, string: (error, msg, messagetype) """ error, msg, messagetype = False, "", "" - archname = f'{display_name}.html' # fake the real nature of the html file + archname = f"{display_name}.html" # fake the real nature of the html file try: archive.write(data_filename, archname) except OSError: @@ -311,17 +320,17 @@ class Data(metaclass=DataMeta): messagetype = "error" return error, msg, messagetype - def _archive_composite_dataset(self, trans, data, headers: Headers, do_action='zip'): + def _archive_composite_dataset(self, trans, data, headers: Headers, do_action="zip"): # save a composite object into a compressed archive for downloading outfname = data.name[0:150] - outfname = ''.join(c in FILENAME_VALID_CHARS and c or '_' for c in outfname) + outfname = "".join(c in FILENAME_VALID_CHARS and c or "_" for c in outfname) archive = ZipstreamWrapper( archive_name=outfname, upstream_mod_zip=trans.app.config.upstream_mod_zip, - upstream_gzip=trans.app.config.upstream_gzip + upstream_gzip=trans.app.config.upstream_gzip, ) error = False - msg = '' + msg = "" ext = data.extension path = data.file_name efp = data.extra_files_path @@ -329,7 +338,7 @@ class Data(metaclass=DataMeta): display_name = os.path.splitext(outfname)[0] if not display_name.endswith(ext): - display_name = f'{display_name}_{ext}' + display_name = f"{display_name}_{ext}" error, msg = self._archive_main_file(archive, display_name, path)[:2] if not error: @@ -356,11 +365,19 @@ class Data(metaclass=DataMeta): yield fpath, rpath def _serve_raw(self, dataset, to_ext, headers: Headers, **kwd): - headers['Content-Length'] = str(os.stat(dataset.file_name).st_size) - headers["content-type"] = "application/octet-stream" # force octet-stream so Safari doesn't append mime extensions to filename - filename = self._download_filename(dataset, to_ext, hdca=kwd.get("hdca"), element_identifier=kwd.get("element_identifier"), filename_pattern=kwd.get("filename_pattern")) + headers["Content-Length"] = str(os.stat(dataset.file_name).st_size) + headers[ + "content-type" + ] = "application/octet-stream" # force octet-stream so Safari doesn't append mime extensions to filename + filename = self._download_filename( + dataset, + to_ext, + hdca=kwd.get("hdca"), + element_identifier=kwd.get("element_identifier"), + filename_pattern=kwd.get("filename_pattern"), + ) headers["Content-Disposition"] = f'attachment; filename="{filename}"' - return open(dataset.file_name, mode='rb'), headers + return open(dataset.file_name, mode="rb"), headers def to_archive(self, dataset, name=""): """ @@ -372,7 +389,7 @@ class Data(metaclass=DataMeta): """ rel_paths = [] file_paths = [] - if dataset.datatype.composite_type or dataset.extension.endswith('html'): + if dataset.datatype.composite_type or dataset.extension.endswith("html"): main_file = f"{name}.html" rel_paths.append(main_file) file_paths.append(dataset.file_name) @@ -397,10 +414,10 @@ class Data(metaclass=DataMeta): headers = kwd.get("headers", {}) # Relocate all composite datatype display to a common location. composite_extensions = trans.app.datatypes_registry.get_composite_extensions() - composite_extensions.append('html') # for archiving composite datatypes + composite_extensions.append("html") # for archiving composite datatypes # Prevent IE8 from sniffing content type since we're explicit about it. This prevents intentionally text/plain # content from being rendered in the browser - headers['X-Content-Type-Options'] = 'nosniff' + headers["X-Content-Type-Options"] = "nosniff" if isinstance(data, str): return smart_str(data), headers if filename and filename != "index": @@ -409,17 +426,22 @@ class Data(metaclass=DataMeta): file_path = trans.app.object_store.get_filename(data.dataset, extra_dir=extra_dir, alt_name=filename) if os.path.exists(file_path): if os.path.isdir(file_path): - with tempfile.NamedTemporaryFile(mode='w', delete=False, dir=trans.app.config.new_file_path, prefix='gx_html_autocreate_') as tmp_fh: + with tempfile.NamedTemporaryFile( + mode="w", delete=False, dir=trans.app.config.new_file_path, prefix="gx_html_autocreate_" + ) as tmp_fh: tmp_file_name = tmp_fh.name dir_items = sorted(os.listdir(file_path)) base_path, item_name = os.path.split(file_path) - tmp_fh.write('

        Directory %s contents: %d items

        \n' % (escape(item_name), len(dir_items))) + tmp_fh.write( + "

        Directory %s contents: %d items

        \n" + % (escape(item_name), len(dir_items)) + ) tmp_fh.write('

        \n') for index, fname in enumerate(dir_items): if index % 2 == 0: - bgcolor = '#D8D8D8' + bgcolor = "#D8D8D8" else: - bgcolor = '#FFFFFF' + bgcolor = "#FFFFFF" # Can't have an href link here because there is no route # defined for files contained within multiple subdirectory # levels of the primary dataset. Something like this is @@ -428,7 +450,7 @@ class Data(metaclass=DataMeta): # dataset_id=trans.security.encode_id(data.dataset.id), # preview=preview, filename=fname, to_ext=to_ext) tmp_fh.write(f'\n') - tmp_fh.write('
        {escape(fname)}
        \n') + tmp_fh.write("\n") return self._yield_user_file_content(trans, data, tmp_file_name, headers), headers mime = mimetypes.guess_type(file_path)[0] if not mime: @@ -449,34 +471,39 @@ class Data(metaclass=DataMeta): text, ) - if to_ext or isinstance( - data.datatype, binary.Binary - ): # Saving the file, or binary file + if to_ext or isinstance(data.datatype, binary.Binary): # Saving the file, or binary file if data.extension in composite_extensions: - return self._archive_composite_dataset(trans, data, headers, do_action=kwd.get('do_action', 'zip')) + return self._archive_composite_dataset(trans, data, headers, do_action=kwd.get("do_action", "zip")) else: - headers['Content-Length'] = str(os.stat(data.file_name).st_size) - filename = self._download_filename(data, to_ext, hdca=kwd.get("hdca"), element_identifier=kwd.get("element_identifier"), filename_pattern=kwd.get("filename_pattern")) - headers['content-type'] = "application/octet-stream" # force octet-stream so Safari doesn't append mime extensions to filename + headers["Content-Length"] = str(os.stat(data.file_name).st_size) + filename = self._download_filename( + data, + to_ext, + hdca=kwd.get("hdca"), + element_identifier=kwd.get("element_identifier"), + filename_pattern=kwd.get("filename_pattern"), + ) + headers[ + "content-type" + ] = "application/octet-stream" # force octet-stream so Safari doesn't append mime extensions to filename headers["Content-Disposition"] = f'attachment; filename="{filename}"' - return open(data.file_name, 'rb'), headers + return open(data.file_name, "rb"), headers if not os.path.exists(data.file_name): raise ObjectNotFound(f"File Not Found ({data.file_name}).") max_peek_size = DEFAULT_MAX_PEEK_SIZE # 1 MB if isinstance(data.datatype, text.Html): max_peek_size = 10000000 # 10 MB for html preview = util.string_as_bool(preview) - if ( - not preview - or isinstance(data.datatype, images.Image) - or os.stat(data.file_name).st_size < max_peek_size - ): + if not preview or isinstance(data.datatype, images.Image) or os.stat(data.file_name).st_size < max_peek_size: return self._yield_user_file_content(trans, data, data.file_name, headers), headers else: headers["content-type"] = "text/html" - return trans.fill_template_mako("/dataset/large_file.mako", - truncated_data=open(data.file_name, 'rb').read(max_peek_size), - data=data), headers + return ( + trans.fill_template_mako( + "/dataset/large_file.mako", truncated_data=open(data.file_name, "rb").read(max_peek_size), data=data + ), + headers, + ) def display_as_markdown(self, dataset_instance, markdown_format_helpers): """Prepare for embedding dataset into a basic Markdown document. @@ -494,7 +521,7 @@ class Data(metaclass=DataMeta): If the data cannot reasonably be displayed, just indicate this and do not throw an exception. """ - if self.file_ext in {'png', 'jpg'}: + if self.file_ext in {"png", "jpg"}: return self.handle_dataset_as_image(dataset_instance) if self.is_binary: result = "*cannot display binary content*\n" @@ -512,20 +539,22 @@ class Data(metaclass=DataMeta): # Sanitize anytime we respond with plain text/html content. # Check to see if this dataset's parent job is allowlisted # We cannot currently trust imported datasets for rendering. - if not from_dataset.creating_job.imported and from_dataset.creating_job.tool_id.startswith(tuple(trans.app.config.sanitize_allowlist)): - return open(filename, mode='rb') + if not from_dataset.creating_job.imported and from_dataset.creating_job.tool_id.startswith( + tuple(trans.app.config.sanitize_allowlist) + ): + return open(filename, mode="rb") # This is returning to the browser, it needs to be encoded. # TODO Ideally this happens a layer higher, but this is a bad # issue affecting many tools with open(filename) as f: - return sanitize_html(f.read()).encode('utf-8') + return sanitize_html(f.read()).encode("utf-8") - return open(filename, mode='rb') + return open(filename, mode="rb") def _download_filename(self, dataset, to_ext, hdca=None, element_identifier=None, filename_pattern=None): def escape(raw_identifier): - return ''.join(c in FILENAME_VALID_CHARS and c or '_' for c in raw_identifier)[0:150] + return "".join(c in FILENAME_VALID_CHARS and c or "_" for c in raw_identifier)[0:150] if not to_ext or to_ext == "data": # If a client requests to_ext with the extension 'data', they are @@ -555,7 +584,7 @@ class Data(metaclass=DataMeta): def display_name(self, dataset): """Returns formatted html of dataset name""" try: - return escape(unicodify(dataset.name, 'utf-8')) + return escape(unicodify(dataset.name, "utf-8")) except Exception: return "name unavailable" @@ -564,14 +593,14 @@ class Data(metaclass=DataMeta): try: # Change new line chars to html info: str = escape(dataset.info) - if info.find('\r\n') >= 0: - info = info.replace('\r\n', '
        ') - if info.find('\r') >= 0: - info = info.replace('\r', '
        ') - if info.find('\n') >= 0: - info = info.replace('\n', '
        ') + if info.find("\r\n") >= 0: + info = info.replace("\r\n", "
        ") + if info.find("\r") >= 0: + info = info.replace("\r", "
        ") + if info.find("\n") >= 0: + info = info.replace("\n", "
        ") - info = unicodify(info, 'utf-8') + info = unicodify(info, "utf-8") return info except Exception: @@ -583,7 +612,7 @@ class Data(metaclass=DataMeta): def get_mime(self): """Returns the mime type of the datatype""" - return 'application/octet-stream' + return "application/octet-stream" def add_display_app(self, app_id, label, file_function, links_function): """ @@ -594,7 +623,11 @@ class Data(metaclass=DataMeta): links_function is a string containing the name of the function that returns a list of (link_name,link) """ self.supported_display_apps = self.supported_display_apps.copy() - self.supported_display_apps[app_id] = {'label': label, 'file_function': file_function, 'links_function': links_function} + self.supported_display_apps[app_id] = { + "label": label, + "file_function": file_function, + "links_function": links_function, + } def remove_display_app(self, app_id): """Removes a display app from the datatype""" @@ -602,14 +635,18 @@ class Data(metaclass=DataMeta): try: del self.supported_display_apps[app_id] except Exception: - log.exception('Tried to remove display app %s from datatype %s, but this display app is not declared.', type, self.__class__.__name__) + log.exception( + "Tried to remove display app %s from datatype %s, but this display app is not declared.", + type, + self.__class__.__name__, + ) def clear_display_apps(self): self.supported_display_apps = {} def add_display_application(self, display_application): """New style display applications""" - assert display_application.id not in self.display_applications, 'Attempted to add a display application twice' + assert display_application.id not in self.display_applications, "Attempted to add a display application twice" self.display_applications[display_application.id] = display_application def get_display_application(self, key, default=None): @@ -630,20 +667,25 @@ class Data(metaclass=DataMeta): def get_display_label(self, type): """Returns primary label for display app""" try: - return self.supported_display_apps[type]['label'] + return self.supported_display_apps[type]["label"] except Exception: - return 'unknown' + return "unknown" def as_display_type(self, dataset, type, **kwd): - """Returns modified file contents for a particular display type """ + """Returns modified file contents for a particular display type""" try: if type in self.get_display_types(): - return getattr(self, self.supported_display_apps[type]['file_function'])(dataset, **kwd) + return getattr(self, self.supported_display_apps[type]["file_function"])(dataset, **kwd) except Exception: - log.exception('Function %s is referred to in datatype %s for displaying as type %s, but is not accessible', self.supported_display_apps[type]['file_function'], self.__class__.__name__, type) + log.exception( + "Function %s is referred to in datatype %s for displaying as type %s, but is not accessible", + self.supported_display_apps[type]["file_function"], + self.__class__.__name__, + type, + ) return f"This display type ({type}) is not implemented for this datatype ({dataset.ext})." - def get_display_links(self, dataset, type, app, base_url, target_frame='_blank', **kwd): + def get_display_links(self, dataset, type, app, base_url, target_frame="_blank", **kwd): """ Returns a list of tuples of (name, link) for a particular display type. No check on 'access' permissions is done here - if you can view the dataset, you can also save it @@ -652,10 +694,16 @@ class Data(metaclass=DataMeta): """ try: if app.config.enable_old_display_applications and type in self.get_display_types(): - return target_frame, getattr(self, self.supported_display_apps[type]['links_function'])(dataset, type, app, base_url, **kwd) + return target_frame, getattr(self, self.supported_display_apps[type]["links_function"])( + dataset, type, app, base_url, **kwd + ) except Exception: - log.exception('Function %s is referred to in datatype %s for generating links for type %s, but is not accessible', - self.supported_display_apps[type]['links_function'], self.__class__.__name__, type) + log.exception( + "Function %s is referred to in datatype %s for generating links for type %s, but is not accessible", + self.supported_display_apps[type]["links_function"], + self.__class__.__name__, + type, + ) return target_frame, [] def get_converter_types(self, original_dataset, datatypes_registry): @@ -666,20 +714,34 @@ class Data(metaclass=DataMeta): self, dataset, accepted_formats: List[str], datatypes_registry, **kwd ) -> Tuple[bool, Optional[str], Optional["DatasetInstance"]]: """Returns ( direct_match, converted_ext, existing converted dataset )""" - return datatypes_registry.find_conversion_destination_for_dataset_by_extensions(dataset, accepted_formats, **kwd) + return datatypes_registry.find_conversion_destination_for_dataset_by_extensions( + dataset, accepted_formats, **kwd + ) - def convert_dataset(self, trans, original_dataset, target_type, return_output=False, visible=True, deps=None, target_context=None, history=None): + def convert_dataset( + self, + trans, + original_dataset, + target_type, + return_output=False, + visible=True, + deps=None, + target_context=None, + history=None, + ): """This function adds a job to the queue to convert a dataset to another type. Returns a message about success/failure.""" converter = trans.app.datatypes_registry.get_converter_by_target_type(original_dataset.ext, target_type) if converter is None: - raise DatatypeConverterNotFoundException(f"A converter does not exist for {original_dataset.ext} to {target_type}.") + raise DatatypeConverterNotFoundException( + f"A converter does not exist for {original_dataset.ext} to {target_type}." + ) params, input_name = get_params_and_input_name(converter, deps, target_context) params[input_name] = original_dataset # Make the target datatype available to the converter - params['__target_datatype__'] = target_type + params["__target_datatype__"] = target_type # Run converter, job is dispatched through Queue job, converted_datasets, *_ = converter.execute(trans, incoming=params, set_output_hid=visible, history=history) trans.app.job_manager.enqueue(job, tool=converter) @@ -703,15 +765,26 @@ class Data(metaclass=DataMeta): """This function is called on the dataset before metadata is set.""" dataset.clear_associated_files(metadata_safe=True) - def __new_composite_file(self, name, optional=False, mimetype=None, description=None, substitute_name_with_metadata=None, is_binary=False, to_posix_lines=True, space_to_tab=False, **kwds): - kwds['name'] = name - kwds['optional'] = optional - kwds['mimetype'] = mimetype - kwds['description'] = description - kwds['substitute_name_with_metadata'] = substitute_name_with_metadata - kwds['is_binary'] = is_binary - kwds['to_posix_lines'] = to_posix_lines - kwds['space_to_tab'] = space_to_tab + def __new_composite_file( + self, + name, + optional=False, + mimetype=None, + description=None, + substitute_name_with_metadata=None, + is_binary=False, + to_posix_lines=True, + space_to_tab=False, + **kwds, + ): + kwds["name"] = name + kwds["optional"] = optional + kwds["mimetype"] = mimetype + kwds["description"] = description + kwds["substitute_name_with_metadata"] = substitute_name_with_metadata + kwds["is_binary"] = is_binary + kwds["to_posix_lines"] = to_posix_lines + kwds["space_to_tab"] = space_to_tab return Bunch(**kwds) def add_composite_file(self, name, **kwds): @@ -730,7 +803,7 @@ class Data(metaclass=DataMeta): @property def writable_files(self): files = {} - if self.composite_type != 'auto_primary_file': + if self.composite_type != "auto_primary_file": files[self.primary_file_name] = self.__new_composite_file(self.primary_file_name) for key, value in self.get_composite_files().items(): files[key] = value @@ -745,6 +818,7 @@ class Data(metaclass=DataMeta): meta_value = self.metadata_spec[composite_file.substitute_name_with_metadata].default return key % meta_value return key + files = {} for key, value in self.composite_files.items(): files[substitute_composite_key(key, value)] = value @@ -768,17 +842,17 @@ class Data(metaclass=DataMeta): @staticmethod def merge(split_files, output_file): """ - Merge files with copy.copyfileobj() will not hit the - max argument limitation of cat. gz and bz2 files are also working. + Merge files with copy.copyfileobj() will not hit the + max argument limitation of cat. gz and bz2 files are also working. """ if not split_files: - raise ValueError(f'Asked to merge zero files as {output_file}') + raise ValueError(f"Asked to merge zero files as {output_file}") elif len(split_files) == 1: - shutil.copyfileobj(open(split_files[0], 'rb'), open(output_file, 'wb')) + shutil.copyfileobj(open(split_files[0], "rb"), open(output_file, "wb")) else: - with open(output_file, 'wb') as fdst: + with open(output_file, "wb") as fdst: for fsrc in split_files: - shutil.copyfileobj(open(fsrc, 'rb'), fdst) + shutil.copyfileobj(open(fsrc, "rb"), fdst) def get_visualizations(self, dataset): """ @@ -786,7 +860,7 @@ class Data(metaclass=DataMeta): """ if self.track_type: - return ['trackster', 'circster'] + return ["trackster", "circster"] return [] # ------------- Dataproviders @@ -813,16 +887,12 @@ class Data(metaclass=DataMeta): dataset_source = p_dataproviders.dataset.DatasetDataProvider(dataset) return p_dataproviders.base.DataProvider(dataset_source, **settings) - @p_dataproviders.decorators.dataprovider_factory( - "chunk", p_dataproviders.chunk.ChunkDataProvider.settings - ) + @p_dataproviders.decorators.dataprovider_factory("chunk", p_dataproviders.chunk.ChunkDataProvider.settings) def chunk_dataprovider(self, dataset, **settings): dataset_source = p_dataproviders.dataset.DatasetDataProvider(dataset) return p_dataproviders.chunk.ChunkDataProvider(dataset_source, **settings) - @p_dataproviders.decorators.dataprovider_factory( - "chunk64", p_dataproviders.chunk.Base64ChunkDataProvider.settings - ) + @p_dataproviders.decorators.dataprovider_factory("chunk64", p_dataproviders.chunk.Base64ChunkDataProvider.settings) def chunk64_dataprovider(self, dataset, **settings): dataset_source = p_dataproviders.dataset.DatasetDataProvider(dataset) return p_dataproviders.chunk.Base64ChunkDataProvider(dataset_source, **settings) @@ -840,17 +910,25 @@ class Data(metaclass=DataMeta): @p_dataproviders.decorators.has_dataproviders class Text(Data): edam_format = "format_2330" - file_ext = 'txt' - line_class = 'line' + file_ext = "txt" + line_class = "line" is_binary = False # Add metadata elements - MetadataElement(name="data_lines", default=0, desc="Number of data lines", readonly=True, optional=True, visible=False, no_value=0) + MetadataElement( + name="data_lines", + default=0, + desc="Number of data lines", + readonly=True, + optional=True, + visible=False, + no_value=0, + ) def get_mime(self): """Returns the mime type of the datatype""" - return 'text/plain' + return "text/plain" def set_meta(self, dataset, **kwd): """ @@ -866,10 +944,10 @@ class Text(Data): try: with compression_utils.get_fileobj(dataset.file_name) as dataset_fh: dataset_read = dataset_fh.read(sample_size) - sample_lines = dataset_read.count('\n') + sample_lines = dataset_read.count("\n") return int(sample_lines * (float(dataset.get_size()) / float(sample_size))) except UnicodeDecodeError: - log.error(f'Unable to estimate lines in file {dataset.file_name}') + log.error(f"Unable to estimate lines in file {dataset.file_name}") return None def count_data_lines(self, dataset): @@ -877,7 +955,7 @@ class Text(Data): Count the number of lines of data in dataset, skipping all blank lines and comments. """ - CHUNK_SIZE = 2 ** 15 # 32Kb + CHUNK_SIZE = 2**15 # 32Kb data_lines = 0 with compression_utils.get_fileobj(dataset.file_name) as in_file: # FIXME: Potential encoding issue can prevent the ability to iterate over lines @@ -886,10 +964,10 @@ class Text(Data): try: for line in iter_start_of_line(in_file, CHUNK_SIZE): line = line.strip() - if line and not line.startswith('#'): + if line and not line.startswith("#"): data_lines += 1 except UnicodeDecodeError: - log.error(f'Unable to count lines in file {dataset.file_name}') + log.error(f"Unable to count lines in file {dataset.file_name}") return None return data_lines @@ -925,8 +1003,8 @@ class Text(Data): else: dataset.blurb = f"{util.commaify(str(line_count))} {inflector.cond_plural(line_count, self.line_class)}" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" @classmethod def split(cls, input_datasets, subdir_generator_function, split_params): @@ -942,15 +1020,16 @@ class Text(Data): lines_per_file = None chunk_size = None - if split_params['split_mode'] == 'number_of_parts': + if split_params["split_mode"] == "number_of_parts": lines_per_file = [] # Computing the length is expensive! def _file_len(fname): with open(fname) as f: return sum(1 for _ in f) + length = _file_len(input_files[0]) - parts = int(split_params['split_size']) + parts = int(split_params["split_size"]) if length < parts: parts = length len_each, remainder = divmod(length, parts) @@ -961,8 +1040,8 @@ class Text(Data): lines_per_file.append(chunk) remainder -= 1 length -= chunk - elif split_params['split_mode'] == 'to_size': - chunk_size = int(split_params['split_size']) + elif split_params["split_mode"] == "to_size": + chunk_size = int(split_params["split_size"]) else: raise Exception(f"Unsupported split mode {split_params['split_mode']}") @@ -981,17 +1060,17 @@ class Text(Data): part_file = None while lines_remaining > 0: a_line = f.readline() - if a_line == '': + if a_line == "": file_done = True break if part_file is None: part_dir = subdir_generator_function() part_path = os.path.join(part_dir, os.path.basename(input_files[0])) - part_file = open(part_path, 'w') + part_file = open(part_path, "w") part_file.write(a_line) lines_remaining -= 1 except Exception as e: - log.error('Unable to split files: %s', unicodify(e)) + log.error("Unable to split files: %s", unicodify(e)) raise finally: f.close() @@ -999,9 +1078,7 @@ class Text(Data): part_file.close() # ------------- Dataproviders - @p_dataproviders.decorators.dataprovider_factory( - "line", p_dataproviders.line.FilteredLineDataProvider.settings - ) + @p_dataproviders.decorators.dataprovider_factory("line", p_dataproviders.line.FilteredLineDataProvider.settings) def line_dataprovider(self, dataset, **settings): """ Returns an iterator over the dataset's lines (that have been stripped) @@ -1010,9 +1087,7 @@ class Text(Data): dataset_source = p_dataproviders.dataset.DatasetDataProvider(dataset) return p_dataproviders.line.FilteredLineDataProvider(dataset_source, **settings) - @p_dataproviders.decorators.dataprovider_factory( - "regex-line", p_dataproviders.line.RegexLineDataProvider.settings - ) + @p_dataproviders.decorators.dataprovider_factory("regex-line", p_dataproviders.line.RegexLineDataProvider.settings) def regex_line_dataprovider(self, dataset, **settings): """ Returns an iterator over the dataset's lines @@ -1028,9 +1103,10 @@ class Directory(Data): class GenericAsn1(Text): """Class for generic ASN.1 text format""" + edam_data = "data_0849" edam_format = "format_1966" - file_ext = 'asn1' + file_ext = "asn1" class LineCount(Text): @@ -1042,24 +1118,26 @@ class LineCount(Text): class Newick(Text): """New Hampshire/Newick Format""" + edam_data = "data_0872" edam_format = "format_1910" file_ext = "newick" def sniff(self, filename): - """ Returning false as the newick format is too general and cannot be sniffed.""" + """Returning false as the newick format is too general and cannot be sniffed.""" return False def get_visualizations(self, dataset): """ Returns a list of visualizations for datatype. """ - return ['phyloviz'] + return ["phyloviz"] @build_sniff_from_prefix class Nexus(Text): """Nexus format as used By Paup, Mr Bayes, etc""" + edam_data = "data_0872" edam_format = "format_1912" file_ext = "nex" @@ -1072,7 +1150,7 @@ class Nexus(Text): """ Returns a list of visualizations for datatype. """ - return ['phyloviz'] + return ["phyloviz"] # ------------- Utility methods -------------- @@ -1086,7 +1164,7 @@ nice_size = util.nice_size def get_test_fname(fname): """Returns test data filename""" path = os.path.dirname(__file__) - full_path = os.path.join(path, 'test', fname) + full_path = os.path.join(path, "test", fname) return full_path @@ -1106,7 +1184,7 @@ def get_file_peek(file_name, WIDTH=256, LINE_COUNT=5, skipchars=None, line_wrap= # Set size for file.readline() to a negative number to force it to # read until either a newline or EOF. Needed for datasets with very # long lines. - if WIDTH == 'unlimited': + if WIDTH == "unlimited": WIDTH = -1 if skipchars is None: skipchars = [] @@ -1123,14 +1201,14 @@ def get_file_peek(file_name, WIDTH=256, LINE_COUNT=5, skipchars=None, line_wrap= if line == "": break last_line_break = False - if line.endswith('\n'): + if line.endswith("\n"): line = line[:-1] last_line_break = True elif not line_wrap: for i in file_reader(temp, 1): - if i == '\n': + if i == "\n": last_line_break = True - if not i or i == '\n': + if not i or i == "\n": break skip_line = False for skipchar in skipchars: @@ -1140,4 +1218,4 @@ def get_file_peek(file_name, WIDTH=256, LINE_COUNT=5, skipchars=None, line_wrap= if not skip_line: lines.append(line) count += 1 - return '\n'.join(lines) + ('\n' if last_line_break else '') + return "\n".join(lines) + ("\n" if last_line_break else "") diff --git a/lib/galaxy/datatypes/dataproviders/__init__.py b/lib/galaxy/datatypes/dataproviders/__init__.py index ee258fd5713..b18734eda2c 100644 --- a/lib/galaxy/datatypes/dataproviders/__init__.py +++ b/lib/galaxy/datatypes/dataproviders/__init__.py @@ -24,7 +24,7 @@ from . import ( exceptions, external, hierarchy, - line + line, ) -__all__ = ('decorators', 'exceptions', 'base', 'chunk', 'line', 'hierarchy', 'column', 'external', 'dataset') +__all__ = ("decorators", "exceptions", "base", "chunk", "line", "hierarchy", "column", "external", "dataset") diff --git a/lib/galaxy/datatypes/dataproviders/base.py b/lib/galaxy/datatypes/dataproviders/base.py index 3d0a536b8e0..22dbb6abcb1 100644 --- a/lib/galaxy/datatypes/dataproviders/base.py +++ b/lib/galaxy/datatypes/dataproviders/base.py @@ -46,19 +46,20 @@ class HasSettings(type): Useful for allowing class level access to expected variable types passed to class `__init__` functions so they can be parsed from a query string. """ + # yeah - this is all too acrobatic def __new__(cls, name, base_classes, attributes): settings = {} # get settings defined in base classes for base_class in base_classes: - base_settings = getattr(base_class, 'settings', None) + base_settings = getattr(base_class, "settings", None) if base_settings: settings.update(base_settings) # get settings defined in this class - new_settings = attributes.pop('settings', None) + new_settings = attributes.pop("settings", None) if new_settings: settings.update(new_settings) - attributes['settings'] = settings + attributes["settings"] = settings return type.__new__(cls, name, base_classes, attributes) @@ -72,6 +73,7 @@ class DataProvider(metaclass=HasSettings): - do not allow write methods (but otherwise implement the other file object interface methods) """ + # a definition of expected types for keyword arguments sent to __init__ # useful for controlling how query string dictionaries can be parsed into correct types for __init__ # empty in this base class @@ -95,7 +97,7 @@ class DataProvider(metaclass=HasSettings): Meant to be overridden in subclasses. """ - if not source or not hasattr(source, '__iter__'): + if not source or not hasattr(source, "__iter__"): # that's by no means a thorough check raise exceptions.InvalidDataProviderSource(source) return source @@ -103,7 +105,7 @@ class DataProvider(metaclass=HasSettings): # TODO: (this might cause problems later...) # TODO: some providers (such as chunk's seek and read) rely on this... remove def __getattr__(self, name): - if name == 'source': + if name == "source": # if we're inside this fn, source hasn't been set - provide some safety just for this attr return None # otherwise, try to get the attr from the source - allows us to get things like provider.encoding, etc. @@ -114,13 +116,13 @@ class DataProvider(metaclass=HasSettings): # write methods should not be allowed def truncate(self, size): - raise NotImplementedError('Write methods are purposely disabled') + raise NotImplementedError("Write methods are purposely disabled") def write(self, string): - raise NotImplementedError('Write methods are purposely disabled') + raise NotImplementedError("Write methods are purposely disabled") def writelines(self, sequence): - raise NotImplementedError('Write methods are purposely disabled') + raise NotImplementedError("Write methods are purposely disabled") # TODO: route read methods through next? # def readline( self ): @@ -140,16 +142,16 @@ class DataProvider(metaclass=HasSettings): # context manager interface def __enter__(self): # make the source's context manager interface optional - if hasattr(self.source, '__enter__'): + if hasattr(self.source, "__enter__"): self.source.__enter__() return self def __exit__(self, *args): # make the source's context manager interface optional, call on source if there - if hasattr(self.source, '__exit__'): + if hasattr(self.source, "__exit__"): self.source.__exit__(*args) # alternately, call close() - elif hasattr(self.source, 'close'): + elif hasattr(self.source, "close"): self.source.close() def __str__(self): @@ -159,8 +161,8 @@ class DataProvider(metaclass=HasSettings): Will call `__str__` on its source so this will display piped dataproviders. """ # we need to protect against recursion (in __getattr__) if self.source hasn't been set - source_str = str(self.source) if hasattr(self, 'source') else '' - return f'{self.__class__.__name__}({str(source_str)})' + source_str = str(self.source) if hasattr(self, "source") else "" + return f"{self.__class__.__name__}({str(source_str)})" class FilteredDataProvider(DataProvider): @@ -173,6 +175,7 @@ class FilteredDataProvider(DataProvider): - `num_valid_data_read`: how many data have been returned from `filter`. - `num_data_returned`: how many data has this provider yielded. """ + # not useful here - we don't want functions over the query string # settings.update({ 'filter_fn': 'function' }) @@ -225,11 +228,9 @@ class LimitedOffsetDataProvider(FilteredDataProvider): Useful for grabbing sections from a source (e.g. pagination). """ + # define the expected types of these __init__ arguments so they can be parsed out from query strings - settings = { - 'limit': 'int', - 'offset': 'int' - } + settings = {"limit": "int", "offset": "int"} # TODO: may want to squash this into DataProvider def __init__(self, source, offset=0, limit=None, **kwargs): diff --git a/lib/galaxy/datatypes/dataproviders/chunk.py b/lib/galaxy/datatypes/dataproviders/chunk.py index 16aa2016182..66cd3540ac0 100644 --- a/lib/galaxy/datatypes/dataproviders/chunk.py +++ b/lib/galaxy/datatypes/dataproviders/chunk.py @@ -10,7 +10,7 @@ import os from . import ( base, - exceptions + exceptions, ) log = logging.getLogger(__name__) @@ -22,12 +22,10 @@ class ChunkDataProvider(base.DataProvider): Note: this version does not account for lines and works with Binary datatypes. """ - MAX_CHUNK_SIZE = 2 ** 16 + + MAX_CHUNK_SIZE = 2**16 DEFAULT_CHUNK_SIZE = MAX_CHUNK_SIZE - settings = { - 'chunk_index': 'int', - 'chunk_size': 'int' - } + settings = {"chunk_index": "int", "chunk_size": "int"} # TODO: subclass from LimitedOffsetDataProvider? # see web/framework/base.iterate_file, util/__init__.file_reader, and datatypes.tabular @@ -49,7 +47,7 @@ class ChunkDataProvider(base.DataProvider): :raises InvalidDataProviderSource: if not. """ source = super().validate_source(source) - if((not hasattr(source, 'seek')) or (not hasattr(source, 'read'))): + if (not hasattr(source, "seek")) or (not hasattr(source, "read")): raise exceptions.InvalidDataProviderSource(source) return source diff --git a/lib/galaxy/datatypes/dataproviders/column.py b/lib/galaxy/datatypes/dataproviders/column.py index 6a94597bcf4..34cee4b8d76 100644 --- a/lib/galaxy/datatypes/dataproviders/column.py +++ b/lib/galaxy/datatypes/dataproviders/column.py @@ -31,18 +31,28 @@ class ColumnarDataProvider(line.RegexLineDataProvider): the same number of columns as the number of indeces asked for (even if they are filled with None). """ + settings = { - 'indeces': 'list:int', - 'column_count': 'int', - 'column_types': 'list:str', - 'parse_columns': 'bool', - 'deliminator': 'str', - 'filters': 'list:str' + "indeces": "list:int", + "column_count": "int", + "column_types": "list:str", + "parse_columns": "bool", + "deliminator": "str", + "filters": "list:str", } - def __init__(self, source, indeces=None, - column_count=None, column_types=None, parsers=None, parse_columns=True, - deliminator='\t', filters=None, **kwargs): + def __init__( + self, + source, + indeces=None, + column_count=None, + column_types=None, + parsers=None, + parse_columns=True, + deliminator="\t", + filters=None, + **kwargs, + ): """ :param indeces: a list of indeces of columns to gather from each row @@ -114,7 +124,7 @@ class ColumnarDataProvider(line.RegexLineDataProvider): self.column_filters.append(parsed) def parse_filter(self, filter_param_str): - split = filter_param_str.split('-', 2) + split = filter_param_str.split("-", 2) if not len(split) >= 3: return None column, op, val = split @@ -123,11 +133,11 @@ class ColumnarDataProvider(line.RegexLineDataProvider): column = int(column) if column > len(self.column_types): return None - if self.column_types[column] in ('float', 'int'): + if self.column_types[column] in ("float", "int"): return self.create_numeric_filter(column, op, val) - if self.column_types[column] in ('str'): + if self.column_types[column] in ("str"): return self.create_string_filter(column, op, val) - if self.column_types[column] in ('list'): + if self.column_types[column] in ("list"): return self.create_list_filter(column, op, val) return None @@ -153,17 +163,17 @@ class ColumnarDataProvider(line.RegexLineDataProvider): val = float(val) except ValueError: return None - if 'lt' == op: + if "lt" == op: return lambda d: d[column] < val - elif 'le' == op: + elif "le" == op: return lambda d: d[column] <= val - elif 'eq' == op: + elif "eq" == op: return lambda d: d[column] == val - elif 'ne' == op: + elif "ne" == op: return lambda d: d[column] != val - elif 'ge' == op: + elif "ge" == op: return lambda d: d[column] >= val - elif 'gt' == op: + elif "gt" == op: return lambda d: d[column] > val return None @@ -180,11 +190,11 @@ class ColumnarDataProvider(line.RegexLineDataProvider): - has: the column contains the substring `val` - re: the column matches the regular expression in `val` """ - if 'eq' == op: + if "eq" == op: return lambda d: d[column] == val - elif 'has' == op: + elif "has" == op: return lambda d: val in d[column] - elif 're' == op: + elif "re" == op: val = unquote_plus(val) val = re.compile(val) return lambda d: val.match(d[column]) is not None @@ -202,10 +212,10 @@ class ColumnarDataProvider(line.RegexLineDataProvider): - eq: the list `val` exactly matches the list in the column - has: the list in the column contains the sublist `val` """ - if 'eq' == op: - val = self.parse_value(val, 'list') + if "eq" == op: + val = self.parse_value(val, "list") return lambda d: d[column] == val - elif 'has' == op: + elif "has" == op: return lambda d: val in d[column] return None @@ -223,10 +233,9 @@ class ColumnarDataProvider(line.RegexLineDataProvider): # TODO: move to module level (or datatypes, util) return { # str is default and not needed here - 'int': int, - 'float': float, - 'bool': bool, - + "int": int, + "float": float, + "bool": bool, # unfortunately, 'list' is used in dataset metadata both for # query style maps (9th col gff) AND comma-sep strings. # (disabled for now) @@ -235,12 +244,9 @@ class ColumnarDataProvider(line.RegexLineDataProvider): # i don't like how urlparses does sub-lists... # 'querystr' : lambda v: dict([ ( p.split( '=', 1 ) if '=' in p else ( p, True ) ) # for p in v.split( ';', 1 ) ]) - # 'scifloat': #floating point which may be in scientific notation - # always with the 1 base, biologists? # 'int1' : ( lambda i: int( i ) - 1 ), - # 'gffval': string or '.' for None # 'gffint': # int or '.' for None # 'gffphase': # 0, 1, 2, or '.' for None @@ -290,7 +296,7 @@ class ColumnarDataProvider(line.RegexLineDataProvider): or `value` if no `type` found in `parsers` or `None` if there was a parser error (ValueError) """ - if type == 'str' or type is None: + if type == "str" or type is None: return val try: return self.parsers[type](val) @@ -332,8 +338,9 @@ class DictDataProvider(ColumnarDataProvider): .. note:: The subclass constructors are passed kwargs - so their params (limit, offset, etc.) are also applicable here. """ + settings = { - 'column_names': 'list:str', + "column_names": "list:str", } def __init__(self, source, column_names=None, **kwargs): diff --git a/lib/galaxy/datatypes/dataproviders/dataset.py b/lib/galaxy/datatypes/dataproviders/dataset.py index 7d856ad847a..b7d68021696 100644 --- a/lib/galaxy/datatypes/dataproviders/dataset.py +++ b/lib/galaxy/datatypes/dataproviders/dataset.py @@ -8,17 +8,15 @@ Dataproviders that use either: import logging import sys -from bx import ( - seq as bx_seq, - wiggle as bx_wig -) +from bx import seq as bx_seq +from bx import wiggle as bx_wig from galaxy.util import sqlite from . import ( base, column, external, - line + line, ) _TODO = """ @@ -52,7 +50,7 @@ class DatasetDataProvider(base.DataProvider): self.dataset = dataset # this dataset file is obviously the source # TODO: this might be a good place to interface with the object_store... - mode = 'rb' if dataset.datatype.is_binary else 'r' + mode = "rb" if dataset.datatype.is_binary else "r" super().__init__(open(dataset.file_name, mode)) # TODO: this is a bit of a mess @@ -66,9 +64,9 @@ class DatasetDataProvider(base.DataProvider): """ # re-map keys to fit ColumnarProvider.__init__ kwargs params = {} - params['column_count'] = dataset.metadata.columns - params['column_types'] = dataset.metadata.column_types - params['column_names'] = dataset.metadata.column_names or getattr(dataset.datatype, 'column_names', None) + params["column_count"] = dataset.metadata.columns + params["column_types"] = dataset.metadata.column_types + params["column_names"] = dataset.metadata.column_names or getattr(dataset.datatype, "column_names", None) return params def get_metadata_column_types(self, indeces=None): @@ -79,8 +77,9 @@ class DatasetDataProvider(base.DataProvider): Optional: defaults to None (return all types) :type indeces: list of ints """ - metadata_column_types = (self.dataset.metadata.column_types - or getattr(self.dataset.datatype, 'column_types', None) or None) + metadata_column_types = ( + self.dataset.metadata.column_types or getattr(self.dataset.datatype, "column_types", None) or None + ) if not metadata_column_types: return metadata_column_types if indeces: @@ -99,8 +98,9 @@ class DatasetDataProvider(base.DataProvider): Optional: defaults to None (return all names) :type indeces: list of ints """ - metadata_column_names = (self.dataset.metadata.column_names - or getattr(self.dataset.datatype, 'column_names', None) or None) + metadata_column_names = ( + self.dataset.metadata.column_names or getattr(self.dataset.datatype, "column_names", None) or None + ) if not metadata_column_names: return metadata_column_names if indeces: @@ -122,11 +122,13 @@ class DatasetDataProvider(base.DataProvider): :raises KeyError: if column_names are not found :raises ValueError: if an entry in list_of_column_names is not in column_names """ - metadata_column_names = (self.dataset.metadata.column_names - or getattr(self.dataset.datatype, 'column_names', None) or None) + metadata_column_names = ( + self.dataset.metadata.column_names or getattr(self.dataset.datatype, "column_names", None) or None + ) if not metadata_column_names: - raise KeyError('No column_names found for ' - + f'datatype: {str(self.dataset.datatype)}, dataset: {str(self.dataset)}') + raise KeyError( + "No column_names found for " + f"datatype: {str(self.dataset.datatype)}, dataset: {str(self.dataset)}" + ) indeces = [] # if indeces and column_names: # pull using indeces and re-name with given names - no need to alter (does as super would) # pass @@ -152,7 +154,7 @@ class DatasetDataProvider(base.DataProvider): :raises ValueError: if check is `True` and one or more indeces were not found. :returns: list of column indeces for the named columns. """ - region_column_names = ('chromCol', 'startCol', 'endCol') + region_column_names = ("chromCol", "startCol", "endCol") region_indices = [self.get_metadata_column_index_by_name(name) for name in region_column_names] if check and not all(_ is not None for _ in region_indices): raise ValueError(f"Could not determine proper column indices for chrom, start, end: {str(region_indices)}") @@ -166,7 +168,7 @@ class ConvertedDatasetDataProvider(DatasetDataProvider): """ def __init__(self, dataset, **kwargs): - raise NotImplementedError('Abstract class') + raise NotImplementedError("Abstract class") # self.original_dataset = dataset # self.converted_dataset = self.convert_dataset(dataset, **kwargs) # super(ConvertedDatasetDataProvider, self).__init__(self.converted_dataset, **kwargs) @@ -198,9 +200,9 @@ class DatasetColumnarDataProvider(column.ColumnarDataProvider): any metadata available. """ dataset_source = DatasetDataProvider(dataset) - if not kwargs.get('column_types', None): - indeces = kwargs.get('indeces', None) - kwargs['column_types'] = dataset_source.get_metadata_column_types(indeces=indeces) + if not kwargs.get("column_types", None): + indeces = kwargs.get("indeces", None) + kwargs["column_types"] = dataset_source.get_metadata_column_types(indeces=indeces) super().__init__(dataset_source, **kwargs) @@ -234,24 +236,24 @@ class DatasetDictDataProvider(column.DictDataProvider): # TODO: getting too complicated - simplify at some lvl, somehow # if no column_types given, get column_types from indeces (or all if indeces == None) - indeces = kwargs.get('indeces', None) - column_names = kwargs.get('column_names', None) + indeces = kwargs.get("indeces", None) + column_names = kwargs.get("column_names", None) if not indeces and column_names: # pull columns by name - indeces = kwargs['indeces'] = dataset_source.get_indeces_by_column_names(column_names) + indeces = kwargs["indeces"] = dataset_source.get_indeces_by_column_names(column_names) elif indeces and not column_names: # pull using indeces, name with meta - column_names = kwargs['column_names'] = dataset_source.get_metadata_column_names(indeces=indeces) + column_names = kwargs["column_names"] = dataset_source.get_metadata_column_names(indeces=indeces) elif not indeces and not column_names: # pull all indeces and name using metadata - column_names = kwargs['column_names'] = dataset_source.get_metadata_column_names(indeces=indeces) + column_names = kwargs["column_names"] = dataset_source.get_metadata_column_names(indeces=indeces) # if no column_types given, use metadata column_types - if not kwargs.get('column_types', None): - kwargs['column_types'] = dataset_source.get_metadata_column_types(indeces=indeces) + if not kwargs.get("column_types", None): + kwargs["column_types"] = dataset_source.get_metadata_column_types(indeces=indeces) super().__init__(dataset_source, **kwargs) @@ -267,13 +269,14 @@ class GenomicRegionDataProvider(column.ColumnarDataProvider): If `named_columns` is true, will return dictionaries with the keys 'chrom', 'start', 'end'. """ + # dictionary keys when named_columns=True - COLUMN_NAMES = ['chrom', 'start', 'end'] + COLUMN_NAMES = ["chrom", "start", "end"] settings = { - 'chrom_column': 'int', - 'start_column': 'int', - 'end_column': 'int', - 'named_columns': 'bool', + "chrom_column": "int", + "start_column": "int", + "end_column": "int", + "named_columns": "bool", } def __init__(self, dataset, chrom_column=None, start_column=None, end_column=None, named_columns=False, **kwargs): @@ -297,19 +300,18 @@ class GenomicRegionDataProvider(column.ColumnarDataProvider): dataset_source = DatasetDataProvider(dataset) if chrom_column is None: - chrom_column = dataset_source.get_metadata_column_index_by_name('chromCol') + chrom_column = dataset_source.get_metadata_column_index_by_name("chromCol") if start_column is None: - start_column = dataset_source.get_metadata_column_index_by_name('startCol') + start_column = dataset_source.get_metadata_column_index_by_name("startCol") if end_column is None: - end_column = dataset_source.get_metadata_column_index_by_name('endCol') + end_column = dataset_source.get_metadata_column_index_by_name("endCol") indeces = [chrom_column, start_column, end_column] if not all(_ is not None for _ in indeces): - raise ValueError("Could not determine proper column indeces for" - + f" chrom, start, end: {str(indeces)}") - kwargs.update({'indeces': indeces}) + raise ValueError("Could not determine proper column indeces for" + f" chrom, start, end: {str(indeces)}") + kwargs.update({"indeces": indeces}) - if not kwargs.get('column_types', None): - kwargs.update({'column_types': dataset_source.get_metadata_column_types(indeces=indeces)}) + if not kwargs.get("column_types", None): + kwargs.update({"column_types": dataset_source.get_metadata_column_types(indeces=indeces)}) self.named_columns = named_columns if self.named_columns: @@ -336,18 +338,28 @@ class IntervalDataProvider(column.ColumnarDataProvider): If `named_columns` is true, will return dictionaries with the keys 'chrom', 'start', 'end' (and 'strand' and 'name' if available). """ - COLUMN_NAMES = ['chrom', 'start', 'end', 'strand', 'name'] + + COLUMN_NAMES = ["chrom", "start", "end", "strand", "name"] settings = { - 'chrom_column': 'int', - 'start_column': 'int', - 'end_column': 'int', - 'strand_column': 'int', - 'name_column': 'int', - 'named_columns': 'bool', + "chrom_column": "int", + "start_column": "int", + "end_column": "int", + "strand_column": "int", + "name_column": "int", + "named_columns": "bool", } - def __init__(self, dataset, chrom_column=None, start_column=None, end_column=None, - strand_column=None, name_column=None, named_columns=False, **kwargs): + def __init__( + self, + dataset, + chrom_column=None, + start_column=None, + end_column=None, + strand_column=None, + name_column=None, + named_columns=False, + **kwargs, + ): """ :param dataset: the Galaxy dataset whose file will be the source :type dataset: model.DatasetInstance @@ -365,34 +377,34 @@ class IntervalDataProvider(column.ColumnarDataProvider): indeces = [] # TODO: this is sort of involved and oogly if chrom_column is None: - chrom_column = dataset_source.get_metadata_column_index_by_name('chromCol') + chrom_column = dataset_source.get_metadata_column_index_by_name("chromCol") if chrom_column is not None: - self.column_names.append('chrom') + self.column_names.append("chrom") indeces.append(chrom_column) if start_column is None: - start_column = dataset_source.get_metadata_column_index_by_name('startCol') + start_column = dataset_source.get_metadata_column_index_by_name("startCol") if start_column is not None: - self.column_names.append('start') + self.column_names.append("start") indeces.append(start_column) if end_column is None: - end_column = dataset_source.get_metadata_column_index_by_name('endCol') + end_column = dataset_source.get_metadata_column_index_by_name("endCol") if end_column is not None: - self.column_names.append('end') + self.column_names.append("end") indeces.append(end_column) if strand_column is None: - strand_column = dataset_source.get_metadata_column_index_by_name('strandCol') + strand_column = dataset_source.get_metadata_column_index_by_name("strandCol") if strand_column is not None: - self.column_names.append('strand') + self.column_names.append("strand") indeces.append(strand_column) if name_column is None: - name_column = dataset_source.get_metadata_column_index_by_name('nameCol') + name_column = dataset_source.get_metadata_column_index_by_name("nameCol") if name_column is not None: - self.column_names.append('name') + self.column_names.append("name") indeces.append(name_column) - kwargs.update({'indeces': indeces}) - if not kwargs.get('column_types', None): - kwargs.update({'column_types': dataset_source.get_metadata_column_types(indeces=indeces)}) + kwargs.update({"indeces": indeces}) + if not kwargs.get("column_types", None): + kwargs.update({"column_types": dataset_source.get_metadata_column_types(indeces=indeces)}) self.named_columns = named_columns @@ -418,8 +430,9 @@ class FastaDataProvider(base.FilteredDataProvider): sequence: } """ + settings = { - 'ids': 'list:str', + "ids": "list:str", } def __init__(self, source, ids=None, **kwargs): @@ -437,10 +450,7 @@ class FastaDataProvider(base.FilteredDataProvider): def __iter__(self): parent_gen = super().__iter__() for fasta_record in parent_gen: - yield { - 'id': fasta_record.name, - 'seq': fasta_record.text - } + yield {"id": fasta_record.name, "seq": fasta_record.text} class TwoBitFastaDataProvider(DatasetDataProvider): @@ -452,8 +462,9 @@ class TwoBitFastaDataProvider(DatasetDataProvider): sequence: } """ + settings = { - 'ids': 'list:str', + "ids": "list:str", } def __init__(self, source, ids=None, **kwargs): @@ -470,10 +481,7 @@ class TwoBitFastaDataProvider(DatasetDataProvider): def __iter__(self): for id_ in self.ids: - yield { - 'id': id_, - 'seq': self.source[id_] - } + yield {"id": id_, "seq": self.source[id_]} # TODO: @@ -481,10 +489,11 @@ class WiggleDataProvider(base.LimitedOffsetDataProvider): """ Class that returns chrom, pos, data from a wiggle source. """ - COLUMN_NAMES = ['chrom', 'pos', 'value'] + + COLUMN_NAMES = ["chrom", "pos", "value"] settings = { - 'named_columns': 'bool', - 'column_names': 'list:str', + "named_columns": "bool", + "column_names": "list:str", } def __init__(self, source, named_columns=False, column_names=None, **kwargs): @@ -523,10 +532,11 @@ class BigWigDataProvider(base.LimitedOffsetDataProvider): """ Class that returns chrom, pos, data from a wiggle source. """ - COLUMN_NAMES = ['chrom', 'pos', 'value'] + + COLUMN_NAMES = ["chrom", "pos", "value"] settings = { - 'named_columns': 'bool', - 'column_names': 'list:str', + "named_columns": "bool", + "column_names": "list:str", } def __init__(self, source, chrom, start, end, named_columns=False, column_names=None, **kwargs): @@ -551,7 +561,7 @@ class BigWigDataProvider(base.LimitedOffsetDataProvider): :type column_names: """ - raise NotImplementedError('Work in progress') + raise NotImplementedError("Work in progress") # TODO: validate is a wig # still good to maintain a ref to the raw source bc Reader won't # self.raw_source = source @@ -579,6 +589,7 @@ class DatasetSubprocessDataProvider(external.SubprocessDataProvider): Uses a subprocess as its source and has a dataset (gen. as an input file for the process). """ + # TODO: below should be a subclass of this and not RegexSubprocess def __init__(self, dataset, *args, **kwargs): @@ -586,7 +597,7 @@ class DatasetSubprocessDataProvider(external.SubprocessDataProvider): :param args: the list of strings used to build commands. :type args: variadic function args """ - raise NotImplementedError('Abstract class') + raise NotImplementedError("Abstract class") # super(DatasetSubprocessDataProvider, self).__init__(*args, **kwargs) # self.dataset = dataset @@ -599,11 +610,12 @@ class SamtoolsDataProvider(line.RegexLineDataProvider): .. note:: that only the samtools 'view' command is currently implemented. """ - FLAGS_WO_ARGS = 'bhHSu1xXcB' - FLAGS_W_ARGS = 'fFqlrs' + + FLAGS_WO_ARGS = "bhHSu1xXcB" + FLAGS_W_ARGS = "fFqlrs" VALID_FLAGS = FLAGS_WO_ARGS + FLAGS_W_ARGS - def __init__(self, dataset, options_string='', options_dict=None, regions=None, **kwargs): + def __init__(self, dataset, options_string="", options_dict=None, regions=None, **kwargs): """ :param options_string: samtools options in string form (flags separated by spaces) @@ -627,10 +639,10 @@ class SamtoolsDataProvider(line.RegexLineDataProvider): # TODO: view only for now # TODO: not properly using overriding super's validate_opts, command here - subcommand = 'view' + subcommand = "view" # TODO:?? do we need a path to samtools? subproc_args = self.build_command_list(subcommand, options_string, options_dict, regions) -# TODO: the composition/inheritance here doesn't make a lot sense + # TODO: the composition/inheritance here doesn't make a lot sense subproc_provider = external.SubprocessDataProvider(*subproc_args) super().__init__(subproc_provider, **kwargs) @@ -638,7 +650,7 @@ class SamtoolsDataProvider(line.RegexLineDataProvider): """ Convert all init args to list form. """ - command = ['samtools', subcommand] + command = ["samtools", subcommand] # add options and switches, input file, regions list (if any) command.extend(self.to_options_list(options_string, options_dict)) command.append(self.dataset.file_name) @@ -654,14 +666,13 @@ class SamtoolsDataProvider(line.RegexLineDataProvider): # strip out any user supplied bash switch formating -> string of option chars # then compress to single option string of unique, VALID flags with prefixed bash switch char '-' - options_string = options_string.strip('- ') + options_string = options_string.strip("- ") validated_flag_list = {flag for flag in options_string if flag in self.FLAGS_WO_ARGS} # if sam add -S # TODO: not the best test in the world... - if((self.dataset.ext == 'sam') - and ('S' not in validated_flag_list)): - validated_flag_list.append('S') + if (self.dataset.ext == "sam") and ("S" not in validated_flag_list): + validated_flag_list.append("S") if validated_flag_list: opt_list.append(f"-{''.join(validated_flag_list)}") @@ -696,9 +707,8 @@ class SQliteDataProvider(base.DataProvider): Allows any query to be run and returns the resulting rows as sqlite3 row objects """ - settings = { - 'query': 'str' - } + + settings = {"query": "str"} def __init__(self, source, query=None, **kwargs): self.query = query @@ -717,11 +727,8 @@ class SQliteDataTableProvider(base.DataProvider): Data provider that uses a sqlite database file as its source. Allows any query to be run and returns the resulting rows as arrays of arrays """ - settings = { - 'query': 'str', - 'headers': 'bool', - 'limit': 'int' - } + + settings = {"query": "str", "headers": "bool", "limit": "int"} def __init__(self, source, query=None, headers=False, limit=sys.maxsize, **kwargs): self.query = query @@ -749,9 +756,8 @@ class SQliteDataDictProvider(base.DataProvider): Data provider that uses a sqlite database file as its source. Allows any query to be run and returns the resulting rows as arrays of dicts """ - settings = { - 'query': 'str' - } + + settings = {"query": "str"} def __init__(self, source, query=None, **kwargs): self.query = query diff --git a/lib/galaxy/datatypes/dataproviders/decorators.py b/lib/galaxy/datatypes/dataproviders/decorators.py index 6a2579260b3..a20c2bb5146 100644 --- a/lib/galaxy/datatypes/dataproviders/decorators.py +++ b/lib/galaxy/datatypes/dataproviders/decorators.py @@ -21,8 +21,8 @@ from urllib.parse import unquote log = logging.getLogger(__name__) -_DATAPROVIDER_CLASS_MAP_KEY = 'dataproviders' -_DATAPROVIDER_METHOD_NAME_KEY = '_dataprovider_name' +_DATAPROVIDER_CLASS_MAP_KEY = "dataproviders" +_DATAPROVIDER_METHOD_NAME_KEY = "_dataprovider_name" def has_dataproviders(cls): @@ -67,9 +67,11 @@ def has_dataproviders(cls): # where it's possible to override a super's provider with a sub's for attr_key, attr_value in cls.__dict__.items(): # can't use isinstance( attr_value, MethodType ) bc of wrapping - if((callable(attr_value)) - and (not attr_key.startswith("__")) - and (getattr(attr_value, _DATAPROVIDER_METHOD_NAME_KEY, None))): + if ( + (callable(attr_value)) + and (not attr_key.startswith("__")) + and (getattr(attr_value, _DATAPROVIDER_METHOD_NAME_KEY, None)) + ): name = getattr(attr_value, _DATAPROVIDER_METHOD_NAME_KEY) dataproviders[name] = attr_value return cls @@ -110,7 +112,9 @@ def dataprovider_factory(name, settings=None): @wraps(func) def wrapped_dataprovider_factory(self, *args, **kwargs): return func(self, *args, **kwargs) + return wrapped_dataprovider_factory + return named_dataprovider_factory @@ -122,15 +126,15 @@ def _parse_query_string_settings(query_kwargs, settings=None): # TODO: this was a relatively late addition: review and re-think def list_from_query_string(s): # assume csv - return s.split(',') + return s.split(",") parsers = { - 'int': int, - 'float': float, - 'bool': bool, - 'list:str': lambda s: list_from_query_string(s), - 'list:escaped': lambda s: [unquote(e) for e in list_from_query_string(s)], - 'list:int': lambda s: [int(i) for i in list_from_query_string(s)], + "int": int, + "float": float, + "bool": bool, + "list:str": lambda s: list_from_query_string(s), + "list:escaped": lambda s: [unquote(e) for e in list_from_query_string(s)], + "list:int": lambda s: [int(i) for i in list_from_query_string(s)], } settings = settings or {} # yay! yet another set of query string parsers! <-- sarcasm @@ -142,7 +146,7 @@ def _parse_query_string_settings(query_kwargs, settings=None): # TODO: this would be the place to sanitize any strings query_value = query_kwargs[key] needed_type = settings[key] - if needed_type != 'str': + if needed_type != "str": try: query_kwargs[key] = parsers[needed_type](query_value) except (KeyError, ValueError): diff --git a/lib/galaxy/datatypes/dataproviders/exceptions.py b/lib/galaxy/datatypes/dataproviders/exceptions.py index a3d9fbd2dd4..b40e001ccd0 100644 --- a/lib/galaxy/datatypes/dataproviders/exceptions.py +++ b/lib/galaxy/datatypes/dataproviders/exceptions.py @@ -8,8 +8,8 @@ class InvalidDataProviderSource(TypeError): Raised when a unusable source is passed to a provider. """ - def __init__(self, source=None, msg=''): - msg = msg or f'Invalid source for provider: {source}' + def __init__(self, source=None, msg=""): + msg = msg or f"Invalid source for provider: {source}" super().__init__(msg) @@ -27,7 +27,7 @@ class NoProviderAvailable(TypeError): Meant to be used within a class that builds dataproviders (e.g. a Datatype) """ - def __init__(self, factory_source, format_requested=None, msg=''): + def __init__(self, factory_source, format_requested=None, msg=""): self.factory_source = factory_source self.format_requested = format_requested msg = msg or f'No provider available in factory_source "{str(factory_source)}" for format requested' diff --git a/lib/galaxy/datatypes/dataproviders/external.py b/lib/galaxy/datatypes/dataproviders/external.py index a0e2c00b0a2..fea7a8b665d 100644 --- a/lib/galaxy/datatypes/dataproviders/external.py +++ b/lib/galaxy/datatypes/dataproviders/external.py @@ -15,7 +15,7 @@ from urllib.request import urlopen from galaxy.util import DEFAULT_SOCKET_TIMEOUT from . import ( base, - line + line, ) _TODO = """ @@ -33,6 +33,7 @@ class SubprocessDataProvider(base.DataProvider): Data provider that uses the output from an intermediate program and subprocess as its data source. """ + # TODO: need better ways of checking returncode, stderr for errors and raising def __init__(self, *args, **kwargs): @@ -57,32 +58,33 @@ class SubprocessDataProvider(base.DataProvider): try: # how expensive is this? popen = subprocess.Popen(command_list, stderr=subprocess.PIPE, stdout=subprocess.PIPE) - log.info(f'opened subrocess ({str(command_list)}), PID: {str(popen.pid)}') + log.info(f"opened subrocess ({str(command_list)}), PID: {str(popen.pid)}") except OSError as os_err: - command_str = ' '.join(self.command) - raise OSError(' '.join((str(os_err), ':', command_str))) + command_str = " ".join(self.command) + raise OSError(" ".join((str(os_err), ":", command_str))) return popen def __exit__(self, *args): # poll the subrocess for an exit code self.exit_code = self.popen.poll() - log.info(f'{str(self)}.__exit__, exit_code: {str(self.exit_code)}') + log.info(f"{str(self)}.__exit__, exit_code: {str(self.exit_code)}") return super().__exit__(*args) def __str__(self): # provide the pid and current return code - source_str = '' - if hasattr(self, 'popen'): - source_str = f'{str(self.popen.pid)}:{str(self.popen.poll())}' - return f'{self.__class__.__name__}({str(source_str)})' + source_str = "" + if hasattr(self, "popen"): + source_str = f"{str(self.popen.pid)}:{str(self.popen.poll())}" + return f"{self.__class__.__name__}({str(source_str)})" class RegexSubprocessDataProvider(line.RegexLineDataProvider): """ RegexLineDataProvider that uses a SubprocessDataProvider as its data source. """ + # this is a conv. class and not really all that necc... def __init__(self, *args, **kwargs): @@ -98,9 +100,10 @@ class URLDataProvider(base.DataProvider): This can be piped through other providers (column, map, genome region, etc.). """ - VALID_METHODS = ('GET', 'POST') - def __init__(self, url, method='GET', data=None, **kwargs): + VALID_METHODS = ("GET", "POST") + + def __init__(self, url, method="GET", data=None, **kwargs): """ :param url: the base URL to open. :param method: the HTTP method to use. @@ -116,15 +119,15 @@ class URLDataProvider(base.DataProvider): encoded_data = urlencode(self.data) scheme = urlparse(url).scheme - assert scheme in ('http', 'https', 'ftp'), f'Invalid URL scheme: {scheme}' + assert scheme in ("http", "https", "ftp"), f"Invalid URL scheme: {scheme}" - if method == 'GET': - self.url += f'?{encoded_data}' + if method == "GET": + self.url += f"?{encoded_data}" opened = urlopen(url, timeout=DEFAULT_SOCKET_TIMEOUT) - elif method == 'POST': + elif method == "POST": opened = urlopen(url, encoded_data, timeout=DEFAULT_SOCKET_TIMEOUT) else: - raise ValueError(f'Not a valid method: {method}') + raise ValueError(f"Not a valid method: {method}") super().__init__(opened, **kwargs) # NOTE: the request object is now accessible as self.source @@ -145,7 +148,7 @@ class GzipDataProvider(base.DataProvider): """ def __init__(self, source, **kwargs): - unzipped = gzip.GzipFile(source, 'rb') + unzipped = gzip.GzipFile(source, "rb") super().__init__(unzipped, **kwargs) # NOTE: the GzipFile is now accessible in self.source @@ -171,6 +174,6 @@ class TempfileDataProvider(base.DataProvider): def write_to_file(self): parent_gen = super().__iter__() - with open(self.tmp_file, 'w') as open_file: + with open(self.tmp_file, "w") as open_file: for datum in parent_gen: open_file.write(f"{datum}\n") diff --git a/lib/galaxy/datatypes/dataproviders/hierarchy.py b/lib/galaxy/datatypes/dataproviders/hierarchy.py index 7a52dd76b29..e0773975548 100644 --- a/lib/galaxy/datatypes/dataproviders/hierarchy.py +++ b/lib/galaxy/datatypes/dataproviders/hierarchy.py @@ -31,13 +31,14 @@ class XMLDataProvider(HierarchalDataProvider): """ Data provider that converts selected XML elements to dictionaries. """ + # using lxml.etree's iterparse method to keep mem down # TODO: this, however (AFAIK), prevents the use of xpath settings = { - 'selector': 'str', # urlencoded - 'max_depth': 'int', + "selector": "str", # urlencoded + "max_depth": "int", } - ITERPARSE_ALL_EVENTS = ('start', 'end', 'start-ns', 'end-ns') + ITERPARSE_ALL_EVENTS = ("start", "end", "start-ns", "end-ns") # TODO: move appropo into super def __init__(self, source, selector=None, max_depth=None, **kwargs): @@ -64,9 +65,8 @@ class XMLDataProvider(HierarchalDataProvider): # TODO: add more flexibility here w/o re-implementing xpath # TODO: fails with '#' - browser thinks it's an anchor - use urlencode # TODO: need removal/replacement of etree namespacing here - then move to string match - Element = getattr(etree, '_Element', etree.Element) - return bool((selector is None) - or (isinstance(element, Element) and selector in element.tag)) + Element = getattr(etree, "_Element", etree.Element) + return bool((selector is None) or (isinstance(element, Element) and selector in element.tag)) def element_as_dict(self, element): """ @@ -76,10 +76,10 @@ class XMLDataProvider(HierarchalDataProvider): """ # TODO: Key collision is unlikely here, but still should be better handled return { - 'tag': element.tag, - 'text': element.text.strip() if element.text else None, + "tag": element.tag, + "text": element.text.strip() if element.text else None, # needs shallow copy to protect v. element.clear() - 'attrib': dict(element.attrib) + "attrib": dict(element.attrib), } def get_children(self, element, max_depth=None): @@ -96,7 +96,7 @@ class XMLDataProvider(HierarchalDataProvider): next_depth = max_depth - 1 if isinstance(max_depth, int) else None grand_children = list(self.get_children(child, next_depth)) if grand_children: - child_data['children'] = grand_children + child_data["children"] = grand_children yield child_data @@ -106,18 +106,17 @@ class XMLDataProvider(HierarchalDataProvider): selected_element = None for event, element in context: - if event == 'start-ns': + if event == "start-ns": ns, uri = element self.namespaces[ns] = uri - elif event == 'start': - if((selected_element is None) - and (self.matches_selector(element, self.selector))): + elif event == "start": + if (selected_element is None) and (self.matches_selector(element, self.selector)): # start tag of selected element - wait for 'end' to emit/yield selected_element = element - elif event == 'end': - if((selected_element is not None) and (element == selected_element)): + elif event == "end": + if (selected_element is not None) and (element == selected_element): self.num_valid_data_read += 1 # offset @@ -126,7 +125,7 @@ class XMLDataProvider(HierarchalDataProvider): selected_element_dict = self.element_as_dict(selected_element) children = list(self.get_children(selected_element, self.max_depth)) if children: - selected_element_dict['children'] = children + selected_element_dict["children"] = children yield selected_element_dict # limit diff --git a/lib/galaxy/datatypes/dataproviders/line.py b/lib/galaxy/datatypes/dataproviders/line.py index 3a95521e992..dcbdfc44012 100644 --- a/lib/galaxy/datatypes/dataproviders/line.py +++ b/lib/galaxy/datatypes/dataproviders/line.py @@ -23,16 +23,24 @@ class FilteredLineDataProvider(base.LimitedOffsetDataProvider): optional control over which line to start on and how many lines to return. """ - DEFAULT_COMMENT_CHAR = '#' + + DEFAULT_COMMENT_CHAR = "#" settings = { - 'strip_lines': 'bool', - 'strip_newlines': 'bool', - 'provide_blank': 'bool', - 'comment_char': 'str', + "strip_lines": "bool", + "strip_newlines": "bool", + "provide_blank": "bool", + "comment_char": "str", } - def __init__(self, source, strip_lines=True, strip_newlines=False, provide_blank=False, - comment_char=DEFAULT_COMMENT_CHAR, **kwargs): + def __init__( + self, + source, + strip_lines=True, + strip_newlines=False, + provide_blank=False, + comment_char=DEFAULT_COMMENT_CHAR, + **kwargs, + ): """ :param strip_lines: remove whitespace from the beginning an ending of each line (or not). @@ -72,8 +80,8 @@ class FilteredLineDataProvider(base.LimitedOffsetDataProvider): if self.strip_lines: line = line.strip() elif self.strip_newlines: - line = line.strip('\n') - if not self.provide_blank and line == '': + line = line.strip("\n") + if not self.provide_blank and line == "": return None elif self.comment_char and line.startswith(self.comment_char): return None @@ -90,9 +98,10 @@ class RegexLineDataProvider(FilteredLineDataProvider): .. note:: the regex matches are effectively OR'd (if **any** regex matches the line it is considered valid and will be provided). """ + settings = { - 'regex_list': 'list:escaped', - 'invert': 'bool', + "regex_list": "list:escaped", + "invert": "bool", } def __init__(self, source, regex_list=None, invert=False, **kwargs): @@ -153,8 +162,7 @@ class BlockDataProvider(base.LimitedOffsetDataProvider): """ # composition - not inheritance # TODO: not a fan of this: - (filter_fn, limit, offset) = (kwargs.pop('filter_fn', None), - kwargs.pop('limit', None), kwargs.pop('offset', 0)) + (filter_fn, limit, offset) = (kwargs.pop("filter_fn", None), kwargs.pop("limit", None), kwargs.pop("offset", 0)) line_provider = FilteredLineDataProvider(source, **kwargs) super().__init__(line_provider, filter_fn=filter_fn, limit=limit, offset=offset) diff --git a/lib/galaxy/datatypes/display_applications/application.py b/lib/galaxy/datatypes/display_applications/application.py index f9519752040..a978e9c61d7 100644 --- a/lib/galaxy/datatypes/display_applications/application.py +++ b/lib/galaxy/datatypes/display_applications/application.py @@ -5,13 +5,13 @@ from urllib.parse import quote_plus from galaxy.util import ( parse_xml, - string_as_bool + string_as_bool, ) from galaxy.util.template import fill_template from .parameters import ( DEFAULT_DATASET_NAME, DisplayApplicationDataParameter, - DisplayApplicationParameter + DisplayApplicationParameter, ) from .util import encode_dataset_user @@ -29,16 +29,16 @@ class DisplayApplicationLink: @classmethod def from_elem(cls, elem, display_application, other_values=None): rval = DisplayApplicationLink(display_application) - rval.id = elem.get('id', None) - assert rval.id, 'Link elements require a id.' - rval.name = elem.get('name', rval.id) - rval.url = elem.find('url') - assert rval.url is not None, 'A url element must be provided for link elements.' + rval.id = elem.get("id", None) + assert rval.id, "Link elements require a id." + rval.name = elem.get("name", rval.id) + rval.url = elem.find("url") + assert rval.url is not None, "A url element must be provided for link elements." rval.other_values = other_values - rval.filters = elem.findall('filter') - for param_elem in elem.findall('param'): + rval.filters = elem.findall("filter") + for param_elem in elem.findall("param"): param = DisplayApplicationParameter.from_elem(param_elem, rval) - assert param, f'Unable to load parameter from element: {param_elem}' + assert param, f"Unable to load parameter from element: {param_elem}" rval.parameters[param.name] = param rval.url_param_name_map[param.url] = param.name return rval @@ -53,21 +53,25 @@ class DisplayApplicationLink: def get_display_url(self, data, trans): dataset_hash, user_hash = encode_dataset_user(trans, data, None) - return trans.app.url_for(controller='dataset', - action="display_application", - dataset_id=dataset_hash, - user_id=user_hash, - app_name=quote_plus(self.display_application.id), - link_name=quote_plus(self.id), - app_action=None) + return trans.app.url_for( + controller="dataset", + action="display_application", + dataset_id=dataset_hash, + user_id=user_hash, + app_name=quote_plus(self.display_application.id), + link_name=quote_plus(self.id), + app_action=None, + ) def get_inital_values(self, data, trans): if self.other_values: rval = dict(self.other_values) else: rval = {} - rval.update({'BASE_URL': trans.request.base, 'APP': trans.app}) # trans automatically appears as a response, need to add properties of trans that we want here - BASE_PARAMS = {'qp': quote_plus_string, 'url_for': trans.app.url_for} + rval.update( + {"BASE_URL": trans.request.base, "APP": trans.app} + ) # trans automatically appears as a response, need to add properties of trans that we want here + BASE_PARAMS = {"qp": quote_plus_string, "url_for": trans.app.url_for} for key, value in BASE_PARAMS.items(): # add helper functions/variables rval[key] = value rval[DEFAULT_DATASET_NAME] = data # always have the display dataset name available @@ -75,8 +79,8 @@ class DisplayApplicationLink: def build_parameter_dict(self, data, dataset_hash, user_hash, trans, app_kwds): other_values = self.get_inital_values(data, trans) - other_values['DATASET_HASH'] = dataset_hash - other_values['USER_HASH'] = user_hash + other_values["DATASET_HASH"] = dataset_hash + other_values["USER_HASH"] = user_hash ready = True for name, param in self.parameters.items(): assert name not in other_values, f"The display parameter '{name}' has been defined more than once." @@ -84,10 +88,14 @@ class DisplayApplicationLink: if name in app_kwds and param.allow_override: other_values[name] = app_kwds[name] else: - other_values[name] = param.get_value(other_values, dataset_hash, user_hash, trans) # subsequent params can rely on this value + other_values[name] = param.get_value( + other_values, dataset_hash, user_hash, trans + ) # subsequent params can rely on this value else: ready = False - other_values[name] = param.get_value(other_values, dataset_hash, user_hash, trans) # subsequent params can rely on this value + other_values[name] = param.get_value( + other_values, dataset_hash, user_hash, trans + ) # subsequent params can rely on this value if other_values[name] is None: # Need to stop here, next params may need this value to determine its own value return False, other_values @@ -96,51 +104,50 @@ class DisplayApplicationLink: def filter_by_dataset(self, data, trans): context = self.get_inital_values(data, trans) for filter_elem in self.filters: - if fill_template(filter_elem.text, context=context) != filter_elem.get('value', 'True'): + if fill_template(filter_elem.text, context=context) != filter_elem.get("value", "True"): return False return True class DynamicDisplayApplicationBuilder: - def __init__(self, elem, display_application, build_sites): filename = None data_table = None - if elem.get('site_type', None) is not None: - filename = build_sites.get(elem.get('site_type')) + if elem.get("site_type", None) is not None: + filename = build_sites.get(elem.get("site_type")) else: - filename = elem.get('from_file', None) + filename = elem.get("from_file", None) if filename is None: - data_table_name = elem.get('from_data_table', None) + data_table_name = elem.get("from_data_table", None) if data_table_name: data_table = display_application.app.tool_data_tables.get(data_table_name, None) assert data_table is not None, f'Unable to find data table named "{data_table_name}".' - assert filename is not None or data_table is not None, 'Filename or data Table is required for dynamic_links.' - skip_startswith = elem.get('skip_startswith', None) - separator = elem.get('separator', '\t') - id_col = elem.get('id', None) + assert filename is not None or data_table is not None, "Filename or data Table is required for dynamic_links." + skip_startswith = elem.get("skip_startswith", None) + separator = elem.get("separator", "\t") + id_col = elem.get("id", None) try: id_col = int(id_col) except (TypeError, ValueError): if data_table: if id_col is None: - id_col = data_table.columns.get('id', None) + id_col = data_table.columns.get("id", None) if id_col is None: - id_col = data_table.columns.get('value', None) + id_col = data_table.columns.get("value", None) try: id_col = int(id_col) except (TypeError, ValueError): # id is set to a string or None, use column by that name if available id_col = data_table.columns.get(id_col, None) id_col = int(id_col) - name_col = elem.get('name', None) + name_col = elem.get("name", None) try: name_col = int(name_col) except (TypeError, ValueError): if data_table: if name_col is None: - name_col = data_table.columns.get('name', None) + name_col = data_table.columns.get("name", None) else: name_col = data_table.columns.get(name_col, None) else: @@ -152,14 +159,14 @@ class DynamicDisplayApplicationBuilder: if data_table is not None: max_col = max([max_col] + list(data_table.columns.values())) for key, value in data_table.columns.items(): - dynamic_params[key] = {'column': value, 'split': False, 'separator': ','} - for dynamic_param in elem.findall('dynamic_param'): - name = dynamic_param.get('name') - value = int(dynamic_param.get('value')) - split = string_as_bool(dynamic_param.get('split', False)) - param_separator = dynamic_param.get('separator', ',') + dynamic_params[key] = {"column": value, "split": False, "separator": ","} + for dynamic_param in elem.findall("dynamic_param"): + name = dynamic_param.get("name") + value = int(dynamic_param.get("value")) + split = string_as_bool(dynamic_param.get("split", False)) + param_separator = dynamic_param.get("separator", ",") max_col = max(max_col, value) - dynamic_params[name] = {'column': value, 'split': split, 'separator': param_separator} + dynamic_params[name] = {"column": value, "split": split, "separator": param_separator} if filename: data_iter = open(filename) elif data_table: @@ -169,7 +176,7 @@ class DynamicDisplayApplicationBuilder: for line in data_iter: if isinstance(line, str): if not skip_startswith or not line.startswith(skip_startswith): - line = line.rstrip('\n\r') + line = line.rstrip("\n\r") if not line: continue fields = line.split(separator) @@ -179,16 +186,18 @@ class DynamicDisplayApplicationBuilder: fields = line if len(fields) > max_col: new_elem = deepcopy(elem) - new_elem.set('id', fields[id_col]) - new_elem.set('name', fields[name_col]) + new_elem.set("id", fields[id_col]) + new_elem.set("name", fields[name_col]) dynamic_values = {} for key, attributes in dynamic_params.items(): - value = fields[attributes['column']] - if attributes['split']: - value = value.split(attributes['separator']) + value = fields[attributes["column"]] + if attributes["split"]: + value = value.split(attributes["separator"]) dynamic_values[key] = value # now populate - links.append(DisplayApplicationLink.from_elem(new_elem, display_application, other_values=dynamic_values)) + links.append( + DisplayApplicationLink.from_elem(new_elem, display_application, other_values=dynamic_values) + ) else: log.warning(f'Invalid dynamic display application link specified in {filename}: "{line}"') self.links = links @@ -204,7 +213,9 @@ class PopulatedDisplayApplicationLink: self.dataset_hash = dataset_hash self.user_hash = user_hash self.trans = trans - self.ready, self.parameters = self.link.build_parameter_dict(self.data, self.dataset_hash, self.user_hash, trans, app_kwds) + self.ready, self.parameters = self.link.build_parameter_dict( + self.data, self.dataset_hash, self.user_hash, trans, app_kwds + ) def display_ready(self): return self.ready @@ -213,7 +224,7 @@ class PopulatedDisplayApplicationLink: value = None if self.ready: value = self.parameters.get(name, None) - assert value, 'Unknown parameter requested' + assert value, "Unknown parameter requested" return value def preparing_display(self): @@ -230,7 +241,7 @@ class PopulatedDisplayApplicationLink: if found_last or list(other_values.keys())[-1] == name: # found last parameter to be populated found_last = True value = param.prepare(other_values, self.dataset_hash, self.user_hash, self.trans) - rval.append({'name': name, 'value': value, 'param': param}) + rval.append({"name": name, "value": value, "param": param}) other_values[name] = value if value is None: # We can go no further until we have a value for this parameter @@ -243,11 +254,11 @@ class PopulatedDisplayApplicationLink: if datasets_only and not isinstance(param, DisplayApplicationDataParameter): continue value = self.parameters.get(name, None) - rval.append({'name': name, 'value': value, 'param': param, 'ready': param.ready(self.parameters)}) + rval.append({"name": name, "value": value, "param": param, "ready": param.ready(self.parameters)}) return rval def display_url(self): - assert self.display_ready(), 'Display is not yet ready, cannot generate display link' + assert self.display_ready(), "Display is not yet ready, cannot generate display link" return fill_template(self.link.url.text, context=self.parameters) def get_param_name_by_url(self, url): @@ -269,16 +280,18 @@ class DisplayApplication: @classmethod def from_elem(cls, elem, app, filename=None): att_dict = cls._get_attributes_from_elem(elem) - rval = DisplayApplication(att_dict['id'], att_dict['name'], app, att_dict['version'], filename=filename, elem=elem) + rval = DisplayApplication( + att_dict["id"], att_dict["name"], app, att_dict["version"], filename=filename, elem=elem + ) rval._load_links_from_elem(elem) return rval @classmethod def _get_attributes_from_elem(cls, elem): - display_id = elem.get('id', None) + display_id = elem.get("id", None) assert display_id, "ID tag is required for a Display Application" - name = elem.get('name', display_id) - version = elem.get('version', None) + name = elem.get("name", display_id) + version = elem.get("version", None) return dict(id=display_id, name=name, version=version) def __init__(self, display_id, name, app, version=None, filename=None, elem=None): @@ -294,13 +307,15 @@ class DisplayApplication: self._data_table_versions = {} def _load_links_from_elem(self, elem): - for link_elem in elem.findall('link'): + for link_elem in elem.findall("link"): link = DisplayApplicationLink.from_elem(link_elem, self) if link: self.links[link.id] = link try: - for dynamic_links in elem.findall('dynamic_links'): - for link in DynamicDisplayApplicationBuilder(dynamic_links, self, self.app.datatypes_registry.build_sites): + for dynamic_links in elem.findall("dynamic_links"): + for link in DynamicDisplayApplicationBuilder( + dynamic_links, self, self.app.datatypes_registry.build_sites + ): self.links[link.id] = link except Exception as e: log.error("Error loading a set of Dynamic Display Application links: %s", e) @@ -328,7 +343,9 @@ class DisplayApplication: # All toolshed-specific attributes added by e.g the registry will remain attr_dict = self._get_attributes_from_elem(elem) # We will not allow changing the id at this time (we'll need to fix several mappings upstream to handle this case) - assert attr_dict.get('id') == self.id, ValueError("You cannot reload a Display application where the ID has changed. You will need to restart the server instead.") + assert attr_dict.get("id") == self.id, ValueError( + "You cannot reload a Display application where the ID has changed. You will need to restart the server instead." + ) # clear old links self.links = {} # clear data table versions: diff --git a/lib/galaxy/datatypes/display_applications/parameters.py b/lib/galaxy/datatypes/display_applications/parameters.py index f5e6c3659bf..c1a28fc8147 100644 --- a/lib/galaxy/datatypes/display_applications/parameters.py +++ b/lib/galaxy/datatypes/display_applications/parameters.py @@ -7,35 +7,41 @@ from galaxy.util import string_as_bool from galaxy.util.bunch import Bunch from galaxy.util.template import fill_template -DEFAULT_DATASET_NAME = 'dataset' +DEFAULT_DATASET_NAME = "dataset" class DisplayApplicationParameter: - """ Abstract Class for Display Application Parameters """ + """Abstract Class for Display Application Parameters""" type: Optional[str] = None @classmethod def from_elem(cls, elem, link): - param_type = elem.get('type', None) - assert param_type, 'DisplayApplicationParameter requires a type' + param_type = elem.get("type", None) + assert param_type, "DisplayApplicationParameter requires a type" return parameter_type_to_class[param_type](elem, link) def __init__(self, elem, link): - self.name = elem.get('name', None) - assert self.name, 'DisplayApplicationParameter requires a name' + self.name = elem.get("name", None) + assert self.name, "DisplayApplicationParameter requires a name" self.link = link - self.url = elem.get('url', self.name) # name used in url for display purposes defaults to name; e.g. want the form of file.ext, where a '.' is not allowed as python variable name/keyword - self.mime_type = elem.get('mimetype', None) - self.guess_mime_type = string_as_bool(elem.get('guess_mimetype', 'False')) - self.viewable = string_as_bool(elem.get('viewable', 'False')) # only allow these to be viewed via direct url when explicitly set to viewable - self.strip = string_as_bool(elem.get('strip', 'False')) - self.strip_https = string_as_bool(elem.get('strip_https', 'False')) - self.allow_override = string_as_bool(elem.get('allow_override', 'False')) # Passing query param app_= to dataset controller allows override if this is true. - self.allow_cors = string_as_bool(elem.get('allow_cors', 'False')) + self.url = elem.get( + "url", self.name + ) # name used in url for display purposes defaults to name; e.g. want the form of file.ext, where a '.' is not allowed as python variable name/keyword + self.mime_type = elem.get("mimetype", None) + self.guess_mime_type = string_as_bool(elem.get("guess_mimetype", "False")) + self.viewable = string_as_bool( + elem.get("viewable", "False") + ) # only allow these to be viewed via direct url when explicitly set to viewable + self.strip = string_as_bool(elem.get("strip", "False")) + self.strip_https = string_as_bool(elem.get("strip_https", "False")) + self.allow_override = string_as_bool( + elem.get("allow_override", "False") + ) # Passing query param app_= to dataset controller allows override if this is true. + self.allow_cors = string_as_bool(elem.get("allow_cors", "False")) def get_value(self, other_values, dataset_hash, user_hash, trans): - raise Exception('get_value() is unimplemented for DisplayApplicationDataParameter') + raise Exception("get_value() is unimplemented for DisplayApplicationDataParameter") def prepare(self, other_values, dataset_hash, user_hash, trans): return self.get_value(other_values, dataset_hash, user_hash, trans) @@ -51,40 +57,53 @@ class DisplayApplicationParameter: class DisplayApplicationDataParameter(DisplayApplicationParameter): - """ Parameter that returns a file_name containing the requested content """ + """Parameter that returns a file_name containing the requested content""" - type = 'data' + type = "data" def __init__(self, elem, link): DisplayApplicationParameter.__init__(self, elem, link) - self.extensions = elem.get('format', None) + self.extensions = elem.get("format", None) if self.extensions: self.extensions = self.extensions.split(",") - self.metadata = elem.get('metadata', None) - self.allow_extra_files_access = string_as_bool(elem.get('allow_extra_files_access', 'False')) - self.dataset = elem.get('dataset', DEFAULT_DATASET_NAME) # 'dataset' is default name assigned to dataset to be displayed - assert not (self.extensions and self.metadata), 'A format or a metadata can be defined for a DisplayApplicationParameter, but not both.' - assert not (self.allow_extra_files_access and self.metadata), 'allow_extra_files_access or metadata can be defined for a DisplayApplicationParameter, but not both.' - self.viewable = string_as_bool(elem.get('viewable', 'True')) # data params should be viewable - self.force_url_param = string_as_bool(elem.get('force_url_param', 'False')) - self.force_conversion = string_as_bool(elem.get('force_conversion', 'False')) + self.metadata = elem.get("metadata", None) + self.allow_extra_files_access = string_as_bool(elem.get("allow_extra_files_access", "False")) + self.dataset = elem.get( + "dataset", DEFAULT_DATASET_NAME + ) # 'dataset' is default name assigned to dataset to be displayed + assert not ( + self.extensions and self.metadata + ), "A format or a metadata can be defined for a DisplayApplicationParameter, but not both." + assert not ( + self.allow_extra_files_access and self.metadata + ), "allow_extra_files_access or metadata can be defined for a DisplayApplicationParameter, but not both." + self.viewable = string_as_bool(elem.get("viewable", "True")) # data params should be viewable + self.force_url_param = string_as_bool(elem.get("force_url_param", "False")) + self.force_conversion = string_as_bool(elem.get("force_conversion", "False")) @property def formats(self): if self.extensions: - return tuple(map(type, map(self.link.display_application.app.datatypes_registry.get_datatype_by_extension, self.extensions))) + return tuple( + map( + type, + map( + self.link.display_application.app.datatypes_registry.get_datatype_by_extension, self.extensions + ), + ) + ) return None def _get_dataset_like_object(self, other_values): # this returned object has file_name, state, and states attributes equivalent to a DatasetAssociation data = other_values.get(self.dataset, None) - assert data, 'Base dataset could not be found in values provided to DisplayApplicationDataParameter' + assert data, "Base dataset could not be found in values provided to DisplayApplicationDataParameter" if isinstance(data, DisplayDataValueWrapper): data = data.value if self.metadata: rval = getattr(data.metadata, self.metadata, None) assert rval, f'Unknown metadata name "{self.metadata}" provided for dataset type "{data.ext}".' - return Bunch(file_name=rval.file_name, state=data.state, states=data.states, extension='data') + return Bunch(file_name=rval.file_name, state=data.state, states=data.states, extension="data") elif self.extensions and (self.force_conversion or not isinstance(data.datatype, self.formats)): for ext in self.extensions: rval = data.get_converted_files_by_type(ext) @@ -110,16 +129,26 @@ class DisplayApplicationDataParameter(DisplayApplicationParameter): # start conversion # FIXME: Much of this is copied (more than once...); should be some abstract method elsewhere called from here # find target ext - direct_match, target_ext, converted_dataset = data.find_conversion_destination(self.formats, converter_safe=True) + direct_match, target_ext, converted_dataset = data.find_conversion_destination( + self.formats, converter_safe=True + ) if not direct_match: if target_ext and not converted_dataset: if isinstance(data, DisplayDataValueWrapper): data = data.value - new_data = next(iter(data.datatype.convert_dataset(trans, data, target_ext, return_output=True, visible=False).values())) + new_data = next( + iter( + data.datatype.convert_dataset( + trans, data, target_ext, return_output=True, visible=False + ).values() + ) + ) new_data.hid = data.hid new_data.name = data.name trans.sa_session.add(new_data) - assoc = trans.app.model.ImplicitlyConvertedDatasetAssociation(parent=data, file_type=target_ext, dataset=new_data, metadata_safe=False) + assoc = trans.app.model.ImplicitlyConvertedDatasetAssociation( + parent=data, file_type=target_ext, dataset=new_data, metadata_safe=False + ) trans.sa_session.add(assoc) trans.sa_session.flush() elif converted_dataset and converted_dataset.state == converted_dataset.states.ERROR: @@ -138,18 +167,18 @@ class DisplayApplicationDataParameter(DisplayApplicationParameter): if value.state == value.states.OK: return True elif value.state == value.states.ERROR: - raise Exception(f'A data display parameter is in the error state: {self.name}') + raise Exception(f"A data display parameter is in the error state: {self.name}") return False class DisplayApplicationTemplateParameter(DisplayApplicationParameter): - """ Parameter that returns a string containing the requested content """ + """Parameter that returns a string containing the requested content""" - type = 'template' + type = "template" def __init__(self, elem, link): DisplayApplicationParameter.__init__(self, elem, link) - self.text = elem.text or '' + self.text = elem.text or "" def get_value(self, other_values, dataset_hash, user_hash, trans): value = fill_template(self.text, context=other_values) @@ -158,12 +187,14 @@ class DisplayApplicationTemplateParameter(DisplayApplicationParameter): return DisplayParameterValueWrapper(value, self, other_values, dataset_hash, user_hash, trans) -parameter_type_to_class = {DisplayApplicationDataParameter.type: DisplayApplicationDataParameter, - DisplayApplicationTemplateParameter.type: DisplayApplicationTemplateParameter} +parameter_type_to_class = { + DisplayApplicationDataParameter.type: DisplayApplicationDataParameter, + DisplayApplicationTemplateParameter.type: DisplayApplicationTemplateParameter, +} class DisplayParameterValueWrapper: - ACTION_NAME = 'param' + ACTION_NAME = "param" def __init__(self, value, parameter, other_values, dataset_hash, user_hash, trans): self.value = value @@ -186,22 +217,26 @@ class DisplayParameterValueWrapper: mime = self.trans.app.datatypes_registry.get_mimetype_by_extension(".".split(self._url)[-1], None) if mime: return mime - return 'text/plain' + return "text/plain" @property def url(self): base_url = self.trans.request.base - if self.parameter.strip_https and base_url[: 5].lower() == 'https': + if self.parameter.strip_https and base_url[:5].lower() == "https": base_url = f"http{base_url[5:]}" - return "{}{}".format(base_url, - self.trans.app.url_for(controller='dataset', - action="display_application", - dataset_id=self._dataset_hash, - user_id=self._user_hash, - app_name=quote_plus(self.parameter.link.display_application.id), - link_name=quote_plus(self.parameter.link.id), - app_action=self.action_name, - action_param=self._url)) + return "{}{}".format( + base_url, + self.trans.app.url_for( + controller="dataset", + action="display_application", + dataset_id=self._dataset_hash, + user_id=self._user_hash, + app_name=quote_plus(self.parameter.link.display_application.id), + link_name=quote_plus(self.parameter.link.id), + app_action=self.action_name, + action_param=self._url, + ), + ) @property def action_name(self): @@ -210,14 +245,14 @@ class DisplayParameterValueWrapper: @property def qp(self): # returns quoted str contents - return self.other_values['qp'](str(self)) + return self.other_values["qp"](str(self)) def __getattr__(self, key): return getattr(self.value, key) class DisplayDataValueWrapper(DisplayParameterValueWrapper): - ACTION_NAME = 'data' + ACTION_NAME = "data" def __str__(self): # string of data param is filename @@ -233,12 +268,14 @@ class DisplayDataValueWrapper(DisplayParameterValueWrapper): mime, encoding = mimetypes.guess_type(self._url) if not mime: if action_param_extra: - mime = self.trans.app.datatypes_registry.get_mimetype_by_extension(".".split(action_param_extra)[-1], None) + mime = self.trans.app.datatypes_registry.get_mimetype_by_extension( + ".".split(action_param_extra)[-1], None + ) if not mime: mime = self.trans.app.datatypes_registry.get_mimetype_by_extension(".".split(self._url)[-1], None) if mime: return mime - if hasattr(self.value, 'get_mime'): + if hasattr(self.value, "get_mime"): return self.value.get_mime() return self.other_values[DEFAULT_DATASET_NAME].get_mime() @@ -251,4 +288,4 @@ class DisplayDataValueWrapper(DisplayParameterValueWrapper): @property def qp(self): # returns quoted url contents - return self.other_values['qp'](self.url) + return self.other_values["qp"](self.url) diff --git a/lib/galaxy/datatypes/display_applications/util.py b/lib/galaxy/datatypes/display_applications/util.py index 6af23bf5b94..aefc65d9ff1 100644 --- a/lib/galaxy/datatypes/display_applications/util.py +++ b/lib/galaxy/datatypes/display_applications/util.py @@ -6,7 +6,7 @@ def encode_dataset_user(trans, dataset, user): # encode user id using the dataset create time as the key dataset_hash = trans.security.encode_id(dataset.id) if user is None: - user_hash = 'None' + user_hash = "None" else: security = IdEncodingHelper(id_secret=dataset.create_time) user_hash = security.encode_id(user.id) @@ -19,7 +19,7 @@ def decode_dataset_user(trans, dataset_hash, user_hash): dataset_id = trans.security.decode_id(dataset_hash) dataset = trans.sa_session.query(trans.app.model.HistoryDatasetAssociation).get(dataset_id) assert dataset, "Bad Dataset id provided to decode_dataset_user" - if user_hash in [None, 'None']: + if user_hash in [None, "None"]: user = None else: security = IdEncodingHelper(id_secret=dataset.create_time) diff --git a/lib/galaxy/datatypes/flow.py b/lib/galaxy/datatypes/flow.py index 48a7a4a2495..34f6de4c90b 100644 --- a/lib/galaxy/datatypes/flow.py +++ b/lib/galaxy/datatypes/flow.py @@ -17,6 +17,7 @@ log = logging.getLogger(__name__) @build_sniff_from_prefix class FCS(Binary): """Class describing an FCS binary file""" + file_ext = "fcs" def set_peek(self, dataset): @@ -24,8 +25,8 @@ class FCS(Binary): dataset.peek = "Binary FCS file" dataset.blurb = data.nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -45,8 +46,8 @@ class FCS(Binary): version = content[:6] if version not in ["FCS2.0", "FCS3.0", "FCS3.1"]: return False - if content[6:10] != ' ': + if content[6:10] != " ": return False # we only need to check ioffs 2 to 5 - int(content[10:42].replace(' ', '')) + int(content[10:42].replace(" ", "")) return True diff --git a/lib/galaxy/datatypes/genetics.py b/lib/galaxy/datatypes/genetics.py index 6b32f006ebe..1c10d06ab77 100644 --- a/lib/galaxy/datatypes/genetics.py +++ b/lib/galaxy/datatypes/genetics.py @@ -40,8 +40,8 @@ gal_Log = logging.getLogger(__name__) verbose = False # https://genome.ucsc.edu/goldenpath/help/hgGenomeHelp.html -VALID_GENOME_GRAPH_MARKERS = re.compile(r'^(chr.*|RH.*|rs.*|SNP_.*|CN.*|A_.*)') -VALID_GENOTYPES_LINE = re.compile(r'^([a-zA-Z0-9]+)(\s([0-9]{2}|[A-Z]{2}|NC|\?\?))+\s*$') +VALID_GENOME_GRAPH_MARKERS = re.compile(r"^(chr.*|RH.*|rs.*|SNP_.*|CN.*|A_.*)") +VALID_GENOTYPES_LINE = re.compile(r"^([a-zA-Z0-9]+)(\s([0-9]{2}|[A-Z]{2}|NC|\?\?))+\s*$") @build_sniff_from_prefix @@ -53,22 +53,22 @@ class GenomeGraphs(Tabular): MetadataElement(name="markerCol", default=1, desc="Marker ID column", param=metadata.ColumnParameter) MetadataElement(name="columns", default=3, desc="Number of columns", readonly=True) MetadataElement(name="column_types", default=[], desc="Column types", readonly=True, visible=False) - file_ext = 'gg' + file_ext = "gg" def __init__(self, **kwd): """ Initialize gg datatype, by adding UCSC display apps """ super().__init__(**kwd) - self.add_display_app('ucsc', 'Genome Graph', 'as_ucsc_display_file', 'ucsc_links') + self.add_display_app("ucsc", "Genome Graph", "as_ucsc_display_file", "ucsc_links") def set_meta(self, dataset, **kwd): super().set_meta(dataset, **kwd) dataset.metadata.markerCol = 1 - header = open(dataset.file_name).readlines()[0].strip().split('\t') + header = open(dataset.file_name).readlines()[0].strip().split("\t") dataset.metadata.columns = len(header) - t = ['numeric' for x in header] - t[0] = 'string' + t = ["numeric" for x in header] + t[0] = "string" dataset.metadata.column_types = t return True @@ -76,7 +76,7 @@ class GenomeGraphs(Tabular): """ Returns file """ - return open(dataset.file_name, 'rb') + return open(dataset.file_name, "rb") def ucsc_links(self, dataset, type, app, base_url): """ @@ -97,30 +97,36 @@ class GenomeGraphs(Tabular): """ ret_val = [] if not dataset.dbkey: - dataset.dbkey = 'hg18' # punt! + dataset.dbkey = "hg18" # punt! if dataset.has_data(): - for site_name, site_url in app.datatypes_registry.get_legacy_sites_by_build('ucsc', dataset.dbkey): - if site_name in app.datatypes_registry.get_display_sites('ucsc'): - site_url = site_url.replace('/hgTracks?', '/hgGenome?') # for genome graphs - internal_url = "%s" % app.url_for(controller='dataset', - dataset_id=dataset.id, - action='display_at', - filename=f"ucsc_{site_name}") - display_url = "%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" % (base_url, app.url_for(controller='root'), dataset.id, type) + for site_name, site_url in app.datatypes_registry.get_legacy_sites_by_build("ucsc", dataset.dbkey): + if site_name in app.datatypes_registry.get_display_sites("ucsc"): + site_url = site_url.replace("/hgTracks?", "/hgGenome?") # for genome graphs + internal_url = "%s" % app.url_for( + controller="dataset", dataset_id=dataset.id, action="display_at", filename=f"ucsc_{site_name}" + ) + display_url = "%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" % ( + base_url, + app.url_for(controller="root"), + dataset.id, + type, + ) display_url = quote_plus(display_url) # was display_url = quote_plus( "%s/display_as?id=%i&display_app=%s" % (base_url, dataset.id, type) ) # redirect_url = quote_plus( "%sdb=%s&position=%s:%s-%s&hgt.customText=%%s" % (site_url, dataset.dbkey, chrom, start, stop) ) - sl = [f"{site_url}db={dataset.dbkey}", ] + sl = [ + f"{site_url}db={dataset.dbkey}", + ] # sl.append("&hgt.customText=%s") sl.append(f"&hgGenome_dataSetName={dataset.name}&hgGenome_dataSetDescription=GalaxyGG_data") sl.append("&hgGenome_formatType=best guess&hgGenome_markerType=best guess") sl.append("&hgGenome_columnLabels=first row&hgGenome_maxVal=&hgGenome_labelVals=") sl.append("&hgGenome_doSubmitUpload=submit") sl.append(f"&hgGenome_maxGapToFill=25000000&hgGenome_uploadFile={display_url}") - s = ''.join(sl) + s = "".join(sl) s = quote_plus(s) redirect_url = s - link = f'{internal_url}?redirect_url={redirect_url}&display_url={display_url}' + link = f"{internal_url}?redirect_url={redirect_url}&display_url={display_url}" ret_val.append((site_name, link)) return ret_val @@ -137,21 +143,21 @@ class GenomeGraphs(Tabular): return out hasheader = 0 try: - [f'{x:f}' for x in d[0][1:]] # first is name - see if starts all numerics + [f"{x:f}" for x in d[0][1:]] # first is name - see if starts all numerics except Exception: hasheader = 1 # Generate column header - out.append('') + out.append("") if hasheader: for i, name in enumerate(d[0].split()): - out.append(f'{i + 1}.{name}') + out.append(f"{i + 1}.{name}") d.pop(0) - out.append('') + out.append("") for row in d: - out.append('') - out.append(''.join(f'{x}' for x in row.split())) - out.append('') - out.append('') + out.append("") + out.append("".join(f"{x}" for x in row.split())) + out.append("") + out.append("") out = "".join(out) except Exception as exc: out = f"Can't create peek {exc}" @@ -164,7 +170,7 @@ class GenomeGraphs(Tabular): with open(dataset.file_name) as infile: next(infile) # header for row in infile: - ll = row.strip().split('\t')[1:] # first is alpha feature identifier + ll = row.strip().split("\t")[1:] # first is alpha feature identifier for x in ll: x = float(x) return DatatypeValidation.validated() @@ -203,7 +209,7 @@ class GenomeGraphs(Tabular): def get_mime(self): """Returns the mime type of the datatype""" - return 'application/vnd.ms-excel' + return "application/vnd.ms-excel" class rgTabList(Tabular): @@ -211,6 +217,7 @@ class rgTabList(Tabular): for sampleid and for featureid lists of exclusions or inclusions in the clean tool featureid subsets on statistical criteria -> specialized display such as gg """ + file_ext = "rgTList" def __init__(self, **kwd): @@ -226,7 +233,7 @@ class rgTabList(Tabular): def get_mime(self): """Returns the mime type of the datatype""" - return 'text/html' + return "text/html" class rgSampleList(rgTabList): @@ -237,6 +244,7 @@ class rgSampleList(rgTabList): but they are persistent at least same infrastructure for expression? """ + file_ext = "rgSList" def __init__(self, **kwd): @@ -244,8 +252,8 @@ class rgSampleList(rgTabList): Initialize samplelist datatype """ super().__init__(**kwd) - self.column_names[0] = 'FID' - self.column_names[1] = 'IID' + self.column_names[0] = "FID" + self.column_names[1] = "IID" # this is what Plink wants as at 2009 @@ -256,12 +264,13 @@ class rgFeatureList(rgTabList): featureid subsets on statistical criteria -> specialized display such as gg same infrastructure for expression? """ + file_ext = "rgFList" def __init__(self, **kwd): """Initialize featurelist datatype""" super().__init__(**kwd) - for i, s in enumerate(['#FeatureId', 'Chr', 'Genpos', 'Mappos']): + for i, s in enumerate(["#FeatureId", "Chr", "Genpos", "Mappos"]): self.column_names[i] = s @@ -272,25 +281,32 @@ class Rgenetics(Html): stored in extra files path """ - MetadataElement(name="base_name", desc="base name for all transformed versions of this genetic dataset", default='RgeneticsData', - readonly=True, set_in_upload=True) + MetadataElement( + name="base_name", + desc="base name for all transformed versions of this genetic dataset", + default="RgeneticsData", + readonly=True, + set_in_upload=True, + ) - composite_type = 'auto_primary_file' - file_ext = 'rgenetics' + composite_type = "auto_primary_file" + file_ext = "rgenetics" def generate_primary_file(self, dataset=None): - rval = ['Rgenetics Galaxy Composite Dataset

        '] - rval.append('

        This composite dataset is composed of the following files:

          ') + rval = ["Rgenetics Galaxy Composite Dataset

          "] + rval.append("

          This composite dataset is composed of the following files:

            ") for composite_name, composite_file in self.get_composite_files(dataset=dataset).items(): fn = composite_name - opt_text = '' + opt_text = "" if composite_file.optional: - opt_text = ' (optional)' - if composite_file.get('description'): - rval.append(f"
          • {fn} ({composite_file.get('description')}){opt_text}
          • ") + opt_text = " (optional)" + if composite_file.get("description"): + rval.append( + f"
          • {fn} ({composite_file.get('description')}){opt_text}
          • " + ) else: rval.append(f'
          • {fn}{opt_text}
          • ') - rval.append('
          ') + rval.append("
        ") return "\n".join(rval) def regenerate_primary_file(self, dataset): @@ -299,19 +315,21 @@ class Rgenetics(Html): """ efp = dataset.extra_files_path flist = os.listdir(efp) - rval = [f'Files for Composite Dataset {dataset.name}

        Composite {dataset.name} contains:

          '] + rval = [ + f"Files for Composite Dataset {dataset.name}

          Composite {dataset.name} contains:

            " + ] for fname in flist: sfname = os.path.split(fname)[-1] f, e = os.path.splitext(fname) rval.append(f'
          • {sfname}
          • ') - rval.append('
          ') - with open(dataset.file_name, 'w') as f: + rval.append("
        ") + with open(dataset.file_name, "w") as f: f.write("\n".join(rval)) - f.write('\n') + f.write("\n") def get_mime(self): """Returns the mime type of the datatype""" - return 'text/html' + return "text/html" def set_meta(self, dataset, **kwd): """ @@ -319,31 +337,31 @@ class Rgenetics(Html): """ super().set_meta(dataset, **kwd) - if not kwd.get('overwrite'): + if not kwd.get("overwrite"): if verbose: - gal_Log.debug('@@@ rgenetics set_meta called with overwrite = False') + gal_Log.debug("@@@ rgenetics set_meta called with overwrite = False") return True try: efp = dataset.extra_files_path except Exception: if verbose: - gal_Log.debug(f'@@@rgenetics set_meta failed {sys.exc_info()[0]} - dataset {dataset.name} has no efp ?') + gal_Log.debug(f"@@@rgenetics set_meta failed {sys.exc_info()[0]} - dataset {dataset.name} has no efp ?") return False try: flist = os.listdir(efp) except Exception: if verbose: - gal_Log.debug(f'@@@rgenetics set_meta failed {sys.exc_info()[0]} - dataset {dataset.name} has no efp ?') + gal_Log.debug(f"@@@rgenetics set_meta failed {sys.exc_info()[0]} - dataset {dataset.name} has no efp ?") return False if len(flist) == 0: if verbose: - gal_Log.debug(f'@@@rgenetics set_meta failed - {dataset.name} efp {efp} is empty?') + gal_Log.debug(f"@@@rgenetics set_meta failed - {dataset.name} efp {efp} is empty?") return False self.regenerate_primary_file(dataset) if not dataset.info: - dataset.info = 'Galaxy genotype datatype object' + dataset.info = "Galaxy genotype datatype object" if not dataset.blurb: - dataset.blurb = 'Composite file - Rgenetics Galaxy toolkit' + dataset.blurb = "Composite file - Rgenetics Galaxy toolkit" return True @@ -351,6 +369,7 @@ class SNPMatrix(Rgenetics): """ BioC SNPMatrix Rgenetics data collections """ + file_ext = "snpmatrix" def set_peek(self, dataset, **kwd): @@ -358,16 +377,15 @@ class SNPMatrix(Rgenetics): dataset.peek = "Binary RGenetics file" dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def sniff(self, filename): - """ need to check the file header hex code - """ + """need to check the file header hex code""" with open(filename, "b") as infile: head = infile.read(16) head = [hex(x) for x in head] - if head != '': + if head != "": return False else: return True @@ -377,32 +395,31 @@ class Lped(Rgenetics): """ linkage pedigree (ped,map) Rgenetics data collections """ + file_ext = "lped" def __init__(self, **kwd): super().__init__(**kwd) - self.add_composite_file('%s.ped', - description='Pedigree File', - substitute_name_with_metadata='base_name', - is_binary=False) - self.add_composite_file('%s.map', - description='Map File', - substitute_name_with_metadata='base_name', - is_binary=False) + self.add_composite_file( + "%s.ped", description="Pedigree File", substitute_name_with_metadata="base_name", is_binary=False + ) + self.add_composite_file( + "%s.map", description="Map File", substitute_name_with_metadata="base_name", is_binary=False + ) class Pphe(Rgenetics): """ Plink phenotype file - header must have FID\tIID... Rgenetics data collections """ + file_ext = "pphe" def __init__(self, **kwd): super().__init__(**kwd) - self.add_composite_file('%s.pphe', - description='Plink Phenotype File', - substitute_name_with_metadata='base_name', - is_binary=False) + self.add_composite_file( + "%s.pphe", description="Plink Phenotype File", substitute_name_with_metadata="base_name", is_binary=False + ) class Fphe(Rgenetics): @@ -410,27 +427,26 @@ class Fphe(Rgenetics): fbat pedigree file - mad format with ! as first char on header row Rgenetics data collections """ + file_ext = "fphe" def __init__(self, **kwd): super().__init__(**kwd) - self.add_composite_file('%s.fphe', - description='FBAT Phenotype File', - substitute_name_with_metadata='base_name') + self.add_composite_file("%s.fphe", description="FBAT Phenotype File", substitute_name_with_metadata="base_name") class Phe(Rgenetics): """ Phenotype file """ + file_ext = "phe" def __init__(self, **kwd): super().__init__(**kwd) - self.add_composite_file('%s.phe', - description='Phenotype File', - substitute_name_with_metadata='base_name', - is_binary=False) + self.add_composite_file( + "%s.phe", description="Phenotype File", substitute_name_with_metadata="base_name", is_binary=False + ) class Fped(Rgenetics): @@ -438,26 +454,28 @@ class Fped(Rgenetics): FBAT pedigree format - single file, map is header row of rs numbers. Strange. Rgenetics data collections """ + file_ext = "fped" def __init__(self, **kwd): super().__init__(**kwd) - self.add_composite_file('%s.fped', description='FBAT format pedfile', - substitute_name_with_metadata='base_name', - is_binary=False) + self.add_composite_file( + "%s.fped", description="FBAT format pedfile", substitute_name_with_metadata="base_name", is_binary=False + ) class Pbed(Rgenetics): """ Plink Binary compressed 2bit/geno Rgenetics data collections """ + file_ext = "pbed" def __init__(self, **kwd): super().__init__(**kwd) - self.add_composite_file('%s.bim', substitute_name_with_metadata='base_name', is_binary=False) - self.add_composite_file('%s.bed', substitute_name_with_metadata='base_name', is_binary=True) - self.add_composite_file('%s.fam', substitute_name_with_metadata='base_name', is_binary=False) + self.add_composite_file("%s.bim", substitute_name_with_metadata="base_name", is_binary=False) + self.add_composite_file("%s.bed", substitute_name_with_metadata="base_name", is_binary=True) + self.add_composite_file("%s.fam", substitute_name_with_metadata="base_name", is_binary=False) class ldIndep(Rgenetics): @@ -466,13 +484,14 @@ class ldIndep(Rgenetics): This is really a plink binary, but some tools work better with less redundancy so are constrained to these files """ + file_ext = "ldreduced" def __init__(self, **kwd): super().__init__(**kwd) - self.add_composite_file('%s.bim', substitute_name_with_metadata='base_name', is_binary=False) - self.add_composite_file('%s.bed', substitute_name_with_metadata='base_name', is_binary=True) - self.add_composite_file('%s.fam', substitute_name_with_metadata='base_name', is_binary=False) + self.add_composite_file("%s.bim", substitute_name_with_metadata="base_name", is_binary=False) + self.add_composite_file("%s.bed", substitute_name_with_metadata="base_name", is_binary=True) + self.add_composite_file("%s.fam", substitute_name_with_metadata="base_name", is_binary=False) class Eigenstratgeno(Rgenetics): @@ -481,13 +500,14 @@ class Eigenstratgeno(Rgenetics): if we move to shellfish Rgenetics data collections """ + file_ext = "eigenstratgeno" def __init__(self, **kwd): super().__init__(**kwd) - self.add_composite_file('%s.eigenstratgeno', substitute_name_with_metadata='base_name', is_binary=False) - self.add_composite_file('%s.ind', substitute_name_with_metadata='base_name', is_binary=False) - self.add_composite_file('%s.map', substitute_name_with_metadata='base_name', is_binary=False) + self.add_composite_file("%s.eigenstratgeno", substitute_name_with_metadata="base_name", is_binary=False) + self.add_composite_file("%s.ind", substitute_name_with_metadata="base_name", is_binary=False) + self.add_composite_file("%s.map", substitute_name_with_metadata="base_name", is_binary=False) class Eigenstratpca(Rgenetics): @@ -495,18 +515,21 @@ class Eigenstratpca(Rgenetics): Eigenstrat PCA file for case control adjustment Rgenetics data collections """ + file_ext = "eigenstratpca" def __init__(self, **kwd): super().__init__(**kwd) - self.add_composite_file('%s.eigenstratpca', - description='Eigenstrat PCA file', substitute_name_with_metadata='base_name') + self.add_composite_file( + "%s.eigenstratpca", description="Eigenstrat PCA file", substitute_name_with_metadata="base_name" + ) class Snptest(Rgenetics): """ BioC snptest Rgenetics data collections """ + file_ext = "snptest" @@ -523,21 +546,27 @@ class IdeasPre(Html): - compressed archived tmp directory containing a number of compressed bed files. """ - MetadataElement(name="base_name", desc="Base name for this dataset", default='IDEASData', readonly=True, set_in_upload=True) + MetadataElement( + name="base_name", desc="Base name for this dataset", default="IDEASData", readonly=True, set_in_upload=True + ) MetadataElement(name="chrom_bed", desc="Bed file specifying window positions", default=None, readonly=True) MetadataElement(name="chrom_windows", desc="Chromosome window positions", default=None, readonly=True) MetadataElement(name="input_config", desc="IDEAS input config", default=None, readonly=True) MetadataElement(name="tmp_archive", desc="Compressed archive of compressed bed files", default=None, readonly=True) - composite_type = 'auto_primary_file' - file_ext = 'ideaspre' + composite_type = "auto_primary_file" + file_ext = "ideaspre" def __init__(self, **kwd): super().__init__(**kwd) - self.add_composite_file('chromosome_windows.txt', description='Chromosome window positions', is_binary=False, optional=True) - self.add_composite_file('chromosomes.bed', description='Bed file specifying window positions', is_binary=False, optional=True) - self.add_composite_file('IDEAS_input_config.txt', description='IDEAS input config', is_binary=False) - self.add_composite_file('tmp.tar.gz', description='Compressed archive of compressed bed files', is_binary=True) + self.add_composite_file( + "chromosome_windows.txt", description="Chromosome window positions", is_binary=False, optional=True + ) + self.add_composite_file( + "chromosomes.bed", description="Bed file specifying window positions", is_binary=False, optional=True + ) + self.add_composite_file("IDEAS_input_config.txt", description="IDEAS input config", is_binary=False) + self.add_composite_file("tmp.tar.gz", description="Compressed archive of compressed bed files", is_binary=True) def set_meta(self, dataset, **kwd): super().set_meta(dataset, **kwd) @@ -553,34 +582,35 @@ class IdeasPre(Html): self.regenerate_primary_file(dataset) def generate_primary_file(self, dataset=None): - rval = [''] - rval.append('

        Files prepared for IDEAS

        ') - rval.append('
          ') + rval = [""] + rval.append("

          Files prepared for IDEAS

          ") + rval.append("
            ") for composite_name in self.get_composite_files(dataset=dataset).keys(): fn = composite_name rval.append(f'
          • \n") return "\n".join(rval) def regenerate_primary_file(self, dataset): # Cannot do this until we are setting metadata. - rval = [''] - rval.append('

            Files prepared for IDEAS

            ') - rval.append('
            ") + with open(dataset.file_name, "w") as f: f.write("\n".join(rval)) - f.write('\n') + f.write("\n") class Pheno(Tabular): """ base class for pheno files """ - file_ext = 'pheno' + + file_ext = "pheno" class RexpBase(Html): @@ -589,31 +619,42 @@ class RexpBase(Html): must be constructed with the pheno data in place since that goes into the metadata for each instance """ + MetadataElement(name="columns", default=0, desc="Number of columns", visible=True) MetadataElement(name="column_names", default=[], desc="Column names", visible=True) MetadataElement(name="pheCols", default=[], desc="Select list for potentially interesting variables", visible=True) - MetadataElement(name="base_name", - desc="base name for all transformed versions of this expression dataset", default='rexpression', set_in_upload=True) - MetadataElement(name="pheno_path", desc="Path to phenotype data for this experiment", default="rexpression.pheno", visible=True) - file_ext = 'rexpbase' + MetadataElement( + name="base_name", + desc="base name for all transformed versions of this expression dataset", + default="rexpression", + set_in_upload=True, + ) + MetadataElement( + name="pheno_path", desc="Path to phenotype data for this experiment", default="rexpression.pheno", visible=True + ) + file_ext = "rexpbase" html_table = None - composite_type = 'auto_primary_file' + composite_type = "auto_primary_file" def __init__(self, **kwd): super().__init__(**kwd) - self.add_composite_file('%s.pheno', description='Phenodata tab text file', - substitute_name_with_metadata='base_name', is_binary=False) + self.add_composite_file( + "%s.pheno", + description="Phenodata tab text file", + substitute_name_with_metadata="base_name", + is_binary=False, + ) def generate_primary_file(self, dataset=None): """ This is called only at upload to write the html file cannot rename the datasets here - they come with the default unfortunately """ - return 'AutoGenerated Primary File for Composite Dataset' + return "AutoGenerated Primary File for Composite Dataset" def get_mime(self): """Returns the mime type of the datatype""" - return 'text/html' + return "text/html" def get_phecols(self, phenolist, maxConc=20): """ @@ -632,7 +673,7 @@ class RexpBase(Html): for nrows, row in enumerate(phenolist): # construct concordance if len(row.strip()) == 0: break - row = row.strip().split('\t') + row = row.strip().split("\t") if nrows == 0: # set up from header head = row totcols = len(row) @@ -640,7 +681,10 @@ class RexpBase(Html): else: for col, code in enumerate(row): # keep column order correct if col >= totcols: - gal_Log.warning('### get_phecols error in pheno file - row %d col %d (%s) longer than header %s' % (nrows, col, row, head)) + gal_Log.warning( + "### get_phecols error in pheno file - row %d col %d (%s) longer than header %s" + % (nrows, col, row, head) + ) else: concordance[col].setdefault(code, 0) # first one is zero concordance[col][code] += 1 @@ -656,15 +700,15 @@ class RexpBase(Html): # now to check for pairs of concordant columns - drop one of these. delme = [] p = phenolist[1:] # drop header - plist = [x.strip().split('\t') for x in p] # list of lists + plist = [x.strip().split("\t") for x in p] # list of lists phe = [[x[i] for i in useCols] for x in plist if len(x) >= totcols] # strip unused data for i in range(0, (nuse - 1)): # for each interesting column for j in range(i + 1, nuse): kdict = {} for row in phe: # row is a list of lists - k = f'{row[i]}{row[j]}' # composite key + k = f"{row[i]}{row[j]}" # composite key kdict[k] = k - if (len(kdict.keys()) == len(concordance[useCols[j]])): # i and j are always matched + if len(kdict.keys()) == len(concordance[useCols[j]]): # i and j are always matched delme.append(j) delme = list(set(delme)) # remove dupes listCol = [] @@ -682,7 +726,14 @@ class RexpBase(Html): res = listCol # metadata.pheCols becomes [('bar;22,zot;113','foo'), ...] else: - res = [('no usable phenotype columns found', [('?', 0), ]), ] + res = [ + ( + "no usable phenotype columns found", + [ + ("?", 0), + ], + ), + ] return res def get_pheno(self, dataset): @@ -695,14 +746,14 @@ class RexpBase(Html): """ p = open(dataset.metadata.pheno_path).readlines() if len(p) > 0: # should only need to fix an R pheno file once - head = p[0].strip().split('\t') - line1 = p[1].strip().split('\t') + head = p[0].strip().split("\t") + line1 = p[1].strip().split("\t") if len(head) < len(line1): - head.insert(0, 'ChipFileName') # fix R write.table b0rken-ness - p[0] = '\t'.join(head) + head.insert(0, "ChipFileName") # fix R write.table b0rken-ness + p[0] = "\t".join(head) else: p = [] - return '\n'.join(p) + return "\n".join(p) def set_peek(self, dataset, **kwd): """ @@ -710,41 +761,43 @@ class RexpBase(Html): note that R is weird and does not include the row.name in the header. why?""" if not dataset.dataset.purged: - pp = os.path.join(dataset.extra_files_path, f'{dataset.metadata.base_name}.pheno') + pp = os.path.join(dataset.extra_files_path, f"{dataset.metadata.base_name}.pheno") try: with open(pp) as f: p = f.readlines() except Exception: - p = [f'##failed to find {pp}', ] - dataset.peek = ''.join(p[:5]) - dataset.blurb = 'Galaxy Rexpression composite file' + p = [ + f"##failed to find {pp}", + ] + dataset.peek = "".join(p[:5]) + dataset.blurb = "Galaxy Rexpression composite file" else: - dataset.peek = 'file does not exist\n' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist\n" + dataset.blurb = "file purged from disk" def get_peek(self, dataset): """ expects a .pheno file in the extra_files_dir - ugh """ - pp = os.path.join(dataset.extra_files_path, f'{dataset.metadata.base_name}.pheno') + pp = os.path.join(dataset.extra_files_path, f"{dataset.metadata.base_name}.pheno") try: with open(pp) as f: p = f.readlines() except Exception: - p = [f'##failed to find {pp}'] - return ''.join(p[:5]) + p = [f"##failed to find {pp}"] + return "".join(p[:5]) def get_file_peek(self, filename): """ can't really peek at a filename - need the extra_files_path and such? """ - h = '## rexpression get_file_peek: no file found' + h = "## rexpression get_file_peek: no file found" try: with open(filename) as f: h = f.readlines() except Exception: pass - return ''.join(h[:5]) + return "".join(h[:5]) def regenerate_primary_file(self, dataset): """ @@ -752,14 +805,16 @@ class RexpBase(Html): """ bn = dataset.metadata.base_name flist = os.listdir(dataset.extra_files_path) - rval = [f'Files for Composite Dataset {bn}

            Comprises the following files:

              '] + rval = [ + f"Files for Composite Dataset {bn}

              Comprises the following files:

                " + ] for fname in flist: sfname = os.path.split(fname)[-1] rval.append(f'
              • {sfname}') - rval.append('
              ') - with open(dataset.file_name, 'w') as f: + rval.append("
            ") + with open(dataset.file_name, "w") as f: f.write("\n".join(rval)) - f.write('\n') + f.write("\n") def init_meta(self, dataset, copy_from=None): if copy_from: @@ -776,7 +831,7 @@ class RexpBase(Html): flist = os.listdir(dataset.extra_files_path) except Exception: if verbose: - gal_Log.debug('@@@rexpression set_meta failed - no dataset?') + gal_Log.debug("@@@rexpression set_meta failed - no dataset?") return False bn = dataset.metadata.base_name if not bn: @@ -785,9 +840,9 @@ class RexpBase(Html): bn = n dataset.metadata.base_name = bn if not bn: - bn = '?' + bn = "?" dataset.metadata.base_name = bn - pn = f'{bn}.pheno' + pn = f"{bn}.pheno" pp = os.path.join(dataset.extra_files_path, pn) dataset.metadata.pheno_path = pp try: @@ -797,45 +852,49 @@ class RexpBase(Html): pf = None if pf: h = pf[0].strip() - h = h.split('\t') # hope is header + h = h.split("\t") # hope is header h = [escape(x) for x in h] dataset.metadata.column_names = h dataset.metadata.columns = len(h) - dataset.peek = ''.join(pf[:5]) + dataset.peek = "".join(pf[:5]) else: dataset.metadata.column_names = [] dataset.metadata.columns = 0 - dataset.peek = 'No pheno file found' + dataset.peek = "No pheno file found" if pf and len(pf) > 1: dataset.metadata.pheCols = self.get_phecols(phenolist=pf) else: - dataset.metadata.pheCols = [('', 'No useable phenotypes found', False), ] + dataset.metadata.pheCols = [ + ("", "No useable phenotypes found", False), + ] if not dataset.info: - dataset.info = 'Galaxy Expression datatype object' + dataset.info = "Galaxy Expression datatype object" if not dataset.blurb: - dataset.blurb = 'R loadable BioC expression object for the Rexpression Galaxy toolkit' + dataset.blurb = "R loadable BioC expression object for the Rexpression Galaxy toolkit" return True - def make_html_table(self, pp='nothing supplied from peek\n'): + def make_html_table(self, pp="nothing supplied from peek\n"): """ Create HTML table, used for displaying peek """ - out = ['', ] + out = [ + '
            ', + ] try: # Generate column header - p = pp.split('\n') + p = pp.split("\n") for i, row in enumerate(p): - lrow = row.strip().split('\t') + lrow = row.strip().split("\t") if i == 0: - orow = [f'' for x in lrow] - orow.insert(0, '') - orow.append('') + orow = [f"" for x in lrow] + orow.insert(0, "") + orow.append("") else: - orow = [f'' for x in lrow] - orow.insert(0, '') - orow.append('') - out.append(''.join(orow)) - out.append('
            {escape(x)}
            {escape(x)}
            {escape(x)}
            ') + orow = [f"{escape(x)}" for x in lrow] + orow.insert(0, "") + orow.append("") + out.append("".join(orow)) + out.append("") out = "\n".join(out) except Exception as exc: out = f"Can't create html table {unicodify(exc)}" @@ -858,44 +917,58 @@ class Affybatch(RexpBase): def __init__(self, **kwd): super().__init__(**kwd) - self.add_composite_file('%s.affybatch', - description='AffyBatch R object saved to file', - substitute_name_with_metadata='base_name', is_binary=True) + self.add_composite_file( + "%s.affybatch", + description="AffyBatch R object saved to file", + substitute_name_with_metadata="base_name", + is_binary=True, + ) class Eset(RexpBase): """ derived class for BioC data structures in Galaxy """ + file_ext = "eset" def __init__(self, **kwd): super().__init__(**kwd) - self.add_composite_file('%s.eset', - description='ESet R object saved to file', - substitute_name_with_metadata='base_name', is_binary=True) + self.add_composite_file( + "%s.eset", + description="ESet R object saved to file", + substitute_name_with_metadata="base_name", + is_binary=True, + ) class MAlist(RexpBase): """ derived class for BioC data structures in Galaxy """ + file_ext = "malist" def __init__(self, **kwd): super().__init__(**kwd) - self.add_composite_file('%s.malist', - description='MAlist R object saved to file', - substitute_name_with_metadata='base_name', is_binary=True) + self.add_composite_file( + "%s.malist", + description="MAlist R object saved to file", + substitute_name_with_metadata="base_name", + is_binary=True, + ) class LinkageStudies(Text): """ superclass for classical linkage analysis suites """ + test_files = [ - 'linkstudies.allegro_fparam', 'linkstudies.alohomora_gts', - 'linkstudies.linkage_datain', 'linkstudies.linkage_map' + "linkstudies.allegro_fparam", + "linkstudies.alohomora_gts", + "linkstudies.linkage_datain", + "linkstudies.linkage_map", ] def __init__(self, **kwd): @@ -909,13 +982,14 @@ class GenotypeMatrix(LinkageStudies): Sample matrix of genotypes - GTs as columns """ + file_ext = "alohomora_gts" def __init__(self, **kwd): super().__init__(**kwd) def header_check(self, fio): - header_elems = fio.readline().split('\t') + header_elems = fio.readline().split("\t") if header_elems[0] != "Name": return False @@ -957,7 +1031,7 @@ class GenotypeMatrix(LinkageStudies): if lcount > self.max_lines: return True - tokens = line.split('\t') + tokens = line.split("\t") if num_cols == -1: num_cols = len(tokens) @@ -977,6 +1051,7 @@ class MarkerMap(LinkageStudies): chrom, genetic pos, markername, physical pos, Nr """ + file_ext = "linkage_map" def header_check(self, fio): @@ -1023,7 +1098,7 @@ class MarkerMap(LinkageStudies): try: int(chrm) except ValueError: - if not chrm.lower()[0] in ('x', 'y', 'm'): + if not chrm.lower()[0] in ("x", "y", "m"): return False except ValueError: @@ -1038,6 +1113,7 @@ class DataIn(LinkageStudies): Common linkage input file for intermarker distances and recombination rates """ + file_ext = "linkage_datain" def __init__(self, **kwd): @@ -1108,13 +1184,12 @@ class AllegroLOD(LinkageStudies): """ Allegro output format for LOD scores """ + file_ext = "allegro_fparam" def header_check(self, fio): header = fio.readline().splitlines()[0].split() - if len(header) == 4 and header == [ - "family", "location", "LOD", "marker" - ]: + if len(header) == 4 and header == ["family", "location", "LOD", "marker"]: return True return False @@ -1163,6 +1238,7 @@ class AllegroLOD(LinkageStudies): return True -if __name__ == '__main__': +if __name__ == "__main__": import doctest + doctest.testmod(sys.modules[__name__]) diff --git a/lib/galaxy/datatypes/gis.py b/lib/galaxy/datatypes/gis.py index 02a5f8931dc..040db4495ed 100644 --- a/lib/galaxy/datatypes/gis.py +++ b/lib/galaxy/datatypes/gis.py @@ -6,44 +6,76 @@ from galaxy.datatypes.binary import Binary class Shapefile(Binary): - """ The Shapefile data format: - For more information please see http://en.wikipedia.org/wiki/Shapefile + """The Shapefile data format: + For more information please see http://en.wikipedia.org/wiki/Shapefile """ - composite_type = 'auto_primary_file' + composite_type = "auto_primary_file" file_ext = "shp" def __init__(self, **kwd): super().__init__(**kwd) - self.add_composite_file('shapefile.shp', description='Geometry File (shp)', is_binary=True, optional=False) - self.add_composite_file('shapefile.shx', description='Geometry index File (shx)', is_binary=True, optional=False) - self.add_composite_file('shapefile.dbf', description='Columnar attributes for each shape (dbf)', is_binary=True, optional=False) + self.add_composite_file("shapefile.shp", description="Geometry File (shp)", is_binary=True, optional=False) + self.add_composite_file( + "shapefile.shx", description="Geometry index File (shx)", is_binary=True, optional=False + ) + self.add_composite_file( + "shapefile.dbf", description="Columnar attributes for each shape (dbf)", is_binary=True, optional=False + ) # optional - self.add_composite_file('shapefile.prj', description='Projection description (prj)', is_binary=False, optional=True) - self.add_composite_file('shapefile.sbn', description='Spatial index of the features (sbn)', is_binary=True, optional=True) - self.add_composite_file('shapefile.sbx', description='Spatial index of the features (sbx)', is_binary=True, optional=True) - self.add_composite_file('shapefile.fbn', description='Read only spatial index of the features (fbn)', is_binary=True, optional=True) - self.add_composite_file('shapefile.fbx', description='Read only spatial index of the features (fbx)', is_binary=True, optional=True) - self.add_composite_file('shapefile.ain', description='Attribute index of the active fields in a table (ain)', is_binary=True, optional=True) - self.add_composite_file('shapefile.aih', description='Attribute index of the active fields in a table (aih)', is_binary=True, optional=True) - self.add_composite_file('shapefile.atx', description='Attribute index for the dbf file (atx)', is_binary=True, optional=True) - self.add_composite_file('shapefile.ixs', description='Geocoding index (ixs)', is_binary=True, optional=True) - self.add_composite_file('shapefile.mxs', description='Geocoding index in ODB format (mxs)', is_binary=True, optional=True) - self.add_composite_file('shapefile.shp.xml', description='Geospatial metadata in XML format (xml)', is_binary=False, optional=True) + self.add_composite_file( + "shapefile.prj", description="Projection description (prj)", is_binary=False, optional=True + ) + self.add_composite_file( + "shapefile.sbn", description="Spatial index of the features (sbn)", is_binary=True, optional=True + ) + self.add_composite_file( + "shapefile.sbx", description="Spatial index of the features (sbx)", is_binary=True, optional=True + ) + self.add_composite_file( + "shapefile.fbn", description="Read only spatial index of the features (fbn)", is_binary=True, optional=True + ) + self.add_composite_file( + "shapefile.fbx", description="Read only spatial index of the features (fbx)", is_binary=True, optional=True + ) + self.add_composite_file( + "shapefile.ain", + description="Attribute index of the active fields in a table (ain)", + is_binary=True, + optional=True, + ) + self.add_composite_file( + "shapefile.aih", + description="Attribute index of the active fields in a table (aih)", + is_binary=True, + optional=True, + ) + self.add_composite_file( + "shapefile.atx", description="Attribute index for the dbf file (atx)", is_binary=True, optional=True + ) + self.add_composite_file("shapefile.ixs", description="Geocoding index (ixs)", is_binary=True, optional=True) + self.add_composite_file( + "shapefile.mxs", description="Geocoding index in ODB format (mxs)", is_binary=True, optional=True + ) + self.add_composite_file( + "shapefile.shp.xml", description="Geospatial metadata in XML format (xml)", is_binary=False, optional=True + ) def generate_primary_file(self, dataset=None): - rval = ['Shapefile Galaxy Composite Dataset

            '] - rval.append('

            This composite dataset is composed of the following files:

              ') + rval = ["Shapefile Galaxy Composite Dataset

              "] + rval.append("

              This composite dataset is composed of the following files:

                ") for composite_name, composite_file in self.get_composite_files(dataset=dataset).items(): fn = composite_name - opt_text = '' + opt_text = "" if composite_file.optional: - opt_text = ' (optional)' - if composite_file.get('description'): - rval.append(f"
              • {fn} ({composite_file.get('description')}){opt_text}
              • ") + opt_text = " (optional)" + if composite_file.get("description"): + rval.append( + f"
              • {fn} ({composite_file.get('description')}){opt_text}
              • " + ) else: rval.append(f'
              • {fn}{opt_text}
              • ') - rval.append('
              \n') + rval.append("
            \n") return "\n".join(rval) def set_peek(self, dataset): diff --git a/lib/galaxy/datatypes/goldenpath.py b/lib/galaxy/datatypes/goldenpath.py index 6f200a57fb4..49dd46cecb0 100755 --- a/lib/galaxy/datatypes/goldenpath.py +++ b/lib/galaxy/datatypes/goldenpath.py @@ -16,8 +16,9 @@ from .tabular import Tabular @build_sniff_from_prefix class GoldenPath(Tabular): """Class describing NCBI's Golden Path assembly format""" - edam_format = 'format_3693' - file_ext = 'agp' + + edam_format = "format_3693" + file_ext = "agp" def set_meta(self, dataset, **kwd): # AGPFile reads and validates entire file. @@ -44,21 +45,40 @@ class GoldenPath(Tabular): """ found_non_comment_lines = False try: - for line in iter_headers(file_prefix, '\t', comment_designator='#'): + for line in iter_headers(file_prefix, "\t", comment_designator="#"): if line: if len(line) != 9: return False - assert line[4] in ['A', 'D', 'F', 'G', 'O', 'P', 'W', 'N', 'U'] + assert line[4] in ["A", "D", "F", "G", "O", "P", "W", "N", "U"] ostensible_numbers = line[1:3] - if line[4] in ['U', 'N']: + if line[4] in ["U", "N"]: ostensible_numbers.append(line[5]) - assert line[6] in ['scaffold', 'contig', 'centromere', 'short_arm', 'heterochromatin', 'telomere', 'repeat'] - assert line[7] in ['yes', 'no'] - assert line[8] in ['na', 'paired-ends', 'align_genus', 'align_xgenus', 'align_trnscript', 'within_clone', 'clone_contig', 'map', 'strobe', 'unspecified'] + assert line[6] in [ + "scaffold", + "contig", + "centromere", + "short_arm", + "heterochromatin", + "telomere", + "repeat", + ] + assert line[7] in ["yes", "no"] + assert line[8] in [ + "na", + "paired-ends", + "align_genus", + "align_xgenus", + "align_trnscript", + "within_clone", + "clone_contig", + "map", + "strobe", + "unspecified", + ] else: ostensible_numbers.extend([line[6], line[7]]) - assert line[8] in ['+', '-', '?', '0', 'na'] - if line[4] == 'U': + assert line[8] in ["+", "-", "?", "0", "na"] + if line[4] == "U": assert int(line[5]) == 100 assert all(map(lambda x: str(x).isnumeric() and int(x) > 0, ostensible_numbers)) found_non_comment_lines = True @@ -192,16 +212,16 @@ class AGPFile: @property def num_lines(self): - """ Calculate the number of lines in the current state of the AGP file. """ + """Calculate the number of lines in the current state of the AGP file.""" return len(self._comment_lines) + sum(obj.num_lines for obj in self._objects) def iterate_objs(self): - """ Iterate over the objects of the AGP file. """ + """Iterate over the objects of the AGP file.""" for obj in self._objects: yield obj def iterate_lines(self): - """ Iterate over the non-comment lines of AGP file. """ + """Iterate over the non-comment lines of AGP file.""" for obj in self.iterate_objs(): for j in obj.iterate_lines(): yield j @@ -271,7 +291,11 @@ class AGPObject: # Check that the object intervals are sequential if self.obj_intervals: if self.obj_intervals[-1][1] != agp_line.obj_beg - 1: - raise AGPError(self.fname, agp_line.line_number, f"some positions in {agp_line.obj} are not accounted for or overlapping") + raise AGPError( + self.fname, + agp_line.line_number, + f"some positions in {agp_line.obj} are not accounted for or overlapping", + ) self.previous_pid = agp_line.pid self.obj_intervals.append((agp_line.obj_beg - 1, agp_line.obj_end)) @@ -312,27 +336,29 @@ class AGPLine(object, metaclass=abc.ABCMeta): @abc.abstractmethod def __str__(self): - """ Return the tab delimited AGP line""" + """Return the tab delimited AGP line""" pass @abc.abstractmethod def __iter__(self): - """ Return the AGP line's iterator""" + """Return the AGP line's iterator""" pass @abc.abstractmethod def _validate_numerics(self): - """ Ensure all numeric fields and positive integers. """ + """Ensure all numeric fields and positive integers.""" pass @abc.abstractmethod def _validate_strings(self): - """ Ensure all text fields are strings. """ + """Ensure all text fields are strings.""" pass def _validate_obj_coords(self): if self.obj_beg > self.obj_end: - raise AGPError(self.fname, self.line_number, f"object_beg ({self.obj_beg}) must be <= object_end ({self.obj_end})") + raise AGPError( + self.fname, self.line_number, f"object_beg ({self.obj_beg}) must be <= object_end ({self.obj_end})" + ) def _validate_component_type(self): if self.comp_type not in self.allowed_comp_types: @@ -340,7 +366,7 @@ class AGPLine(object, metaclass=abc.ABCMeta): @abc.abstractmethod def _validate_line(self): - """ Final remaining validations specific to the gap or sequence AGP lines. """ + """Final remaining validations specific to the gap or sequence AGP lines.""" pass @@ -353,7 +379,9 @@ class AGPSeqLine(AGPLine): allowed_comp_types = {"A", "D", "F", "G", "O", "P", "W"} allowed_orientations = {"+", "-", "?", "0", "na"} - def __init__(self, fname, line_number, obj, obj_beg, obj_end, pid, comp_type, comp, comp_beg, comp_end, orientation): + def __init__( + self, fname, line_number, obj, obj_beg, obj_end, pid, comp_type, comp, comp_beg, comp_end, orientation + ): self.comp = comp self.comp_beg = comp_beg self.comp_end = comp_end @@ -363,28 +391,32 @@ class AGPSeqLine(AGPLine): super(AGPSeqLine, self).__init__(fname, line_number, obj, obj_beg, obj_end, pid, comp_type) self.is_gap = False - self.seqdict = dict(obj=str(self.obj), - obj_beg=int(self.obj_beg), - obj_end=int(self.obj_end), - pid=int(self.pid), - comp_type=str(self.comp_type), - comp=str(self.comp), - comp_beg=int(self.comp_beg), - comp_end=int(self.comp_end), - orientation=str(self.orientation)) + self.seqdict = dict( + obj=str(self.obj), + obj_beg=int(self.obj_beg), + obj_end=int(self.obj_end), + pid=int(self.pid), + comp_type=str(self.comp_type), + comp=str(self.comp), + comp_beg=int(self.comp_beg), + comp_end=int(self.comp_end), + orientation=str(self.orientation), + ) def __str__(self): - return "\t".join([ - self.obj, - str(self.obj_beg), - str(self.obj_end), - str(self.pid), - self.comp_type, - self.comp, - str(self.comp_beg), - str(self.comp_end), - self.orientation - ]) + return "\t".join( + [ + self.obj, + str(self.obj_beg), + str(self.obj_end), + str(self.pid), + self.comp_type, + self.comp, + str(self.comp_beg), + str(self.comp_end), + self.orientation, + ] + ) def __iter__(self): for key in self.seqdict: @@ -403,21 +435,23 @@ class AGPSeqLine(AGPLine): raise AGPError(self.fname, self.line_number, "encountered an invalid non-integer numeric AGP field") # Ensure that all numeric values are positive - if not all([ - self.obj_beg > 0, - self.obj_end > 0, - self.pid > 0, - self.comp_beg > 0, - self.comp_end > 0 - ]): + if not all([self.obj_beg > 0, self.obj_end > 0, self.pid > 0, self.comp_beg > 0, self.comp_end > 0]): raise AGPError(self.fname, self.line_number, "encountered an invalid zero or negative numeric AGP field.") # Check the coordinates if self.comp_beg > self.comp_end: - raise AGPError(self.fname, self.line_number, f"component_beg ({self.comp_beg}) must be <= component_end ({self.comp_end})") + raise AGPError( + self.fname, + self.line_number, + f"component_beg ({self.comp_beg}) must be <= component_end ({self.comp_end})", + ) if self.obj_end - (self.obj_beg - 1) != self.comp_end - (self.comp_beg - 1): - raise AGPError(self.fname, self.line_number, f"object coordinates ({self.obj_beg}, {self.obj_end}) and component coordinates ({self.comp_beg}, {self.comp_end}) do not have the same length") + raise AGPError( + self.fname, + self.line_number, + f"object coordinates ({self.obj_beg}, {self.obj_end}) and component coordinates ({self.comp_beg}, {self.comp_end}) do not have the same length", + ) def _validate_strings(self): try: @@ -442,15 +476,33 @@ class AGPGapLine(AGPLine): allowed_comp_types = {"N", "U"} allowed_linkage_types = {"yes", "no"} allowed_gap_types = { - "scaffold", "contig", "centromere", "short_arm", "heterochromatin", "telomere", "repeat", "contamination" + "scaffold", + "contig", + "centromere", + "short_arm", + "heterochromatin", + "telomere", + "repeat", + "contamination", } allowed_evidence_types = { - "na", "paired-ends", "align_genus", "align_xgenus", - "align_trnscpt", "within_clone", "clone_contig", "map", - "pcr", "proximity_ligation", "strobe", "unspecified" + "na", + "paired-ends", + "align_genus", + "align_xgenus", + "align_trnscpt", + "within_clone", + "clone_contig", + "map", + "pcr", + "proximity_ligation", + "strobe", + "unspecified", } - def __init__(self, fname, line_number, obj, obj_beg, obj_end, pid, comp_type, gap_len, gap_type, linkage, linkage_evidence): + def __init__( + self, fname, line_number, obj, obj_beg, obj_end, pid, comp_type, gap_len, gap_type, linkage, linkage_evidence + ): self.gap_len = gap_len self.gap_type = gap_type self.linkage = linkage @@ -460,28 +512,32 @@ class AGPGapLine(AGPLine): super(AGPGapLine, self).__init__(fname, line_number, obj, obj_beg, obj_end, pid, comp_type) self.is_gap = True - self.gapdict = dict(obj=str(self.obj), - obj_beg=int(self.obj_beg), - obj_end=int(self.obj_end), - pid=int(self.pid), - comp_type=str(self.comp_type), - gap_len=int(self.gap_len), - gap_type=str(self.gap_type), - linkage=str(self.linkage), - linkage_evidence=str(self.linkage_evidence)) + self.gapdict = dict( + obj=str(self.obj), + obj_beg=int(self.obj_beg), + obj_end=int(self.obj_end), + pid=int(self.pid), + comp_type=str(self.comp_type), + gap_len=int(self.gap_len), + gap_type=str(self.gap_type), + linkage=str(self.linkage), + linkage_evidence=str(self.linkage_evidence), + ) def __str__(self): - return "\t".join([ - self.obj, - str(self.obj_beg), - str(self.obj_end), - str(self.pid), - self.comp_type, - str(self.gap_len), - self.gap_type, - self.linkage, - self.linkage_evidence - ]) + return "\t".join( + [ + self.obj, + str(self.obj_beg), + str(self.obj_end), + str(self.pid), + self.comp_type, + str(self.gap_len), + self.gap_type, + self.linkage, + self.linkage_evidence, + ] + ) def __iter__(self): for key in self.gapdict: @@ -499,17 +555,16 @@ class AGPGapLine(AGPLine): raise AGPError(self.fname, self.line_number, "encountered an invalid non-integer numeric AGP field") # Ensure that all numeric values are positive - if not all([ - self.obj_beg > 0, - self.obj_end > 0, - self.pid > 0, - self.gap_len > 0 - ]): + if not all([self.obj_beg > 0, self.obj_end > 0, self.pid > 0, self.gap_len > 0]): raise AGPError(self.fname, self.line_number, "encountered an invalid negative numeric AGP field") # Make sure the coordinates match if self.obj_end - (self.obj_beg - 1) != self.gap_len: - raise AGPError(self.fname, self.line_number, f"object coordinates ({self.obj_beg}, {self.obj_end}) and gap length ({self.gap_len}) are not the same length") + raise AGPError( + self.fname, + self.line_number, + f"object coordinates ({self.obj_beg}, {self.obj_end}) and gap length ({self.gap_len}) are not the same length", + ) def _validate_strings(self): try: @@ -522,9 +577,13 @@ class AGPGapLine(AGPLine): raise AGPError(self.fname, self.line_number, "encountered an invalid type for an AGP text field") def _validate_line(self): - """ Validation specific to AGP gap lines. """ + """Validation specific to AGP gap lines.""" if self.comp_type == "U" and self.gap_len != 100: - raise AGPError(self.fname, self.line_number, f"invalid gap length for component type 'U': {self.gap_len} (should be 100)") + raise AGPError( + self.fname, + self.line_number, + f"invalid gap length for component type 'U': {self.gap_len} (should be 100)", + ) if self.gap_type not in AGPGapLine.allowed_gap_types: raise AGPError(self.fname, self.line_number, f"invalid gap type: {self.gap_type}") @@ -542,7 +601,11 @@ class AGPGapLine(AGPLine): raise AGPError(self.fname, self.line_number, "invalid 'scaffold' gap without linkage evidence") if self.linkage_evidence != "na": - raise AGPError(self.fname, self.line_number, f"linkage evidence must be 'na' when not asserting linkage. Got {self.linkage_evidence}") + raise AGPError( + self.fname, + self.line_number, + f"linkage evidence must be 'na' when not asserting linkage. Got {self.linkage_evidence}", + ) else: if "na" in all_evidence: raise AGPError(self.fname, self.line_number, "'na' is invalid linkage evidence when asserting linkage") diff --git a/lib/galaxy/datatypes/graph.py b/lib/galaxy/datatypes/graph.py index 94f0156b1f4..7b3f5c97cfb 100644 --- a/lib/galaxy/datatypes/graph.py +++ b/lib/galaxy/datatypes/graph.py @@ -8,7 +8,7 @@ from . import ( data, dataproviders, tabular, - xml + xml, ) log = logging.getLogger(__name__) @@ -20,6 +20,7 @@ class Xgmml(xml.GenericXml): XGMML graph format (http://wiki.cytoscape.org/Cytoscape_User_Manual/Network_Formats). """ + file_ext = "xgmml" def set_peek(self, dataset): @@ -28,10 +29,10 @@ class Xgmml(xml.GenericXml): """ if not dataset.dataset.purged: dataset.peek = data.get_file_peek(dataset.file_name) - dataset.blurb = 'XGMML data' + dataset.blurb = "XGMML data" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def sniff(self, filename): """ @@ -45,12 +46,13 @@ class Xgmml(xml.GenericXml): Merging multiple XML files is non-trivial and must be done in subclasses. """ if len(split_files) > 1: - raise NotImplementedError("Merging multiple XML files is non-trivial " - + "and must be implemented for each XML type") + raise NotImplementedError( + "Merging multiple XML files is non-trivial " + "and must be implemented for each XML type" + ) # For one file only, use base class method (move/copy) data.Text.merge(split_files, output_file) - @dataproviders.decorators.dataprovider_factory('node-edge', dataproviders.hierarchy.XMLDataProvider.settings) + @dataproviders.decorators.dataprovider_factory("node-edge", dataproviders.hierarchy.XMLDataProvider.settings) def node_edge_dataprovider(self, dataset, **settings): dataset_source = dataproviders.dataset.DatasetDataProvider(dataset) return XGMMLGraphDataProvider(dataset_source, **settings) @@ -66,6 +68,7 @@ class Sif(tabular.Tabular): Second column: relationship type Third to Nth column: target ids for link """ + file_ext = "sif" def set_peek(self, dataset): @@ -74,10 +77,10 @@ class Sif(tabular.Tabular): """ if not dataset.dataset.purged: dataset.peek = data.get_file_peek(dataset.file_name) - dataset.blurb = 'SIF data' + dataset.blurb = "SIF data" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def sniff(self, filename): """ @@ -89,7 +92,7 @@ class Sif(tabular.Tabular): def merge(split_files, output_file): data.Text.merge(split_files, output_file) - @dataproviders.decorators.dataprovider_factory('node-edge', dataproviders.column.ColumnarDataProvider.settings) + @dataproviders.decorators.dataprovider_factory("node-edge", dataproviders.column.ColumnarDataProvider.settings) def node_edge_dataprovider(self, dataset, **settings): dataset_source = dataproviders.dataset.DatasetDataProvider(dataset) return SIFGraphDataProvider(dataset_source, **settings) @@ -113,18 +116,18 @@ class XGMMLGraphDataProvider(dataproviders.hierarchy.XMLDataProvider): parent_gen = super().__iter__() for graph_elem in parent_gen: - if 'children' not in graph_elem: + if "children" not in graph_elem: continue - for elem in graph_elem['children']: + for elem in graph_elem["children"]: # use endswith to work around Elementtree namespaces - if elem['tag'].endswith('node'): - node_id = elem['attrib']['id'] + if elem["tag"].endswith("node"): + node_id = elem["attrib"]["id"] # pass the entire, parsed xml element as the data graph.add_node(node_id, **elem) - elif elem['tag'].endswith('edge'): - source_id = elem['attrib']['source'] - target_id = elem['attrib']['target'] + elif elem["tag"].endswith("edge"): + source_id = elem["attrib"]["source"] + target_id = elem["attrib"]["target"] graph.add_edge(source_id, target_id, **elem) yield graph.as_dict() diff --git a/lib/galaxy/datatypes/hdf5.py b/lib/galaxy/datatypes/hdf5.py index 388998bf11e..13a9e57126d 100644 --- a/lib/galaxy/datatypes/hdf5.py +++ b/lib/galaxy/datatypes/hdf5.py @@ -28,20 +28,20 @@ class HDF5SummarizedExperiment(Data): set_in_upload=True, ) - file_ext = 'rdata.se' - composite_type = 'auto_primary_file' + file_ext = "rdata.se" + composite_type = "auto_primary_file" allow_datatype_change = False def __init__(self, **kwd): """Construct object from input files.""" Data.__init__(self, **kwd) self.add_composite_file( - 'se.rds', + "se.rds", is_binary=True, description="Summarized experiment RDS object", ) self.add_composite_file( - 'assays.h5', + "assays.h5", is_binary=True, description="Summarized experiment data array", ) @@ -52,8 +52,7 @@ class HDF5SummarizedExperiment(Data): def generate_primary_file(self, dataset=None): """Generate primary file to represent dataset.""" - return ( - f''' + return f""" Files for Composite Dataset ({self.file_ext}) @@ -66,8 +65,7 @@ class HDF5SummarizedExperiment(Data):
          • array.h5
          - ''' - ) + """ def sniff(self, filename): """Not sure whether this is necessary (or possible) with binaries.""" @@ -75,4 +73,4 @@ class HDF5SummarizedExperiment(Data): def get_mime(self): """Return the mime type of the datatype.""" - return 'text/html' + return "text/html" diff --git a/lib/galaxy/datatypes/images.py b/lib/galaxy/datatypes/images.py index faaefbc7d39..3ef57bb3620 100644 --- a/lib/galaxy/datatypes/images.py +++ b/lib/galaxy/datatypes/images.py @@ -42,9 +42,10 @@ log = logging.getLogger(__name__) class Image(data.Data): """Class describing an image""" - edam_data = 'data_2968' + + edam_data = "data_2968" edam_format = "format_3547" - file_ext = '' + file_ext = "" def __init__(self, **kwd): super().__init__(**kwd) @@ -52,11 +53,11 @@ class Image(data.Data): def set_peek(self, dataset): if not dataset.dataset.purged: - dataset.peek = f'Image in {dataset.extension} format' + dataset.peek = f"Image in {dataset.extension} format" dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def sniff(self, filename): """Determine if the file is in this format""" @@ -64,7 +65,7 @@ class Image(data.Data): def handle_dataset_as_image(self, hda): dataset = hda.dataset - name = hda.name or '' + name = hda.name or "" with open(dataset.file_name, "rb") as f: base64_image_data = base64.b64encode(f.read()).decode("utf-8") return f"![{name}](data:image/{self.file_ext};base64,{base64_image_data})" @@ -76,7 +77,7 @@ class Jpg(Image): def __init__(self, **kwd): super().__init__(**kwd) - self.image_formats = ['JPEG'] + self.image_formats = ["JPEG"] class Png(Image): @@ -91,16 +92,24 @@ class Tiff(Image): class OMETiff(Tiff): file_ext = "ome.tiff" - MetadataElement(name="offsets", desc="Offsets File", param=FileParameter, file_ext="json", readonly=True, visible=False, optional=True) + MetadataElement( + name="offsets", + desc="Offsets File", + param=FileParameter, + file_ext="json", + readonly=True, + visible=False, + optional=True, + ) def set_meta(self, dataset, overwrite=True, **kwd): - spec_key = 'offsets' + spec_key = "offsets" offsets_file = dataset.metadata.offsets if not offsets_file: offsets_file = dataset.metadata.spec[spec_key].param.new_file(dataset=dataset) with tifffile.TiffFile(dataset.file_name) as tif: offsets = [page.offset for page in tif.pages] - with open(offsets_file.file_name, 'w') as f: + with open(offsets_file.file_name, "w") as f: json.dump(offsets, f) dataset.metadata.offsets = offsets_file @@ -203,7 +212,7 @@ class Pdf(Image): def sniff(self, filename): """Determine if the file is in pdf format.""" - with open(filename, 'rb') as fh: + with open(filename, "rb") as fh: return fh.read(4) == b"%PDF" @@ -242,11 +251,17 @@ class Tck(Binary): >>> Tck().sniff( fname ) False """ - file_ext = 'tck' + + file_ext = "tck" def sniff_prefix(self, file_prefix: FilePrefix): - format_def = [[b'mrtrix tracks'], [b'datatype: Float32LE', b'datatype: Float32BE', b'datatype: Float64BE', b'datatype: Float64LE'], - [b'count: '], [b'file: .'], [b'END']] + format_def = [ + [b"mrtrix tracks"], + [b"datatype: Float32LE", b"datatype: Float32BE", b"datatype: Float64BE", b"datatype: Float64LE"], + [b"count: "], + [b"file: ."], + [b"END"], + ] matches = 0 for elem in format_def: @@ -272,40 +287,41 @@ class Trk(Binary): >>> Trk().sniff( fname ) False """ - file_ext = 'trk' + + file_ext = "trk" def sniff_prefix(self, file_prefix: FilePrefix): # quick check header_raw = None header_raw = file_prefix.contents_header_bytes[:1000] - if header_raw[:5] != b'TRACK': + if header_raw[:5] != b"TRACK": return False # detailed check header_def = [ - ('magic', 'S6'), - ('dim', 'h', 3), - ('voxel_size', 'f4', 3), - ('origin', 'f4', 3), - ('n_scalars', 'h'), - ('scalar_name', 'S20', 10), - ('n_properties', 'h'), - ('property_name', 'S20', 10), - ('vox_to_ras', 'f4', (4, 4)), - ('reserved', 'S444'), - ('voxel_order', 'S4'), - ('pad2', 'S4'), - ('image_orientation_patient', 'f4', 6), - ('pad1', 'S2'), - ('invert_x', 'S1'), - ('invert_y', 'S1'), - ('invert_z', 'S1'), - ('swap_xy', 'S1'), - ('swap_yz', 'S1'), - ('swap_zx', 'S1'), - ('n_count', 'i4'), - ('version', 'i4'), - ('header_size', 'i4'), + ("magic", "S6"), + ("dim", "h", 3), + ("voxel_size", "f4", 3), + ("origin", "f4", 3), + ("n_scalars", "h"), + ("scalar_name", "S20", 10), + ("n_properties", "h"), + ("property_name", "S20", 10), + ("vox_to_ras", "f4", (4, 4)), + ("reserved", "S444"), + ("voxel_order", "S4"), + ("pad2", "S4"), + ("image_orientation_patient", "f4", 6), + ("pad1", "S2"), + ("invert_x", "S1"), + ("invert_y", "S1"), + ("invert_z", "S1"), + ("swap_xy", "S1"), + ("swap_yz", "S1"), + ("swap_zx", "S1"), + ("n_count", "i4"), + ("version", "i4"), + ("header_size", "i4"), ] np_dtype = np.dtype(header_def) header: np.ndarray = np.ndarray(shape=(), dtype=np_dtype, buffer=header_raw) @@ -332,7 +348,8 @@ class Mrc2014(Binary): >>> Mrc2014().sniff(fname) False """ - file_ext = 'mrc' + + file_ext = "mrc" def sniff(self, filename): # Handle the wierdness of mrcfile: @@ -350,31 +367,43 @@ class Mrc2014(Binary): class Gmaj(data.Data): """Class describing a GMAJ Applet""" + edam_format = "format_3547" file_ext = "gmaj.zip" copy_safe_peek = False def set_peek(self, dataset): if not dataset.dataset.purged: - if hasattr(dataset, 'history_id'): + if hasattr(dataset, "history_id"): params = { "bundle": f"display?id={dataset.id}&tofile=yes&toext=.zip", "buttonlabel": "Launch GMAJ", "nobutton": "false", "urlpause": "100", "debug": "false", - "posturl": "history_add_to?%s" % "&".join(f"{x[0]}={quote_plus(str(x[1]))}" for x in [('copy_access_from', dataset.id), ('history_id', dataset.history_id), ('ext', 'maf'), ('name', f'GMAJ Output on data {dataset.hid}'), ('info', 'Added by GMAJ'), ('dbkey', dataset.dbkey)]) + "posturl": "history_add_to?%s" + % "&".join( + f"{x[0]}={quote_plus(str(x[1]))}" + for x in [ + ("copy_access_from", dataset.id), + ("history_id", dataset.history_id), + ("ext", "maf"), + ("name", f"GMAJ Output on data {dataset.hid}"), + ("info", "Added by GMAJ"), + ("dbkey", dataset.dbkey), + ] + ), } class_name = "edu.psu.bx.gmaj.MajApplet.class" archive = "/static/gmaj/gmaj.jar" dataset.peek = create_applet_tag_peek(class_name, archive, params) - dataset.blurb = 'GMAJ Multiple Alignment Viewer' + dataset.blurb = "GMAJ Multiple Alignment Viewer" else: dataset.peek = "After you add this item to your history, you will be able to launch the GMAJ applet." - dataset.blurb = 'GMAJ Multiple Alignment Viewer' + dataset.blurb = "GMAJ Multiple Alignment Viewer" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -384,7 +413,7 @@ class Gmaj(data.Data): def get_mime(self): """Returns the mime type of the datatype""" - return 'application/zip' + return "application/zip" def sniff(self, filename): """ @@ -397,7 +426,7 @@ class Gmaj(data.Data): contains_gmaj_file = False with zipfile.ZipFile(filename, "r") as zip_file: for name in zip_file.namelist(): - if name.split(".")[1].strip().lower() == 'gmaj': + if name.split(".")[1].strip().lower() == "gmaj": contains_gmaj_file = True break if not contains_gmaj_file: @@ -407,48 +436,41 @@ class Gmaj(data.Data): class Analyze75(Binary): """ - Mayo Analyze 7.5 files - http://www.imzml.org + Mayo Analyze 7.5 files + http://www.imzml.org """ - file_ext = 'analyze75' - composite_type = 'auto_primary_file' + + file_ext = "analyze75" + composite_type = "auto_primary_file" def __init__(self, **kwd): super().__init__(**kwd) # The header file provides information about dimensions, identification, # and processing history. - self.add_composite_file( - 'hdr', - description='The Analyze75 header file.', - is_binary=True) + self.add_composite_file("hdr", description="The Analyze75 header file.", is_binary=True) # The image file contains the actual data, whose data type and ordering # are described by the header file. - self.add_composite_file( - 'img', - description='The Analyze75 image file.', - is_binary=True) + self.add_composite_file("img", description="The Analyze75 image file.", is_binary=True) - self.add_composite_file( - 't2m', - description='The Analyze75 t2m file.', - optional=True, - is_binary=True) + self.add_composite_file("t2m", description="The Analyze75 t2m file.", optional=True, is_binary=True) def generate_primary_file(self, dataset=None): - rval = ['Analyze75 Composite Dataset.

          '] - rval.append('

          This composite dataset is composed of the following files:

            ') + rval = ["Analyze75 Composite Dataset.

            "] + rval.append("

            This composite dataset is composed of the following files:

              ") for composite_name, composite_file in self.get_composite_files(dataset=dataset).items(): fn = composite_name - opt_text = '' + opt_text = "" if composite_file.optional: - opt_text = ' (optional)' - if composite_file.get('description'): - rval.append(f"
            • {fn} ({composite_file.get('description')}){opt_text}
            • ") + opt_text = " (optional)" + if composite_file.get("description"): + rval.append( + f"
            • {fn} ({composite_file.get('description')}){opt_text}
            • " + ) else: rval.append(f'
            • {fn}{opt_text}
            • ') - rval.append('
            ') + rval.append("
          ") return "\n".join(rval) @@ -466,11 +488,12 @@ class Nifti1(Binary): >>> Nifti1().sniff( fname ) False """ - file_ext = 'nii1' + + file_ext = "nii1" def sniff_prefix(self, file_prefix: FilePrefix): magic = file_prefix.contents_header_bytes[344:348] - if magic == b'n+1\0': + if magic == b"n+1\0": return True return False @@ -489,11 +512,12 @@ class Nifti2(Binary): >>> Nifti2().sniff( fname ) False """ - file_ext = 'nii2' + + file_ext = "nii2" def sniff_prefix(self, file_prefix: FilePrefix): magic = file_prefix.contents_header_bytes[4:8] - if magic in [b'n+2\0', b'ni2\0']: + if magic in [b"n+2\0", b"ni2\0"]: return True return False @@ -501,6 +525,7 @@ class Nifti2(Binary): @build_sniff_from_prefix class Gifti(GenericXml): """Class describing a Gifti format""" + file_ext = "gii" def sniff_prefix(self, file_prefix: FilePrefix): @@ -528,7 +553,7 @@ class Gifti(GenericXml): if line.strip() == '': return True line = handle.readline() - if line.strip().startswith(' 2: - if overwrite or not dataset.metadata.element_is_set('chromCol'): + if overwrite or not dataset.metadata.element_is_set("chromCol"): dataset.metadata.chromCol = 1 try: int(elems[1]) - if overwrite or not dataset.metadata.element_is_set('startCol'): + if overwrite or not dataset.metadata.element_is_set("startCol"): dataset.metadata.startCol = 2 except Exception: pass # Metadata default will be used try: int(elems[2]) - if overwrite or not dataset.metadata.element_is_set('endCol'): + if overwrite or not dataset.metadata.element_is_set("endCol"): dataset.metadata.endCol = 3 except Exception: pass # Metadata default will be used @@ -127,10 +156,10 @@ class Interval(Tabular): # if overwrite or not dataset.metadata.element_is_set( 'nameCol' ): # dataset.metadata.nameCol = 4 if len(elems) < 6 or elems[5] not in data.valid_strand: - if overwrite or not dataset.metadata.element_is_set('strandCol'): + if overwrite or not dataset.metadata.element_is_set("strandCol"): dataset.metadata.strandCol = 0 else: - if overwrite or not dataset.metadata.element_is_set('strandCol'): + if overwrite or not dataset.metadata.element_is_set("strandCol"): dataset.metadata.strandCol = 6 break if (i - empty_line_count) > num_check_lines: @@ -140,13 +169,15 @@ class Interval(Tabular): def displayable(self, dataset): try: - return dataset.has_data() \ - and dataset.state == dataset.states.OK \ - and dataset.metadata.columns > 0 \ - and dataset.metadata.data_lines != 0 \ - and dataset.metadata.chromCol \ - and dataset.metadata.startCol \ + return ( + dataset.has_data() + and dataset.state == dataset.states.OK + and dataset.metadata.columns > 0 + and dataset.metadata.data_lines != 0 + and dataset.metadata.chromCol + and dataset.metadata.startCol and dataset.metadata.endCol + ) except Exception: return False @@ -172,9 +203,9 @@ class Interval(Tabular): with compression_utils.get_fileobj(dataset.file_name) as fh: for line in util.iter_start_of_line(fh, VIEWPORT_READLINE_BUFFER_SIZE): # Skip comment lines - if not line.startswith('#'): + if not line.startswith("#"): try: - fields = line.rstrip().split('\t') + fields = line.rstrip().split("\t") if len(fields) > max_col: if chrom is None or chrom == fields[chrom_col]: start = min(start, int(fields[start_col])) @@ -189,8 +220,10 @@ class Interval(Tabular): pass # Make sure we are at the next new line readline_count = VIEWPORT_MAX_READS_PER_LINE - while line.rstrip('\n\r') == line: - assert readline_count > 0, Exception(f'Viewport readline count exceeded for dataset {dataset.id}.') + while line.rstrip("\n\r") == line: + assert readline_count > 0, Exception( + f"Viewport readline count exceeded for dataset {dataset.id}." + ) line = fh.readline(VIEWPORT_READLINE_BUFFER_SIZE) if not line: break # EOF @@ -208,8 +241,14 @@ class Interval(Tabular): def as_ucsc_display_file(self, dataset, **kwd): """Returns file contents with only the bed data""" - with tempfile.NamedTemporaryFile(delete=False, mode='w') as fh: - c, s, e, t, n = dataset.metadata.chromCol, dataset.metadata.startCol, dataset.metadata.endCol, dataset.metadata.strandCol or 0, dataset.metadata.nameCol or 0 + with tempfile.NamedTemporaryFile(delete=False, mode="w") as fh: + c, s, e, t, n = ( + dataset.metadata.chromCol, + dataset.metadata.startCol, + dataset.metadata.endCol, + dataset.metadata.strandCol or 0, + dataset.metadata.nameCol or 0, + ) c, s, e, t, n = int(c) - 1, int(s) - 1, int(e) - 1, int(t) - 1, int(n) - 1 if t >= 0: # strand column (should) exists for i, elems in enumerate(compression_utils.file_iter(dataset.file_name)): @@ -219,24 +258,33 @@ class Interval(Tabular): name = elems[n] if t < len(elems): strand = elems[t] - tmp = [elems[c], elems[s], elems[e], name, '0', strand] - fh.write('%s\n' % '\t'.join(tmp)) + tmp = [elems[c], elems[s], elems[e], name, "0", strand] + fh.write("%s\n" % "\t".join(tmp)) elif n >= 0: # name column (should) exists for i, elems in enumerate(compression_utils.file_iter(dataset.file_name)): name = "region_%i" % i if n >= 0 and n < len(elems): name = elems[n] tmp = [elems[c], elems[s], elems[e], name] - fh.write('%s\n' % '\t'.join(tmp)) + fh.write("%s\n" % "\t".join(tmp)) else: for elems in compression_utils.file_iter(dataset.file_name): tmp = [elems[c], elems[s], elems[e]] - fh.write('%s\n' % '\t'.join(tmp)) - return compression_utils.get_fileobj(fh.name, mode='rb') + fh.write("%s\n" % "\t".join(tmp)) + return compression_utils.get_fileobj(fh.name, mode="rb") def display_peek(self, dataset): """Returns formated html of peek""" - return self.make_html_table(dataset, column_parameter_alias={'chromCol': 'Chrom', 'startCol': 'Start', 'endCol': 'End', 'strandCol': 'Strand', 'nameCol': 'Name'}) + return self.make_html_table( + dataset, + column_parameter_alias={ + "chromCol": "Chrom", + "startCol": "Start", + "endCol": "End", + "strandCol": "Strand", + "nameCol": "Name", + }, + ) def ucsc_links(self, dataset, type, app, base_url): """ @@ -245,9 +293,11 @@ class Interval(Tabular): """ # Filter UCSC sites to only those that are supported by this build and # enabled. - valid_sites = [(name, url) - for name, url in app.datatypes_registry.get_legacy_sites_by_build('ucsc', dataset.dbkey) - if name in app.datatypes_registry.get_display_sites('ucsc')] + valid_sites = [ + (name, url) + for name, url in app.datatypes_registry.get_legacy_sites_by_build("ucsc", dataset.dbkey) + if name in app.datatypes_registry.get_display_sites("ucsc") + ] if not valid_sites: return [] # If there are any valid sites, we need to generate the estimated @@ -258,27 +308,31 @@ class Interval(Tabular): # Accumulate links for valid sites ret_val = [] for site_name, site_url in valid_sites: - internal_url = app.url_for(controller='dataset', dataset_id=dataset.id, - action='display_at', filename='ucsc_' + site_name) - display_url = quote_plus("%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" % - (base_url, app.url_for(controller='root'), dataset.id, type)) - redirect_url = quote_plus("%sdb=%s&position=%s:%s-%s&hgt.customText=%%s" % - (site_url, dataset.dbkey, chrom, start, stop)) - link = f'{internal_url}?redirect_url={redirect_url}&display_url={display_url}' + internal_url = app.url_for( + controller="dataset", dataset_id=dataset.id, action="display_at", filename="ucsc_" + site_name + ) + display_url = quote_plus( + "%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" + % (base_url, app.url_for(controller="root"), dataset.id, type) + ) + redirect_url = quote_plus( + "%sdb=%s&position=%s:%s-%s&hgt.customText=%%s" % (site_url, dataset.dbkey, chrom, start, stop) + ) + link = f"{internal_url}?redirect_url={redirect_url}&display_url={display_url}" ret_val.append((site_name, link)) return ret_val def validate(self, dataset, **kwd): """Validate an interval file using the bx GenomicIntervalReader""" - c, s, e, t = dataset.metadata.chromCol, dataset.metadata.startCol, dataset.metadata.endCol, dataset.metadata.strandCol + c, s, e, t = ( + dataset.metadata.chromCol, + dataset.metadata.startCol, + dataset.metadata.endCol, + dataset.metadata.strandCol, + ) c, s, e, t = int(c) - 1, int(s) - 1, int(e) - 1, int(t) - 1 with compression_utils.get_fileobj(dataset.file_name, "r") as infile: - reader = GenomicIntervalReader( - infile, - chrom_col=c, - start_col=s, - end_col=e, - strand_col=t) + reader = GenomicIntervalReader(infile, chrom_col=c, start_col=s, end_col=e, strand_col=t) while True: try: @@ -309,7 +363,7 @@ class Interval(Tabular): """ found_valid_lines = False try: - headers = iter_headers(file_prefix, '\t', comment_designator='#') + headers = iter_headers(file_prefix, "\t", comment_designator="#") # If we got here, we already know the file is_column_based and is not bed, # so we'll just look for some valid data. for hdr in headers: @@ -329,31 +383,32 @@ class Interval(Tabular): return None # ------------- Dataproviders - @dataproviders.decorators.dataprovider_factory('genomic-region', - dataproviders.dataset.GenomicRegionDataProvider.settings) + @dataproviders.decorators.dataprovider_factory( + "genomic-region", dataproviders.dataset.GenomicRegionDataProvider.settings + ) def genomic_region_dataprovider(self, dataset, **settings): return dataproviders.dataset.GenomicRegionDataProvider(dataset, **settings) - @dataproviders.decorators.dataprovider_factory('genomic-region-dict', - dataproviders.dataset.GenomicRegionDataProvider.settings) + @dataproviders.decorators.dataprovider_factory( + "genomic-region-dict", dataproviders.dataset.GenomicRegionDataProvider.settings + ) def genomic_region_dict_dataprovider(self, dataset, **settings): - settings['named_columns'] = True + settings["named_columns"] = True return self.genomic_region_dataprovider(dataset, **settings) - @dataproviders.decorators.dataprovider_factory('interval', - dataproviders.dataset.IntervalDataProvider.settings) + @dataproviders.decorators.dataprovider_factory("interval", dataproviders.dataset.IntervalDataProvider.settings) def interval_dataprovider(self, dataset, **settings): return dataproviders.dataset.IntervalDataProvider(dataset, **settings) - @dataproviders.decorators.dataprovider_factory('interval-dict', - dataproviders.dataset.IntervalDataProvider.settings) + @dataproviders.decorators.dataprovider_factory("interval-dict", dataproviders.dataset.IntervalDataProvider.settings) def interval_dict_dataprovider(self, dataset, **settings): - settings['named_columns'] = True + settings["named_columns"] = True return self.interval_dataprovider(dataset, **settings) class BedGraph(Interval): """Tab delimited chrom/start/end/datavalue dataset""" + edam_format = "format_3583" file_ext = "bedgraph" track_type = "LineTrack" @@ -361,33 +416,62 @@ class BedGraph(Interval): def as_ucsc_display_file(self, dataset, **kwd): """ - Returns file contents as is with no modifications. - TODO: this is a functional stub and will need to be enhanced moving forward to provide additional support for bedgraph. + Returns file contents as is with no modifications. + TODO: this is a functional stub and will need to be enhanced moving forward to provide additional support for bedgraph. """ - return open(dataset.file_name, 'rb') + return open(dataset.file_name, "rb") def get_estimated_display_viewport(self, dataset, chrom_col=0, start_col=1, end_col=2): """ - Set viewport based on dataset's first 100 lines. + Set viewport based on dataset's first 100 lines. """ - return Interval.get_estimated_display_viewport(self, dataset, chrom_col=chrom_col, start_col=start_col, end_col=end_col) + return Interval.get_estimated_display_viewport( + self, dataset, chrom_col=chrom_col, start_col=start_col, end_col=end_col + ) class Bed(Interval): """Tab delimited data in BED format""" + edam_format = "format_3003" file_ext = "bed" data_sources = {"data": "tabix", "index": "bigwig", "feature_search": "fli"} track_type = Interval.track_type - column_names = ['Chrom', 'Start', 'End', 'Name', 'Score', 'Strand', 'ThickStart', 'ThickEnd', 'ItemRGB', 'BlockCount', 'BlockSizes', 'BlockStarts'] + column_names = [ + "Chrom", + "Start", + "End", + "Name", + "Score", + "Strand", + "ThickStart", + "ThickEnd", + "ItemRGB", + "BlockCount", + "BlockSizes", + "BlockStarts", + ] MetadataElement(name="chromCol", default=1, desc="Chrom column", param=metadata.ColumnParameter) MetadataElement(name="startCol", default=2, desc="Start column", param=metadata.ColumnParameter) MetadataElement(name="endCol", default=3, desc="End column", param=metadata.ColumnParameter) - MetadataElement(name="strandCol", desc="Strand column (click box & select)", param=metadata.ColumnParameter, optional=True, no_value=0) + MetadataElement( + name="strandCol", + desc="Strand column (click box & select)", + param=metadata.ColumnParameter, + optional=True, + no_value=0, + ) MetadataElement(name="columns", default=3, desc="Number of columns", readonly=True, visible=False) - MetadataElement(name="viz_filter_cols", desc="Score column for visualization", default=[4], param=metadata.ColumnParameter, optional=True, multiple=True) + MetadataElement( + name="viz_filter_cols", + desc="Score column for visualization", + default=[4], + param=metadata.ColumnParameter, + optional=True, + multiple=True, + ) # do we need to repeat these? they are the same as should be inherited from interval type def set_meta(self, dataset, overwrite=True, **kwd): @@ -395,18 +479,18 @@ class Bed(Interval): if dataset.has_data(): i = 0 for i, line in enumerate(open(dataset.file_name)): # noqa: B007 - line = line.rstrip('\r\n') - if line and not line.startswith('#'): - elems = line.split('\t') + line = line.rstrip("\r\n") + if line and not line.startswith("#"): + elems = line.split("\t") if len(elems) > 2: if len(elems) > 3: - if overwrite or not dataset.metadata.element_is_set('nameCol'): + if overwrite or not dataset.metadata.element_is_set("nameCol"): dataset.metadata.nameCol = 4 if len(elems) < 6: - if overwrite or not dataset.metadata.element_is_set('strandCol'): + if overwrite or not dataset.metadata.element_is_set("strandCol"): dataset.metadata.strandCol = 0 else: - if overwrite or not dataset.metadata.element_is_set('strandCol'): + if overwrite or not dataset.metadata.element_is_set("strandCol"): dataset.metadata.strandCol = 6 break Tabular.set_meta(self, dataset, overwrite=overwrite, skip=i) @@ -417,7 +501,7 @@ class Bed(Interval): line = line.strip() if line == "" or line.startswith("#"): continue - fields = line.split('\t') + fields = line.split("\t") """check to see if this file doesn't conform to strict genome browser accepted bed""" try: if len(fields) > 12: @@ -432,11 +516,15 @@ class Bed(Interval): if len(fields) > 9: int(fields[9]) if len(fields) > 10: - fields2 = fields[10].rstrip(",").split(",") # remove trailing comma and split on comma + fields2 = ( + fields[10].rstrip(",").split(",") + ) # remove trailing comma and split on comma for field in fields2: int(field) if len(fields) > 11: - fields2 = fields[11].rstrip(",").split(",") # remove trailing comma and split on comma + fields2 = ( + fields[11].rstrip(",").split(",") + ) # remove trailing comma and split on comma for field in fields2: int(field) except Exception: @@ -445,7 +533,7 @@ class Bed(Interval): break try: - return open(dataset.file_name, 'rb') + return open(dataset.file_name, "rb") except Exception: return "This item contains no content" @@ -473,12 +561,12 @@ class Bed(Interval): >>> Bed().sniff( fname ) True """ - if not get_headers(file_prefix, '\t', comment_designator='#', count=1): + if not get_headers(file_prefix, "\t", comment_designator="#", count=1): return False try: found_valid_lines = False - for hdr in iter_headers(file_prefix, '\t', comment_designator='#'): - if not hdr or hdr == ['']: + for hdr in iter_headers(file_prefix, "\t", comment_designator="#"): + if not hdr or hdr == [""]: continue if len(hdr) < 3 or len(hdr) > 12: return False @@ -517,7 +605,7 @@ class Bed(Interval): int(hdr[8]) except Exception: try: - hdr[8].split(',') + hdr[8].split(",") except Exception: return False if len(hdr) > 9: @@ -530,13 +618,13 @@ class Bed(Interval): # hdr[10] is blockSizes - A comma-separated list of the block sizes. # Sometimes the blosck_sizes and block_starts lists end in extra commas try: - block_sizes = hdr[10].rstrip(',').split(',') + block_sizes = hdr[10].rstrip(",").split(",") except Exception: return False if len(hdr) > 11: # hdr[11] is blockStarts - A comma-separated list of block starts. try: - block_starts = hdr[11].rstrip(',').split(',') + block_starts = hdr[11].rstrip(",").split(",") except Exception: return False if len(block_sizes) != block_count or len(block_starts) != block_count: @@ -549,13 +637,41 @@ class Bed(Interval): class ProBed(Bed): """Tab delimited data in proBED format - adaptation of BED for proteomics data.""" + edam_format = "format_3827" file_ext = "probed" - column_names = ['Chrom', 'Start', 'End', 'Name', 'Score', 'Strand', 'ThickStart', 'ThickEnd', 'ItemRGB', 'BlockCount', 'BlockSizes', 'BlockStarts', 'ProteinAccession', 'PeptideSequence', 'Uniqueness', 'GenomeReferenceVersion', 'PsmScore', 'Fdr', 'Modifications', 'Charge', 'ExpMassToCharge', 'CalcMassToCharge', 'PsmRank', 'DatasetID', 'Uri'] + column_names = [ + "Chrom", + "Start", + "End", + "Name", + "Score", + "Strand", + "ThickStart", + "ThickEnd", + "ItemRGB", + "BlockCount", + "BlockSizes", + "BlockStarts", + "ProteinAccession", + "PeptideSequence", + "Uniqueness", + "GenomeReferenceVersion", + "PsmScore", + "Fdr", + "Modifications", + "Charge", + "ExpMassToCharge", + "CalcMassToCharge", + "PsmRank", + "DatasetID", + "Uri", + ] class BedStrict(Bed): """Tab delimited data in strict BED format - no non-standard columns allowed""" + edam_format = "format_3584" file_ext = "bedstrict" @@ -564,10 +680,26 @@ class BedStrict(Bed): # Read only metadata elements MetadataElement(name="chromCol", default=1, desc="Chrom column", readonly=True, param=metadata.MetadataParameter) - MetadataElement(name="startCol", default=2, desc="Start column", readonly=True, param=metadata.MetadataParameter) # TODO: start and end should be able to be set to these or the proper thick[start/end]? + MetadataElement( + name="startCol", default=2, desc="Start column", readonly=True, param=metadata.MetadataParameter + ) # TODO: start and end should be able to be set to these or the proper thick[start/end]? MetadataElement(name="endCol", default=3, desc="End column", readonly=True, param=metadata.MetadataParameter) - MetadataElement(name="strandCol", desc="Strand column (click box & select)", readonly=True, param=metadata.MetadataParameter, no_value=0, optional=True) - MetadataElement(name="nameCol", desc="Name/Identifier column (click box & select)", readonly=True, param=metadata.MetadataParameter, no_value=0, optional=True) + MetadataElement( + name="strandCol", + desc="Strand column (click box & select)", + readonly=True, + param=metadata.MetadataParameter, + no_value=0, + optional=True, + ) + MetadataElement( + name="nameCol", + desc="Name/Identifier column (click box & select)", + readonly=True, + param=metadata.MetadataParameter, + no_value=0, + optional=True, + ) MetadataElement(name="columns", default=3, desc="Number of columns", readonly=True, visible=False) def __init__(self, **kwd): @@ -587,12 +719,14 @@ class BedStrict(Bed): class Bed6(BedStrict): """Tab delimited data in strict BED format - no non-standard columns allowed; column count forced to 6""" + edam_format = "format_3585" file_ext = "bed6" class Bed12(BedStrict): """Tab delimited data in strict BED format - no non-standard columns allowed; column count forced to 12""" + edam_format = "format_3586" file_ext = "bed12" @@ -606,9 +740,11 @@ class _RemoteCallMixin: """ internal_url = f"{app.url_for(controller='dataset', dataset_id=dataset.id, action='display_at', filename=f'{type}_{site_name}')}" base_url = app.config.get("display_at_callback", base_url) - display_url = quote_plus("%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" % - (base_url, app.url_for(controller='root'), dataset.id, type)) - link = f'{internal_url}?redirect_url={redirect_url}&display_url={display_url}' + display_url = quote_plus( + "%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" + % (base_url, app.url_for(controller="root"), dataset.id, type) + ) + link = f"{internal_url}?redirect_url={redirect_url}&display_url={display_url}" return link @@ -616,26 +752,41 @@ class _RemoteCallMixin: @build_sniff_from_prefix class Gff(Tabular, _RemoteCallMixin): """Tab delimited data in Gff format""" + edam_data = "data_1255" edam_format = "format_2305" file_ext = "gff" - valid_gff_frame = ['.', '0', '1', '2'] - column_names = ['Seqname', 'Source', 'Feature', 'Start', 'End', 'Score', 'Strand', 'Frame', 'Group'] + valid_gff_frame = [".", "0", "1", "2"] + column_names = ["Seqname", "Source", "Feature", "Start", "End", "Score", "Strand", "Frame", "Group"] data_sources = {"data": "interval_index", "index": "bigwig", "feature_search": "fli"} track_type = Interval.track_type MetadataElement(name="columns", default=9, desc="Number of columns", readonly=True, visible=False) - MetadataElement(name="column_types", default=['str', 'str', 'str', 'int', 'int', 'int', 'str', 'str', 'str'], - param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False) + MetadataElement( + name="column_types", + default=["str", "str", "str", "int", "int", "int", "str", "str", "str"], + param=metadata.ColumnTypesParameter, + desc="Column types", + readonly=True, + visible=False, + ) MetadataElement(name="attributes", default=0, desc="Number of attributes", readonly=True, visible=False, no_value=0) - MetadataElement(name="attribute_types", default={}, desc="Attribute types", param=metadata.DictParameter, readonly=True, visible=False, no_value=[]) + MetadataElement( + name="attribute_types", + default={}, + desc="Attribute types", + param=metadata.DictParameter, + readonly=True, + visible=False, + no_value=[], + ) def __init__(self, **kwd): """Initialize datatype, by adding GBrowse display app""" Tabular.__init__(self, **kwd) - self.add_display_app('ucsc', 'display at UCSC', 'as_ucsc_display_file', 'ucsc_links') - self.add_display_app('gbrowse', 'display in Gbrowse', 'as_gbrowse_display_file', 'gbrowse_links') + self.add_display_app("ucsc", "display at UCSC", "as_ucsc_display_file", "ucsc_links") + self.add_display_app("gbrowse", "display in Gbrowse", "as_gbrowse_display_file", "gbrowse_links") def set_attribute_metadata(self, dataset): """ @@ -648,8 +799,8 @@ class Gff(Tabular, _RemoteCallMixin): attribute_types = {} with compression_utils.get_fileobj(dataset.file_name) as in_fh: for i, line in enumerate(in_fh): - if line and not line.startswith('#'): - elems = line.split('\t') + if line and not line.startswith("#"): + elems = line.split("\t") if len(elems) == 9: try: # Loop through attributes to set types. @@ -683,9 +834,9 @@ class Gff(Tabular, _RemoteCallMixin): i = 0 with compression_utils.get_fileobj(dataset.file_name) as in_fh: for i, line in enumerate(in_fh): # noqa: B007 - line = line.rstrip('\r\n') - if line and not line.startswith('#'): - elems = line.split('\t') + line = line.rstrip("\r\n") + if line and not line.startswith("#"): + elems = line.split("\t") if len(elems) == 9: try: int(elems[3]) @@ -714,8 +865,8 @@ class Gff(Tabular, _RemoteCallMixin): with compression_utils.get_fileobj(dataset.file_name) as fh: for line in util.iter_start_of_line(fh, VIEWPORT_READLINE_BUFFER_SIZE): try: - if line.startswith('##sequence-region'): # ##sequence-region IV 6000000 6030000 - elems = line.rstrip('\n\r').split() + if line.startswith("##sequence-region"): # ##sequence-region IV 6000000 6030000 + elems = line.rstrip("\n\r").split() if len(elems) > 3: # line looks like: # sequence-region ctg123 1 1497228 @@ -723,13 +874,13 @@ class Gff(Tabular, _RemoteCallMixin): start = int(elems[2]) # 6000000 stop = int(elems[3]) # 6030000 break # use location declared in file - elif len(elems) == 2 and elems[1].find('..') > 0: + elif len(elems) == 2 and elems[1].find("..") > 0: # line looks like this: # sequence-region X:120000..140000 - elems = elems[1].split(':') + elems = elems[1].split(":") seqid = elems[0] - start = int(elems[1].split('..')[0]) - stop = int(elems[1].split('..')[1]) + start = int(elems[1].split("..")[0]) + stop = int(elems[1].split("..")[1]) break # use location declared in file else: log.debug(f"line ({str(line)}) uses an unsupported ##sequence-region definition.") @@ -740,9 +891,9 @@ class Gff(Tabular, _RemoteCallMixin): seqid, startend = pos_info.split(":") start, stop = map(int, startend.split("-")) break # use location declared in file - elif not line.startswith(('#', 'track', 'browser')): + elif not line.startswith(("#", "track", "browser")): viewport_feature_count -= 1 - elems = line.rstrip('\n\r').split('\t') + elems = line.rstrip("\n\r").split("\t") if len(elems) > 3: if not seqid: # We can only set the viewport for a single chromosome @@ -756,8 +907,10 @@ class Gff(Tabular, _RemoteCallMixin): pass # make sure we are at the next new line readline_count = VIEWPORT_MAX_READS_PER_LINE - while line.rstrip('\n\r') == line: - assert readline_count > 0, Exception(f'Viewport readline count exceeded for dataset {dataset.id}.') + while line.rstrip("\n\r") == line: + assert readline_count > 0, Exception( + f"Viewport readline count exceeded for dataset {dataset.id}." + ) line = fh.readline(VIEWPORT_READLINE_BUFFER_SIZE) if not line: break # EOF @@ -769,18 +922,18 @@ class Gff(Tabular, _RemoteCallMixin): if seqid is not None: return (seqid, str(start), str(stop)) # Necessary to return strings? except Exception: - log.exception('Unexpected error') + log.exception("Unexpected error") return (None, None, None) # could not determine viewport def ucsc_links(self, dataset, type, app, base_url): ret_val = [] seqid, start, stop = self.get_estimated_display_viewport(dataset) if seqid is not None: - for site_name, site_url in app.datatypes_registry.get_legacy_sites_by_build('ucsc', dataset.dbkey): - if site_name in app.datatypes_registry.get_display_sites('ucsc'): + for site_name, site_url in app.datatypes_registry.get_legacy_sites_by_build("ucsc", dataset.dbkey): + if site_name in app.datatypes_registry.get_display_sites("ucsc"): redirect_url = quote_plus( - "%sdb=%s&position=%s:%s-%s&hgt.customText=%%s" % - (site_url, dataset.dbkey, seqid, start, stop)) + "%sdb=%s&position=%s:%s-%s&hgt.customText=%%s" % (site_url, dataset.dbkey, seqid, start, stop) + ) link = self._get_remote_call_url(redirect_url, site_name, dataset, type, app, base_url) ret_val.append((site_name, link)) return ret_val @@ -789,9 +942,9 @@ class Gff(Tabular, _RemoteCallMixin): ret_val = [] seqid, start, stop = self.get_estimated_display_viewport(dataset) if seqid is not None: - for site_name, site_url in app.datatypes_registry.get_legacy_sites_by_build('gbrowse', dataset.dbkey): - if site_name in app.datatypes_registry.get_display_sites('gbrowse'): - if seqid.startswith('chr') and len(seqid) > 3: + for site_name, site_url in app.datatypes_registry.get_legacy_sites_by_build("gbrowse", dataset.dbkey): + if site_name in app.datatypes_registry.get_display_sites("gbrowse"): + if seqid.startswith("chr") and len(seqid) > 3: seqid = seqid[3:] redirect_url = quote_plus(f"{site_url}/?q={seqid}:{start}..{stop}&eurl=%s") link = self._get_remote_call_url(redirect_url, site_name, dataset, type, app, base_url) @@ -814,18 +967,18 @@ class Gff(Tabular, _RemoteCallMixin): >>> Gff().sniff( fname ) True """ - if len(get_headers(file_prefix, '\t', count=2)) < 2: + if len(get_headers(file_prefix, "\t", count=2)) < 2: return False try: found_valid_lines = False - for hdr in iter_headers(file_prefix, '\t'): - if not hdr or hdr == ['']: + for hdr in iter_headers(file_prefix, "\t"): + if not hdr or hdr == [""]: continue hdr0_parts = hdr[0].split() - if hdr0_parts[0] == '##gff-version': - return hdr0_parts[1].startswith('2') + if hdr0_parts[0] == "##gff-version": + return hdr0_parts[1].startswith("2") # The gff-version header comment may have been stripped, so inspect the data - if hdr[0].startswith('#'): + if hdr[0].startswith("#"): continue if len(hdr) != 9: return False @@ -834,7 +987,7 @@ class Gff(Tabular, _RemoteCallMixin): int(hdr[4]) except Exception: return False - if hdr[5] != '.': + if hdr[5] != ".": try: float(hdr[5]) except Exception: @@ -850,40 +1003,47 @@ class Gff(Tabular, _RemoteCallMixin): # ------------- Dataproviders # redefine bc super is Tabular - @dataproviders.decorators.dataprovider_factory('genomic-region', - dataproviders.dataset.GenomicRegionDataProvider.settings) + @dataproviders.decorators.dataprovider_factory( + "genomic-region", dataproviders.dataset.GenomicRegionDataProvider.settings + ) def genomic_region_dataprovider(self, dataset, **settings): return dataproviders.dataset.GenomicRegionDataProvider(dataset, 0, 3, 4, **settings) - @dataproviders.decorators.dataprovider_factory('genomic-region-dict', - dataproviders.dataset.GenomicRegionDataProvider.settings) + @dataproviders.decorators.dataprovider_factory( + "genomic-region-dict", dataproviders.dataset.GenomicRegionDataProvider.settings + ) def genomic_region_dict_dataprovider(self, dataset, **settings): - settings['named_columns'] = True + settings["named_columns"] = True return self.genomic_region_dataprovider(dataset, **settings) - @dataproviders.decorators.dataprovider_factory('interval', - dataproviders.dataset.IntervalDataProvider.settings) + @dataproviders.decorators.dataprovider_factory("interval", dataproviders.dataset.IntervalDataProvider.settings) def interval_dataprovider(self, dataset, **settings): return dataproviders.dataset.IntervalDataProvider(dataset, 0, 3, 4, 6, 2, **settings) - @dataproviders.decorators.dataprovider_factory('interval-dict', - dataproviders.dataset.IntervalDataProvider.settings) + @dataproviders.decorators.dataprovider_factory("interval-dict", dataproviders.dataset.IntervalDataProvider.settings) def interval_dict_dataprovider(self, dataset, **settings): - settings['named_columns'] = True + settings["named_columns"] = True return self.interval_dataprovider(dataset, **settings) class Gff3(Gff): """Tab delimited data in Gff3 format""" + edam_format = "format_1975" file_ext = "gff3" - valid_gff3_strand = ['+', '-', '.', '?'] + valid_gff3_strand = ["+", "-", ".", "?"] valid_gff3_phase = Gff.valid_gff_frame - column_names = ['Seqid', 'Source', 'Type', 'Start', 'End', 'Score', 'Strand', 'Phase', 'Attributes'] + column_names = ["Seqid", "Source", "Type", "Start", "End", "Score", "Strand", "Phase", "Attributes"] track_type = Interval.track_type - MetadataElement(name="column_types", default=['str', 'str', 'str', 'int', 'int', 'float', 'str', 'int', 'list'], - param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False) + MetadataElement( + name="column_types", + default=["str", "str", "str", "int", "int", "float", "str", "int", "list"], + param=metadata.ColumnTypesParameter, + desc="Column types", + readonly=True, + visible=False, + ) def __init__(self, **kwd): """Initialize datatype, by adding GBrowse display app""" @@ -894,9 +1054,9 @@ class Gff3(Gff): i = 0 with compression_utils.get_fileobj(dataset.file_name) as in_fh: for i, line in enumerate(in_fh): # noqa: B007 - line = line.rstrip('\r\n') - if line and not line.startswith('#'): - elems = line.split('\t') + line = line.rstrip("\r\n") + if line and not line.startswith("#"): + elems = line.split("\t") valid_start = False valid_end = False if len(elems) == 9: @@ -904,17 +1064,23 @@ class Gff3(Gff): start = int(elems[3]) valid_start = True except Exception: - if elems[3] == '.': + if elems[3] == ".": valid_start = True try: end = int(elems[4]) valid_end = True except Exception: - if elems[4] == '.': + if elems[4] == ".": valid_end = True strand = elems[6] phase = elems[7] - if valid_start and valid_end and start < end and strand in self.valid_gff3_strand and phase in self.valid_gff3_phase: + if ( + valid_start + and valid_end + and start < end + and strand in self.valid_gff3_strand + and phase in self.valid_gff3_phase + ): break Tabular.set_meta(self, dataset, overwrite=overwrite, skip=i) @@ -954,32 +1120,32 @@ class Gff3(Gff): >>> Gff3().sniff( fname ) False """ - if len(get_headers(file_prefix, '\t', count=2)) < 2: + if len(get_headers(file_prefix, "\t", count=2)) < 2: return False try: found_valid_lines = False - for hdr in iter_headers(file_prefix, '\t'): - if not hdr or hdr == ['']: + for hdr in iter_headers(file_prefix, "\t"): + if not hdr or hdr == [""]: continue hdr0_parts = hdr[0].split() - if hdr0_parts[0] == '##gff-version': - return hdr0_parts[1].startswith('3') + if hdr0_parts[0] == "##gff-version": + return hdr0_parts[1].startswith("3") # The gff-version header comment may have been stripped, so inspect the data - if hdr[0].startswith('#'): + if hdr[0].startswith("#"): continue if len(hdr) != 9: return False try: int(hdr[3]) except Exception: - if hdr[3] != '.': + if hdr[3] != ".": return False try: int(hdr[4]) except Exception: - if hdr[4] != '.': + if hdr[4] != ".": return False - if hdr[5] != '.': + if hdr[5] != ".": try: float(hdr[5]) except Exception: @@ -997,14 +1163,21 @@ class Gff3(Gff): class Gtf(Gff): """Tab delimited data in Gtf format""" + edam_format = "format_2306" file_ext = "gtf" - column_names = ['Seqname', 'Source', 'Feature', 'Start', 'End', 'Score', 'Strand', 'Frame', 'Attributes'] + column_names = ["Seqname", "Source", "Feature", "Start", "End", "Score", "Strand", "Frame", "Attributes"] track_type = Interval.track_type MetadataElement(name="columns", default=9, desc="Number of columns", readonly=True, visible=False) - MetadataElement(name="column_types", default=['str', 'str', 'str', 'int', 'int', 'float', 'str', 'int', 'list'], - param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False) + MetadataElement( + name="column_types", + default=["str", "str", "str", "int", "int", "float", "str", "int", "list"], + param=metadata.ColumnTypesParameter, + desc="Column types", + readonly=True, + visible=False, + ) def sniff_prefix(self, file_prefix: FilePrefix): """ @@ -1034,18 +1207,18 @@ class Gtf(Gff): >>> Gtf().sniff( fname ) True """ - if len(get_headers(file_prefix, '\t', count=2)) < 2: + if len(get_headers(file_prefix, "\t", count=2)) < 2: return False try: found_valid_lines = False - for hdr in iter_headers(file_prefix, '\t'): - if not hdr or hdr == ['']: + for hdr in iter_headers(file_prefix, "\t"): + if not hdr or hdr == [""]: continue hdr0_parts = hdr[0].split() - if hdr0_parts[0] == '##gff-version' and not hdr0_parts[1].startswith('2'): + if hdr0_parts[0] == "##gff-version" and not hdr0_parts[1].startswith("2"): return False # The gff-version header comment may have been stripped, so inspect the data - if hdr[0].startswith('#'): + if hdr[0].startswith("#"): continue if len(hdr) != 9: return False @@ -1054,7 +1227,7 @@ class Gtf(Gff): int(hdr[4]) except Exception: return False - if hdr[5] != '.': + if hdr[5] != ".": try: float(hdr[5]) except Exception: @@ -1066,7 +1239,7 @@ class Gtf(Gff): # Check attributes for gene_id (transcript_id is also mandatory # but not for genes) attributes = parse_gff_attributes(hdr[8]) - if 'gene_id' not in attributes: + if "gene_id" not in attributes: return False found_valid_lines = True return found_valid_lines @@ -1078,6 +1251,7 @@ class Gtf(Gff): @build_sniff_from_prefix class Wiggle(Tabular, _RemoteCallMixin): """Tab delimited data in wiggle format""" + edam_format = "format_3005" file_ext = "wig" track_type = "LineTrack" @@ -1087,8 +1261,8 @@ class Wiggle(Tabular, _RemoteCallMixin): def __init__(self, **kwd): Tabular.__init__(self, **kwd) - self.add_display_app('ucsc', 'display at UCSC', 'as_ucsc_display_file', 'ucsc_links') - self.add_display_app('gbrowse', 'display in Gbrowse', 'as_gbrowse_display_file', 'gbrowse_links') + self.add_display_app("ucsc", "display at UCSC", "as_ucsc_display_file", "ucsc_links") + self.add_display_app("gbrowse", "display in Gbrowse", "as_gbrowse_display_file", "gbrowse_links") def get_estimated_display_viewport(self, dataset): """Return a chrom, start, stop tuple for viewing a file.""" @@ -1105,22 +1279,24 @@ class Wiggle(Tabular, _RemoteCallMixin): for line in util.iter_start_of_line(fh, VIEWPORT_READLINE_BUFFER_SIZE): try: if line.startswith("browser"): - chr_info = line.rstrip('\n\r').split()[-1] + chr_info = line.rstrip("\n\r").split()[-1] chrom, coords = chr_info.split(":") start, end = map(int, coords.split("-")) break # use the browser line # variableStep chrom=chr20 - if line and (line.lower().startswith("variablestep") or line.lower().startswith("fixedstep")): + if line and ( + line.lower().startswith("variablestep") or line.lower().startswith("fixedstep") + ): if chrom is not None: break # different chrom or different section of the chrom - chrom = line.rstrip('\n\r').split("chrom=")[1].split()[0] - if 'span=' in line: - span = int(line.rstrip('\n\r').split("span=")[1].split()[0]) - if 'step=' in line: - step = int(line.rstrip('\n\r').split("step=")[1].split()[0]) - start = int(line.rstrip('\n\r').split("start=")[1].split()[0]) + chrom = line.rstrip("\n\r").split("chrom=")[1].split()[0] + if "span=" in line: + span = int(line.rstrip("\n\r").split("span=")[1].split()[0]) + if "step=" in line: + step = int(line.rstrip("\n\r").split("step=")[1].split()[0]) + start = int(line.rstrip("\n\r").split("start=")[1].split()[0]) else: - fields = line.rstrip('\n\r').split() + fields = line.rstrip("\n\r").split() if fields: if step is not None: if not end: @@ -1135,8 +1311,10 @@ class Wiggle(Tabular, _RemoteCallMixin): pass # make sure we are at the next new line readline_count = VIEWPORT_MAX_READS_PER_LINE - while line.rstrip('\n\r') == line: - assert readline_count > 0, Exception(f'Viewport readline count exceeded for dataset {dataset.id}.') + while line.rstrip("\n\r") == line: + assert readline_count > 0, Exception( + f"Viewport readline count exceeded for dataset {dataset.id}." + ) line = fh.readline(VIEWPORT_READLINE_BUFFER_SIZE) if not line: break # EOF @@ -1148,16 +1326,16 @@ class Wiggle(Tabular, _RemoteCallMixin): if chrom is not None: return (chrom, str(start), str(end)) # Necessary to return strings? except Exception: - log.exception('Unexpected error') + log.exception("Unexpected error") return (None, None, None) # could not determine viewport def gbrowse_links(self, dataset, type, app, base_url): ret_val = [] chrom, start, stop = self.get_estimated_display_viewport(dataset) if chrom is not None: - for site_name, site_url in app.datatypes_registry.get_legacy_sites_by_build('gbrowse', dataset.dbkey): - if site_name in app.datatypes_registry.get_display_sites('gbrowse'): - if chrom.startswith('chr') and len(chrom) > 3: + for site_name, site_url in app.datatypes_registry.get_legacy_sites_by_build("gbrowse", dataset.dbkey): + if site_name in app.datatypes_registry.get_display_sites("gbrowse"): + if chrom.startswith("chr") and len(chrom) > 3: chrom = chrom[3:] redirect_url = quote_plus(f"{site_url}/?q={chrom}:{start}..{stop}&eurl=%s") link = self._get_remote_call_url(redirect_url, site_name, dataset, type, app, base_url) @@ -1168,24 +1346,26 @@ class Wiggle(Tabular, _RemoteCallMixin): ret_val = [] chrom, start, stop = self.get_estimated_display_viewport(dataset) if chrom is not None: - for site_name, site_url in app.datatypes_registry.get_legacy_sites_by_build('ucsc', dataset.dbkey): - if site_name in app.datatypes_registry.get_display_sites('ucsc'): - redirect_url = quote_plus(f"{site_url}db={dataset.dbkey}&position={chrom}:{start}-{stop}&hgt.customText=%s") + for site_name, site_url in app.datatypes_registry.get_legacy_sites_by_build("ucsc", dataset.dbkey): + if site_name in app.datatypes_registry.get_display_sites("ucsc"): + redirect_url = quote_plus( + f"{site_url}db={dataset.dbkey}&position={chrom}:{start}-{stop}&hgt.customText=%s" + ) link = self._get_remote_call_url(redirect_url, site_name, dataset, type, app, base_url) ret_val.append((site_name, link)) return ret_val def display_peek(self, dataset): """Returns formated html of peek""" - return self.make_html_table(dataset, skipchars=['track', '#']) + return self.make_html_table(dataset, skipchars=["track", "#"]) def set_meta(self, dataset, overwrite=True, **kwd): max_data_lines = None i = 0 for i, line in enumerate(open(dataset.file_name)): # noqa: B007 - line = line.rstrip('\r\n') - if line and not line.startswith('#'): - elems = line.split('\t') + line = line.rstrip("\r\n") + if line and not line.startswith("#"): + elems = line.split("\t") try: # variableStep format is nucleotide position\tvalue\n, # fixedStep is value\n @@ -1229,7 +1409,7 @@ class Wiggle(Tabular, _RemoteCallMixin): try: headers = iter_headers(file_prefix, None) for hdr in headers: - if len(hdr) > 1 and hdr[0] == 'track' and hdr[1].startswith('type=wiggle'): + if len(hdr) > 1 and hdr[0] == "track" and hdr[1].startswith("type=wiggle"): return True return False except Exception: @@ -1245,35 +1425,36 @@ class Wiggle(Tabular, _RemoteCallMixin): return resolution # ------------- Dataproviders - @dataproviders.decorators.dataprovider_factory('wiggle', dataproviders.dataset.WiggleDataProvider.settings) + @dataproviders.decorators.dataprovider_factory("wiggle", dataproviders.dataset.WiggleDataProvider.settings) def wiggle_dataprovider(self, dataset, **settings): dataset_source = dataproviders.dataset.DatasetDataProvider(dataset) return dataproviders.dataset.WiggleDataProvider(dataset_source, **settings) - @dataproviders.decorators.dataprovider_factory('wiggle-dict', dataproviders.dataset.WiggleDataProvider.settings) + @dataproviders.decorators.dataprovider_factory("wiggle-dict", dataproviders.dataset.WiggleDataProvider.settings) def wiggle_dict_dataprovider(self, dataset, **settings): dataset_source = dataproviders.dataset.DatasetDataProvider(dataset) - settings['named_columns'] = True + settings["named_columns"] = True return dataproviders.dataset.WiggleDataProvider(dataset_source, **settings) @build_sniff_from_prefix class CustomTrack(Tabular): """UCSC CustomTrack""" + edam_format = "format_3588" file_ext = "customtrack" def __init__(self, **kwd): """Initialize interval datatype, by adding UCSC display app""" Tabular.__init__(self, **kwd) - self.add_display_app('ucsc', 'display at UCSC', 'as_ucsc_display_file', 'ucsc_links') + self.add_display_app("ucsc", "display at UCSC", "as_ucsc_display_file", "ucsc_links") def set_meta(self, dataset, overwrite=True, **kwd): Tabular.set_meta(self, dataset, overwrite=overwrite, skip=1) def display_peek(self, dataset): """Returns formated html of peek""" - return self.make_html_table(dataset, skipchars=['track', '#']) + return self.make_html_table(dataset, skipchars=["track", "#"]) def get_estimated_display_viewport(self, dataset, chrom_col=None, start_col=None, end_col=None): """Return a chrom, start, stop tuple for viewing a file.""" @@ -1287,24 +1468,26 @@ class CustomTrack(Tabular): try: with open(dataset.file_name) as fh: for line in util.iter_start_of_line(fh, VIEWPORT_READLINE_BUFFER_SIZE): - if not line.startswith('#'): + if not line.startswith("#"): try: if variable_step_wig: fields = line.rstrip().split() if len(fields) == 2: start = int(fields[0]) return (chrom, str(start), str(start + span)) - elif line and (line.lower().startswith("variablestep") or line.lower().startswith("fixedstep")): - chrom = line.rstrip('\n\r').split("chrom=")[1].split()[0] - if 'span=' in line: - span = int(line.rstrip('\n\r').split("span=")[1].split()[0]) - if 'start=' in line: - start = int(line.rstrip('\n\r').split("start=")[1].split()[0]) + elif line and ( + line.lower().startswith("variablestep") or line.lower().startswith("fixedstep") + ): + chrom = line.rstrip("\n\r").split("chrom=")[1].split()[0] + if "span=" in line: + span = int(line.rstrip("\n\r").split("span=")[1].split()[0]) + if "start=" in line: + start = int(line.rstrip("\n\r").split("start=")[1].split()[0]) return (chrom, str(start), str(start + span)) else: variable_step_wig = True else: - fields = line.rstrip().split('\t') + fields = line.rstrip().split("\t") if len(fields) >= 3: chrom = fields[0] start = int(fields[1]) @@ -1315,8 +1498,10 @@ class CustomTrack(Tabular): continue # make sure we are at the next new line readline_count = VIEWPORT_MAX_READS_PER_LINE - while line.rstrip('\n\r') == line: - assert readline_count > 0, Exception(f'Viewport readline count exceeded for dataset {dataset.id}.') + while line.rstrip("\n\r") == line: + assert readline_count > 0, Exception( + f"Viewport readline count exceeded for dataset {dataset.id}." + ) line = fh.readline(VIEWPORT_READLINE_BUFFER_SIZE) if not line: break # EOF @@ -1326,19 +1511,24 @@ class CustomTrack(Tabular): # exceeded viewport or total line count to check break except Exception: - log.exception('Unexpected error') + log.exception("Unexpected error") return (None, None, None) # could not determine viewport def ucsc_links(self, dataset, type, app, base_url): ret_val = [] chrom, start, stop = self.get_estimated_display_viewport(dataset) if chrom is not None: - for site_name, site_url in app.datatypes_registry.get_legacy_sites_by_build('ucsc', dataset.dbkey): - if site_name in app.datatypes_registry.get_display_sites('ucsc'): + for site_name, site_url in app.datatypes_registry.get_legacy_sites_by_build("ucsc", dataset.dbkey): + if site_name in app.datatypes_registry.get_display_sites("ucsc"): internal_url = f"{app.url_for(controller='dataset', dataset_id=dataset.id, action='display_at', filename='ucsc_' + site_name)}" - display_url = quote_plus("%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" % (base_url, app.url_for(controller='root'), dataset.id, type)) - redirect_url = quote_plus(f"{site_url}db={dataset.dbkey}&position={chrom}:{start}-{stop}&hgt.customText=%s") - link = f'{internal_url}?redirect_url={redirect_url}&display_url={display_url}' + display_url = quote_plus( + "%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" + % (base_url, app.url_for(controller="root"), dataset.id, type) + ) + redirect_url = quote_plus( + f"{site_url}db={dataset.dbkey}&position={chrom}:{start}-{stop}&hgt.customText=%s" + ) + link = f"{internal_url}?redirect_url={redirect_url}&display_url={display_url}" ret_val.append((site_name, link)) return ret_val @@ -1366,13 +1556,13 @@ class CustomTrack(Tabular): if first_line: first_line = False try: - if hdr[0].startswith('track'): + if hdr[0].startswith("track"): color_found = False visibility_found = False for elem in hdr[1:]: - if elem.startswith('color'): + if elem.startswith("color"): color_found = True - if elem.startswith('visibility'): + if elem.startswith("visibility"): visibility_found = True if color_found and visibility_found: break @@ -1384,7 +1574,7 @@ class CustomTrack(Tabular): return False else: try: - if hdr[0] and not hdr[0].startswith('#'): + if hdr[0] and not hdr[0].startswith("#"): if len(hdr) < 3: return False try: @@ -1399,7 +1589,7 @@ class CustomTrack(Tabular): class ENCODEPeak(Interval): - ''' + """ Human ENCODE peak format. There are both broad and narrow peak formats. Formats are very similar; narrow peak has an additional column, though. @@ -1410,16 +1600,23 @@ class ENCODEPeak(Interval): Narrow peak http://genome.ucsc.edu/FAQ/FAQformat#format12 and : This format is used to provide called peaks of signal enrichment based on pooled, normalized (interpreted) data. It is a BED6+4 format. - ''' + """ + edam_format = "format_3612" file_ext = "encodepeak" - column_names = ['Chrom', 'Start', 'End', 'Name', 'Score', 'Strand', 'SignalValue', 'pValue', 'qValue', 'Peak'] + column_names = ["Chrom", "Start", "End", "Name", "Score", "Strand", "SignalValue", "pValue", "qValue", "Peak"] data_sources = {"data": "tabix", "index": "bigwig"} MetadataElement(name="chromCol", default=1, desc="Chrom column", param=metadata.ColumnParameter) MetadataElement(name="startCol", default=2, desc="Start column", param=metadata.ColumnParameter) MetadataElement(name="endCol", default=3, desc="End column", param=metadata.ColumnParameter) - MetadataElement(name="strandCol", desc="Strand column (click box & select)", param=metadata.ColumnParameter, optional=True, no_value=0) + MetadataElement( + name="strandCol", + desc="Strand column (click box & select)", + param=metadata.ColumnParameter, + optional=True, + no_value=0, + ) MetadataElement(name="columns", default=3, desc="Number of columns", readonly=True, visible=False) def sniff(self, filename): @@ -1427,13 +1624,14 @@ class ENCODEPeak(Interval): class ChromatinInteractions(Interval): - ''' + """ Chromatin interactions obtained from 3C/5C/Hi-C experiments. - ''' + """ + file_ext = "chrint" track_type = "DiagonalHeatmapTrack" data_sources = {"data": "tabix", "index": "bigwig"} - column_names = ['Chrom1', 'Start1', 'End1', 'Chrom2', 'Start2', 'End2', 'Value'] + column_names = ["Chrom1", "Start1", "End1", "Chrom2", "Start2", "End2", "Value"] MetadataElement(name="chrom1Col", default=1, desc="Chrom1 column", param=metadata.ColumnParameter) MetadataElement(name="start1Col", default=2, desc="Start1 column", param=metadata.ColumnParameter) @@ -1471,10 +1669,19 @@ class ScIdx(Tabular): >>> ScIdx().sniff(fname) False """ + file_ext = "scidx" MetadataElement(name="columns", default=0, desc="Number of columns", readonly=True, visible=False) - MetadataElement(name="column_types", default=[], param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False, no_value=[]) + MetadataElement( + name="column_types", + default=[], + param=metadata.ColumnTypesParameter, + desc="Column types", + readonly=True, + visible=False, + no_value=[], + ) def __init__(self, **kwd): """ @@ -1483,18 +1690,18 @@ class ScIdx(Tabular): Tabular.__init__(self, **kwd) # Don't set column names since the first # line of the dataset displays them. - self.column_names = ['chrom', 'index', 'forward', 'reverse', 'value'] + self.column_names = ["chrom", "index", "forward", "reverse", "value"] def sniff_prefix(self, file_prefix: FilePrefix): """ Checks for 'scidx-ness.' """ count = 0 - for count, line in enumerate(iter_headers(file_prefix, '\t')): + for count, line in enumerate(iter_headers(file_prefix, "\t")): # The first line is always a comment like this: # 2015-11-23 20:18:56.51;input.bam;READ1 if count == 0: - if not line[0].startswith('#'): + if not line[0].startswith("#"): return False # The 2nd line is always a specific header elif count == 1: @@ -1517,6 +1724,7 @@ class ScIdx(Tabular): return False -if __name__ == '__main__': +if __name__ == "__main__": import doctest + doctest.testmod(sys.modules[__name__]) diff --git a/lib/galaxy/datatypes/isa.py b/lib/galaxy/datatypes/isa.py index 66a3338e892..f991e6e6a65 100644 --- a/lib/galaxy/datatypes/isa.py +++ b/lib/galaxy/datatypes/isa.py @@ -16,7 +16,7 @@ import tempfile logging.getLogger("isatools.isatab").setLevel(logging.ERROR) from isatools import ( isajson, - isatab_meta + isatab_meta, ) from markupsafe import escape @@ -50,9 +50,11 @@ logger = logging.getLogger(__name__) # ISA class {{{1 ################################################################ + class _Isa(data.Data): - """ Base class for implementing ISA datatypes """ - composite_type = 'auto_primary_file' + """Base class for implementing ISA datatypes""" + + composite_type = "auto_primary_file" is_binary = True _main_file_regex = None @@ -78,7 +80,7 @@ class _Isa(data.Data): def _get_isa_folder_path(self, dataset): isa_folder = dataset.extra_files_path if not isa_folder: - raise Exception('Unvalid dataset object, or no extra files path found for this dataset.') + raise Exception("Unvalid dataset object, or no extra files path found for this dataset.") return isa_folder # Get main file {{{2 @@ -99,7 +101,7 @@ class _Isa(data.Data): main_file = self._find_main_file_in_archive(isa_files) if main_file is None: - raise Exception('Invalid ISA archive. No main file found.') + raise Exception("Invalid ISA archive. No main file found.") # Make full path main_file = os.path.join(isa_folder, main_file) @@ -111,7 +113,7 @@ class _Isa(data.Data): def _get_investigation(self, dataset): """Create a contained instance specific to the exact ISA type (Tab or Json). - We will use it to parse and access information from the archive.""" + We will use it to parse and access information from the archive.""" investigation = None main_file = self._get_main_file(dataset) @@ -134,7 +136,11 @@ class _Isa(data.Data): if found_file is None: found_file = match.group() else: - raise Exception('More than one file match the pattern "', str(self._main_file_regex), '" to identify the investigation file') + raise Exception( + 'More than one file match the pattern "', + str(self._main_file_regex), + '" to identify the investigation file', + ) return found_file @@ -150,7 +156,7 @@ class _Isa(data.Data): raise RuntimeError("Unable to find the main file within the 'files_path' folder") # Read first lines of main file - with open(main_file, encoding='utf-8') as f: + with open(main_file, encoding="utf-8") as f: data = [] for line in f: if len(data) < _MAX_LINES_HISTORY_PEEK: @@ -159,10 +165,10 @@ class _Isa(data.Data): break if not dataset.dataset.purged and data: dataset.peek = json.dumps({"data": data}) - dataset.blurb = 'data' + dataset.blurb = "data" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" # Display peek {{{2 ################################################################ @@ -180,7 +186,7 @@ class _Isa(data.Data): if not line: continue out.append(f"{escape(util.unicodify(line, 'utf-8'))}") - out.append('') + out.append("") out = "".join(out) except Exception as exc: out = f"Can't create peek: {util.unicodify(exc)}" @@ -191,17 +197,17 @@ class _Isa(data.Data): def generate_primary_file(self, dataset=None): """Generate the primary file. It is an HTML file containing description of the composite dataset - as well as a list of the composite files that it contains.""" + as well as a list of the composite files that it contains.""" if dataset: - rval = ['ISA Dataset

          '] + rval = ["ISA Dataset

          "] if hasattr(dataset, "extra_files_path"): - rval.append('

          ISA Dataset composed of the following files:

            ') + rval.append("
            ISA Dataset composed of the following files:

              ") for cmp_file in os.listdir(dataset.extra_files_path): rval.append(f'
            • {escape(cmp_file)}
            • ') - rval.append('
            ') + rval.append("
          ") else: - rval.append('
          ISA Dataset is empty!

            ') + rval.append("
            ISA Dataset is empty!

              ") return "\n".join(rval) return "
              No dataset available
              " @@ -232,7 +238,7 @@ class _Isa(data.Data): CompressedFile(file_name).extract(temp_folder) shutil.rmtree(output_path) extracted_files = os.listdir(temp_folder) - logger.debug(' '.join(extracted_files)) + logger.debug(" ".join(extracted_files)) if len(extracted_files) == 0: os.makedirs(output_path) shutil.rmtree(temp_folder) @@ -247,8 +253,8 @@ class _Isa(data.Data): def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, offset=None, ck_size=None, **kwd): """Downloads the ISA dataset if `preview` is `False`; - if `preview` is `True`, it returns a preview of the ISA dataset as a HTML page. - The preview is triggered when user clicks on the eye icon of the composite dataset.""" + if `preview` is `True`, it returns a preview of the ISA dataset as a HTML page. + The preview is triggered when user clicks on the eye icon of the composite dataset.""" headers = kwd.get("headers", {}) # if it is not required a preview use the default behaviour of `display_data` @@ -265,45 +271,46 @@ class _Isa(data.Data):

              You may also try to look into your zip file in order to find out if this is a proper ISA archive. If you see a file i_Investigation.txt inside, then it is an ISA-Tab archive. If you see a file with extension .json inside, then it is an ISA-JSON archive. If you see nothing like that, then either your ISA archive is corrupted, or it is not an ISA archive.

              """ else: - html = '' - html += f'

              {investigation.title} {investigation.identifier}

              ' + html = "" + html += f"

              {investigation.title} {investigation.identifier}

              " # Loop on all studies for study in investigation.studies: - html += f'

              Study {study.identifier}

              ' - html += f'

              {study.title}

              ' - html += f'

              {study.description}

              ' - html += f'

              Submitted the {study.submission_date}

              ' - html += f'

              Released on {study.public_release_date}

              ' + html += f"

              Study {study.identifier}

              " + html += f"

              {study.title}

              " + html += f"

              {study.description}

              " + html += f"

              Submitted the {study.submission_date}

              " + html += f"

              Released on {study.public_release_date}

              " html += f"

              Experimental factors used: {', '.join(x.name for x in study.factors)}

              " # Loop on all assays of this study for assay in study.assays: - html += f'

              Assay {assay.filename}

              ' - html += f'

              Measurement type: {assay.measurement_type.term}

              ' # OntologyAnnotation - html += f'

              Technology type: {assay.technology_type.term}

              ' # OntologyAnnotation - html += f'

              Technology platform: {assay.technology_platform}

              ' + html += f"

              Assay {assay.filename}

              " + html += f"

              Measurement type: {assay.measurement_type.term}

              " # OntologyAnnotation + html += f"

              Technology type: {assay.technology_type.term}

              " # OntologyAnnotation + html += f"

              Technology platform: {assay.technology_platform}

              " if assay.data_files is not None: - html += '

              Data files:

              ' - html += '
                ' + html += "

                Data files:

                " + html += "
                  " for data_file in assay.data_files: - if data_file.filename != '': + if data_file.filename != "": html += f"
                • {escape(util.unicodify(str(data_file.filename), 'utf-8'))} - {escape(util.unicodify(str(data_file.label), 'utf-8'))}
                • " - html += '
                ' + html += "
              " - html += '' + html += "" # Set mime type - mime = 'text/html' + mime = "text/html" self._clean_and_set_mime_type(trans, mime, headers) - return sanitize_html(html).encode('utf-8'), headers + return sanitize_html(html).encode("utf-8"), headers # ISA-Tab class {{{1 ################################################################ + class IsaTab(_Isa): file_ext = "isa-tab" @@ -321,7 +328,7 @@ class IsaTab(_Isa): # Parse ISA-Tab investigation file parser = isatab_meta.InvestigationParser() isa_dir = os.path.dirname(filename) - with open(filename, newline='', encoding='utf8') as fp: + with open(filename, newline="", encoding="utf8") as fp: parser.parse(fp) for study in parser.isa.studies: s_parser = isatab_meta.LazyStudySampleTableParser(parser.isa) @@ -337,6 +344,7 @@ class IsaTab(_Isa): # ISA-JSON class {{{1 ################################################################ + class IsaJson(_Isa): file_ext = "isa-json" @@ -352,7 +360,7 @@ class IsaJson(_Isa): def _make_investigation_instance(self, filename): # Parse JSON file - with open(filename, newline='', encoding='utf8') as fp: + with open(filename, newline="", encoding="utf8") as fp: isa = isajson.load(fp) return isa diff --git a/lib/galaxy/datatypes/media.py b/lib/galaxy/datatypes/media.py index 9b3e9ab9ddc..4c7b54418eb 100644 --- a/lib/galaxy/datatypes/media.py +++ b/lib/galaxy/datatypes/media.py @@ -4,75 +4,176 @@ import subprocess import wave from galaxy.datatypes.binary import Binary -from galaxy.datatypes.metadata import ListParameter, MetadataElement +from galaxy.datatypes.metadata import ( + ListParameter, + MetadataElement, +) from galaxy.util import which def ffprobe(path): - data = json.loads(subprocess.check_output(['ffprobe', '-loglevel', 'quiet', '-show_format', '-show_streams', '-of', 'json', path]).decode("utf-8")) - return data['format'], data['streams'] + data = json.loads( + subprocess.check_output( + ["ffprobe", "-loglevel", "quiet", "-show_format", "-show_streams", "-of", "json", path] + ).decode("utf-8") + ) + return data["format"], data["streams"] class Audio(Binary): - MetadataElement(name="duration", default=0, desc="Length of audio sample", readonly=True, visible=True, optional=True, no_value=0) - MetadataElement(name="audio_codecs", default=[], desc="Audio codec(s)", param=ListParameter, readonly=True, visible=True, optional=True, no_value=[]) - MetadataElement(name="sample_rates", default=[], desc="Sampling Rate(s)", param=ListParameter, readonly=True, visible=True, optional=True, no_value=[]) - MetadataElement(name="audio_streams", default=0, desc="Number of audio streams", readonly=True, visible=True, optional=True, no_value=0) + MetadataElement( + name="duration", + default=0, + desc="Length of audio sample", + readonly=True, + visible=True, + optional=True, + no_value=0, + ) + MetadataElement( + name="audio_codecs", + default=[], + desc="Audio codec(s)", + param=ListParameter, + readonly=True, + visible=True, + optional=True, + no_value=[], + ) + MetadataElement( + name="sample_rates", + default=[], + desc="Sampling Rate(s)", + param=ListParameter, + readonly=True, + visible=True, + optional=True, + no_value=[], + ) + MetadataElement( + name="audio_streams", + default=0, + desc="Number of audio streams", + readonly=True, + visible=True, + optional=True, + no_value=0, + ) def set_meta(self, dataset, **kwd): - if which('ffprobe'): + if which("ffprobe"): metadata, streams = ffprobe(dataset.file_name) - dataset.metadata.duration = metadata['duration'] - dataset.metadata.audio_codecs = [stream['codec_name'] for stream in streams if stream['codec_type'] == 'audio'] - dataset.metadata.sample_rates = [stream['sample_rate'] for stream in streams if stream['codec_type'] == 'audio'] - dataset.metadata.audio_streams = len([stream for stream in streams if stream['codec_type'] == 'audio']) + dataset.metadata.duration = metadata["duration"] + dataset.metadata.audio_codecs = [ + stream["codec_name"] for stream in streams if stream["codec_type"] == "audio" + ] + dataset.metadata.sample_rates = [ + stream["sample_rate"] for stream in streams if stream["codec_type"] == "audio" + ] + dataset.metadata.audio_streams = len([stream for stream in streams if stream["codec_type"] == "audio"]) class Video(Binary): - MetadataElement(name="resolution_w", default=0, desc="Width of video stream", readonly=True, visible=True, optional=True, no_value=0) - MetadataElement(name="resolution_h", default=0, desc="Height of video stream", readonly=True, visible=True, optional=True, no_value=0) - MetadataElement(name="fps", default=0, desc="FPS of video stream", readonly=True, visible=True, optional=True, no_value=0) - MetadataElement(name="video_codecs", default=[], desc="Video codec(s)", param=ListParameter, readonly=True, visible=True, optional=True, no_value=[]) - MetadataElement(name="audio_codecs", default=[], desc="Audio codec(s)", param=ListParameter, readonly=True, visible=True, optional=True, no_value=[]) - MetadataElement(name="video_streams", default=0, desc="Number of video streams", readonly=True, visible=True, optional=True, no_value=0) - MetadataElement(name="audio_streams", default=0, desc="Number of audio streams", readonly=True, visible=True, optional=True, no_value=0) + MetadataElement( + name="resolution_w", + default=0, + desc="Width of video stream", + readonly=True, + visible=True, + optional=True, + no_value=0, + ) + MetadataElement( + name="resolution_h", + default=0, + desc="Height of video stream", + readonly=True, + visible=True, + optional=True, + no_value=0, + ) + MetadataElement( + name="fps", default=0, desc="FPS of video stream", readonly=True, visible=True, optional=True, no_value=0 + ) + MetadataElement( + name="video_codecs", + default=[], + desc="Video codec(s)", + param=ListParameter, + readonly=True, + visible=True, + optional=True, + no_value=[], + ) + MetadataElement( + name="audio_codecs", + default=[], + desc="Audio codec(s)", + param=ListParameter, + readonly=True, + visible=True, + optional=True, + no_value=[], + ) + MetadataElement( + name="video_streams", + default=0, + desc="Number of video streams", + readonly=True, + visible=True, + optional=True, + no_value=0, + ) + MetadataElement( + name="audio_streams", + default=0, + desc="Number of audio streams", + readonly=True, + visible=True, + optional=True, + no_value=0, + ) def _get_resolution(self, streams): for stream in streams: - if stream['codec_type'] == 'video': - w = stream['width'] - h = stream['height'] - dividend, divisor = stream['avg_frame_rate'].split('/') + if stream["codec_type"] == "video": + w = stream["width"] + h = stream["height"] + dividend, divisor = stream["avg_frame_rate"].split("/") fps = float(dividend) / float(divisor) else: w = h = fps = 0 return w, h, fps def set_meta(self, dataset, **kwd): - if which('ffprobe'): + if which("ffprobe"): metadata, streams = ffprobe(dataset.file_name) (w, h, fps) = self._get_resolution(streams) dataset.metadata.resolution_w = w dataset.metadata.resolution_h = h dataset.metadata.fps = fps - dataset.metadata.audio_codecs = [stream['codec_name'] for stream in streams if stream['codec_type'] == 'audio'] - dataset.metadata.video_codecs = [stream['codec_name'] for stream in streams if stream['codec_type'] == 'video'] + dataset.metadata.audio_codecs = [ + stream["codec_name"] for stream in streams if stream["codec_type"] == "audio" + ] + dataset.metadata.video_codecs = [ + stream["codec_name"] for stream in streams if stream["codec_type"] == "video" + ] - dataset.metadata.audio_streams = len([stream for stream in streams if stream['codec_type'] == 'audio']) - dataset.metadata.video_streams = len([stream for stream in streams if stream['codec_type'] == 'video']) + dataset.metadata.audio_streams = len([stream for stream in streams if stream["codec_type"] == "audio"]) + dataset.metadata.video_streams = len([stream for stream in streams if stream["codec_type"] == "video"]) class Mkv(Video): file_ext = "mkv" def sniff(self, filename): - if which('ffprobe'): + if which("ffprobe"): metadata, streams = ffprobe(filename) - return 'matroska' in metadata['format_name'].split(',') + return "matroska" in metadata["format_name"].split(",") class Mp4(Video): @@ -88,27 +189,27 @@ class Mp4(Video): file_ext = "mp4" def sniff(self, filename): - if which('ffprobe'): + if which("ffprobe"): metadata, streams = ffprobe(filename) - return 'mp4' in metadata['format_name'].split(',') + return "mp4" in metadata["format_name"].split(",") class Flv(Video): file_ext = "flv" def sniff(self, filename): - if which('ffprobe'): + if which("ffprobe"): metadata, streams = ffprobe(filename) - return 'flv' in metadata['format_name'].split(',') + return "flv" in metadata["format_name"].split(",") class Mpg(Video): file_ext = "mpg" def sniff(self, filename): - if which('ffprobe'): + if which("ffprobe"): metadata, streams = ffprobe(filename) - return 'mpegvideo' in metadata['format_name'].split(',') + return "mpegvideo" in metadata["format_name"].split(",") class Mp3(Audio): @@ -120,12 +221,13 @@ class Mp3(Audio): >>> sniff_with_cls(Mp3, 'audio_1.wav') False """ + file_ext = "mp3" def sniff(self, filename): - if which('ffprobe'): + if which("ffprobe"): metadata, streams = ffprobe(filename) - return 'mp3' in metadata['format_name'].split(',') + return "mp3" in metadata["format_name"].split(",") class Wav(Audio): @@ -138,27 +240,34 @@ class Wav(Audio): >>> sniff_with_cls(Wav, 'drugbank_drugs.cml') False """ + file_ext = "wav" blurb = "RIFF WAV Audio file" is_binary = True MetadataElement(name="rate", desc="Sample Rate", default=0, no_value=0, readonly=True, visible=True, optional=True) - MetadataElement(name="nframes", desc="Number of Samples", default=0, no_value=0, readonly=True, visible=True, optional=True) - MetadataElement(name="nchannels", desc="Number of Channels", default=0, no_value=0, readonly=True, visible=True, optional=True) - MetadataElement(name="sampwidth", desc="Sample Width", default=0, no_value=0, readonly=True, visible=True, optional=True) + MetadataElement( + name="nframes", desc="Number of Samples", default=0, no_value=0, readonly=True, visible=True, optional=True + ) + MetadataElement( + name="nchannels", desc="Number of Channels", default=0, no_value=0, readonly=True, visible=True, optional=True + ) + MetadataElement( + name="sampwidth", desc="Sample Width", default=0, no_value=0, readonly=True, visible=True, optional=True + ) def get_mime(self): """Returns the mime type of the datatype.""" - return 'audio/wav' + return "audio/wav" def sniff(self, filename): - with wave.open(filename, 'rb'): + with wave.open(filename, "rb"): return True def set_meta(self, dataset, overwrite=True, **kwd): """Set the metadata for this dataset from the file contents.""" try: - with wave.open(dataset.dataset.file_name, 'rb') as fd: + with wave.open(dataset.dataset.file_name, "rb") as fd: dataset.metadata.rate = fd.getframerate() dataset.metadata.nframes = fd.getnframes() dataset.metadata.sampwidth = fd.getsampwidth() diff --git a/lib/galaxy/datatypes/metacyto.py b/lib/galaxy/datatypes/metacyto.py index d18759fcb34..33e05752fb9 100644 --- a/lib/galaxy/datatypes/metacyto.py +++ b/lib/galaxy/datatypes/metacyto.py @@ -12,13 +12,14 @@ log = logging.getLogger(__name__) class mStats(Tabular): """Class describing the table of cluster statistics output from MetaCyto""" + file_ext = "metacyto_stats.txt" def sniff_prefix(self, file_prefix: FilePrefix): """Quick test on file headings""" if file_prefix.startswith("fcs_files\tcluster_id\tlabel\tfcs_names"): header_line = file_prefix.string_io().readline() - if header_line.strip().split("\t")[-1] == 'fraction': + if header_line.strip().split("\t")[-1] == "fraction": return True elif file_prefix.truncated and file_prefix.string_io().read() == header_line: return True @@ -27,7 +28,8 @@ class mStats(Tabular): class mSummary(Tabular): """Class describing the summary table output by MetaCyto after FCS preprocessing""" + file_ext = "metacyto_summary.txt" def sniff_prefix(self, file_prefix: FilePrefix): - return file_prefix.startswith('study_id\tantibodies\tfilenames') + return file_prefix.startswith("study_id\tantibodies\tfilenames") diff --git a/lib/galaxy/datatypes/microarrays.py b/lib/galaxy/datatypes/microarrays.py index fc83c051f2e..56ab225198c 100644 --- a/lib/galaxy/datatypes/microarrays.py +++ b/lib/galaxy/datatypes/microarrays.py @@ -7,7 +7,7 @@ from galaxy.datatypes.metadata import MetadataElement from galaxy.datatypes.sniff import ( build_sniff_from_prefix, FilePrefix, - get_headers + get_headers, ) log = logging.getLogger(__name__) @@ -17,24 +17,64 @@ class GenericMicroarrayFile(data.Text): """ Abstract class for most of the microarray files. """ - MetadataElement(name="version_number", default="1.0", desc="Version number", readonly=True, visible=True, - optional=True, no_value="1.0") - MetadataElement(name="file_format", default="ATF", desc="File format", readonly=True, visible=True, - optional=True, no_value="ATF") - MetadataElement(name="number_of_optional_header_records", default=1, desc="Number of optional header records", - readonly=True, visible=True, optional=True, no_value=1) - MetadataElement(name="number_of_data_columns", default=1, desc="Number of data columns", - readonly=True, visible=True, - optional=True, no_value=1) - MetadataElement(name="file_type", default="GenePix", desc="File type", - readonly=True, visible=True, - optional=True, no_value="GenePix") - MetadataElement(name="block_count", default=1, desc="Number of blocks described in the file", - readonly=True, visible=True, - optional=True, no_value=1) - MetadataElement(name="block_type", default=0, desc="Type of block", - readonly=True, visible=True, - optional=True, no_value=0) + + MetadataElement( + name="version_number", + default="1.0", + desc="Version number", + readonly=True, + visible=True, + optional=True, + no_value="1.0", + ) + MetadataElement( + name="file_format", + default="ATF", + desc="File format", + readonly=True, + visible=True, + optional=True, + no_value="ATF", + ) + MetadataElement( + name="number_of_optional_header_records", + default=1, + desc="Number of optional header records", + readonly=True, + visible=True, + optional=True, + no_value=1, + ) + MetadataElement( + name="number_of_data_columns", + default=1, + desc="Number of data columns", + readonly=True, + visible=True, + optional=True, + no_value=1, + ) + MetadataElement( + name="file_type", + default="GenePix", + desc="File type", + readonly=True, + visible=True, + optional=True, + no_value="GenePix", + ) + MetadataElement( + name="block_count", + default=1, + desc="Number of blocks described in the file", + readonly=True, + visible=True, + optional=True, + no_value=1, + ) + MetadataElement( + name="block_type", default=0, desc="Type of block", readonly=True, visible=True, optional=True, no_value=0 + ) def set_peek(self, dataset): if not dataset.dataset.purged: @@ -44,17 +84,17 @@ class GenericMicroarrayFile(data.Text): dataset.blurb = f"{dataset.metadata.file_type} {dataset.metadata.version_number}: Format {dataset.metadata.file_format}, {dataset.metadata.block_count} blocks, {dataset.metadata.number_of_optional_header_records} headers and {dataset.metadata.number_of_data_columns} columns" dataset.peek = get_file_peek(dataset.file_name) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def get_mime(self): - return 'text/plain' + return "text/plain" @build_sniff_from_prefix class Gal(GenericMicroarrayFile): - """ Gal File format described at: - http://mdc.custhelp.com/app/answers/detail/a_id/18883/#gal + """Gal File format described at: + http://mdc.custhelp.com/app/answers/detail/a_id/18883/#gal """ edam_format = "format_3829" @@ -94,8 +134,8 @@ class Gal(GenericMicroarrayFile): @build_sniff_from_prefix class Gpr(GenericMicroarrayFile): - """ Gpr File format described at: - http://mdc.custhelp.com/app/answers/detail/a_id/18883/#gpr + """Gpr File format described at: + http://mdc.custhelp.com/app/answers/detail/a_id/18883/#gpr """ edam_format = "format_3829" diff --git a/lib/galaxy/datatypes/molecules.py b/lib/galaxy/datatypes/molecules.py index 5c95158a9e2..852b6f2206e 100644 --- a/lib/galaxy/datatypes/molecules.py +++ b/lib/galaxy/datatypes/molecules.py @@ -13,14 +13,14 @@ from galaxy.datatypes.sniff import ( build_sniff_from_prefix, FilePrefix, get_headers, - iter_headers + iter_headers, ) from galaxy.datatypes.tabular import Tabular from galaxy.datatypes.util.generic_util import count_special_lines from galaxy.datatypes.xml import GenericXml from galaxy.util import ( commands, - unicodify + unicodify, ) log = logging.getLogger(__name__) @@ -31,9 +31,9 @@ def count_lines(filename, non_empty=False): counting the number of lines from the 'filename' file """ if non_empty: - cmd = ['grep', '-cve', r'^\s*$', filename] + cmd = ["grep", "-cve", r"^\s*$", filename] else: - cmd = ['wc', '-l', filename] + cmd = ["wc", "-l", filename] try: out = commands.execute(cmd) except commands.CommandLineException as e: @@ -46,21 +46,30 @@ class GenericMolFile(Text): """ Abstract class for most of the molecule files. """ - MetadataElement(name="number_of_molecules", default=0, desc="Number of molecules", readonly=True, visible=True, optional=True, no_value=0) + + MetadataElement( + name="number_of_molecules", + default=0, + desc="Number of molecules", + readonly=True, + visible=True, + optional=True, + no_value=0, + ) def set_peek(self, dataset): if not dataset.dataset.purged: - if (dataset.metadata.number_of_molecules == 1): + if dataset.metadata.number_of_molecules == 1: dataset.blurb = "1 molecule" else: dataset.blurb = f"{dataset.metadata.number_of_molecules} molecules" dataset.peek = get_file_peek(dataset.file_name) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def get_mime(self): - return 'text/plain' + return "text/plain" class MOL(GenericMolFile): @@ -104,17 +113,16 @@ class SDF(GenericMolFile): idx = 0 for line in file_prefix.line_iterator(): idx += 1 - line = line.rstrip('\n\r') + line = line.rstrip("\n\r") if idx < 4: continue elif idx == 4: - if len(line) != 39 or not(line.endswith(' V2000') - or line.endswith(' V3000')): + if len(line) != 39 or not (line.endswith(" V2000") or line.endswith(" V3000")): return False elif not m_end_found: - if line == 'M END': + if line == "M END": m_end_found = True - elif line == '$$$$': + elif line == "$$$$": return True if idx == limit: break @@ -139,10 +147,10 @@ class SDF(GenericMolFile): input_files = [ds.file_name for ds in input_datasets] chunk_size = None - if split_params['split_mode'] == 'number_of_parts': + if split_params["split_mode"] == "number_of_parts": raise Exception(f"Split mode \"{split_params['split_mode']}\" is currently not implemented for SD-files.") - elif split_params['split_mode'] == 'to_size': - chunk_size = int(split_params['split_size']) + elif split_params["split_mode"] == "to_size": + chunk_size = int(split_params["split_size"]) else: raise Exception(f"Unsupported split mode {split_params['split_mode']}") @@ -158,7 +166,7 @@ class SDF(GenericMolFile): def _write_part_sdf_file(accumulated_lines): part_dir = subdir_generator_function() part_path = os.path.join(part_dir, os.path.basename(input_files[0])) - with open(part_path, 'w') as part_file: + with open(part_path, "w") as part_file: part_file.writelines(accumulated_lines) try: @@ -172,7 +180,7 @@ class SDF(GenericMolFile): if sdf_lines_accumulated: _write_part_sdf_file(sdf_lines_accumulated) except Exception as e: - log.error('Unable to split files: %s', unicodify(e)) + log.error("Unable to split files: %s", unicodify(e)) raise @@ -195,8 +203,8 @@ class MOL2(GenericMolFile): limit = 60 idx = 0 for line in file_prefix.line_iterator(): - line = line.rstrip('\n\r') - if line == '@MOLECULE': + line = line.rstrip("\n\r") + if line == "@MOLECULE": return True idx += 1 if idx == limit: @@ -222,10 +230,10 @@ class MOL2(GenericMolFile): input_files = [ds.file_name for ds in input_datasets] chunk_size = None - if split_params['split_mode'] == 'number_of_parts': + if split_params["split_mode"] == "number_of_parts": raise Exception(f"Split mode \"{split_params['split_mode']}\" is currently not implemented for MOL2-files.") - elif split_params['split_mode'] == 'to_size': - chunk_size = int(split_params['split_size']) + elif split_params["split_mode"] == "to_size": + chunk_size = int(split_params["split_size"]) else: raise Exception(f"Unsupported split mode {split_params['split_mode']}") @@ -245,7 +253,7 @@ class MOL2(GenericMolFile): def _write_part_mol2_file(accumulated_lines): part_dir = subdir_generator_function() part_path = os.path.join(part_dir, os.path.basename(input_files[0])) - with open(part_path, 'w') as part_file: + with open(part_path, "w") as part_file: part_file.writelines(accumulated_lines) try: @@ -259,7 +267,7 @@ class MOL2(GenericMolFile): if mol2_lines_accumulated: _write_part_mol2_file(mol2_lines_accumulated) except Exception as e: - log.error('Unable to split files: %s', unicodify(e)) + log.error("Unable to split files: %s", unicodify(e)) raise @@ -268,6 +276,7 @@ class FPS(GenericMolFile): """ chemfp fingerprint file: http://code.google.com/p/chem-fingerprints/wiki/FPS """ + file_ext = "fps" def sniff_prefix(self, file_prefix: FilePrefix): @@ -282,8 +291,8 @@ class FPS(GenericMolFile): >>> FPS().sniff(fname) False """ - header = get_headers(file_prefix, sep='\t', count=1) - if header[0][0].strip() == '#FPS1': + header = get_headers(file_prefix, sep="\t", count=1) + if header[0][0].strip() == "#FPS1": return True else: return False @@ -292,7 +301,7 @@ class FPS(GenericMolFile): """ Set the number of lines of data in dataset. """ - dataset.metadata.number_of_molecules = count_special_lines('^#', dataset.file_name, invert=True) + dataset.metadata.number_of_molecules = count_special_lines("^#", dataset.file_name, invert=True) @classmethod def split(cls, input_datasets, subdir_generator_function, split_params): @@ -307,17 +316,17 @@ class FPS(GenericMolFile): input_files = [ds.file_name for ds in input_datasets] chunk_size = None - if split_params['split_mode'] == 'number_of_parts': + if split_params["split_mode"] == "number_of_parts": raise Exception(f"Split mode \"{split_params['split_mode']}\" is currently not implemented for MOL2-files.") - elif split_params['split_mode'] == 'to_size': - chunk_size = int(split_params['split_size']) + elif split_params["split_mode"] == "to_size": + chunk_size = int(split_params["split_size"]) else: raise Exception(f"Unsupported split mode {split_params['split_mode']}") def _write_part_fingerprint_file(accumulated_lines): part_dir = subdir_generator_function() part_path = os.path.join(part_dir, os.path.basename(input_files[0])) - with open(part_path, 'w') as part_file: + with open(part_path, "w") as part_file: part_file.writelines(accumulated_lines) try: @@ -327,7 +336,7 @@ class FPS(GenericMolFile): for line in open(input_files[0]): if not line.strip(): continue - if line.startswith('#'): + if line.startswith("#"): header_lines.append(line) else: fingerprint_counter += 1 @@ -338,7 +347,7 @@ class FPS(GenericMolFile): if lines_accumulated: _write_part_fingerprint_file(header_lines + lines_accumulated) except Exception as e: - log.error('Unable to split files: %s', unicodify(e)) + log.error("Unable to split files: %s", unicodify(e)) raise @staticmethod @@ -351,14 +360,13 @@ class FPS(GenericMolFile): # For one file only, use base class method (move/copy) return Text.merge(split_files, output_file) if not split_files: - raise ValueError("No fps files given, %r, to merge into %s" - % (split_files, output_file)) + raise ValueError("No fps files given, %r, to merge into %s" % (split_files, output_file)) with open(output_file, "w") as out: first = True for filename in split_files: with open(filename) as handle: for line in handle: - if line.startswith('#'): + if line.startswith("#"): if first: out.write(line) else: @@ -369,30 +377,30 @@ class FPS(GenericMolFile): class OBFS(Binary): """OpenBabel Fastsearch format (fs).""" - file_ext = 'obfs' - composite_type = 'basic' - MetadataElement(name="base_name", default='OpenBabel Fastsearch Index', - readonly=True, visible=True, optional=True,) + file_ext = "obfs" + composite_type = "basic" + + MetadataElement( + name="base_name", + default="OpenBabel Fastsearch Index", + readonly=True, + visible=True, + optional=True, + ) def __init__(self, **kwd): """ - A Fastsearch Index consists of a binary file with the fingerprints - and a pointer the actual molecule file. + A Fastsearch Index consists of a binary file with the fingerprints + and a pointer the actual molecule file. """ super().__init__(**kwd) - self.add_composite_file('molecule.fs', is_binary=True, - description='OpenBabel Fastsearch Index') - self.add_composite_file('molecule.sdf', optional=True, - is_binary=False, description='Molecule File') - self.add_composite_file('molecule.smi', optional=True, - is_binary=False, description='Molecule File') - self.add_composite_file('molecule.inchi', optional=True, - is_binary=False, description='Molecule File') - self.add_composite_file('molecule.mol2', optional=True, - is_binary=False, description='Molecule File') - self.add_composite_file('molecule.cml', optional=True, - is_binary=False, description='Molecule File') + self.add_composite_file("molecule.fs", is_binary=True, description="OpenBabel Fastsearch Index") + self.add_composite_file("molecule.sdf", optional=True, is_binary=False, description="Molecule File") + self.add_composite_file("molecule.smi", optional=True, is_binary=False, description="Molecule File") + self.add_composite_file("molecule.inchi", optional=True, is_binary=False, description="Molecule File") + self.add_composite_file("molecule.mol2", optional=True, is_binary=False, description="Molecule File") + self.add_composite_file("molecule.cml", optional=True, is_binary=False, description="Molecule File") def set_peek(self, dataset): """Set the peek and blurb text.""" @@ -412,7 +420,7 @@ class OBFS(Binary): def get_mime(self): """Returns the mime type of the datatype (pretend it is text for peek)""" - return 'text/plain' + return "text/plain" def merge(split_files, output_file, extra_merge_args): """Merging Fastsearch indices is not supported.""" @@ -432,13 +440,14 @@ class DRF(GenericMolFile): """ Set the number of lines of data in dataset. """ - dataset.metadata.number_of_molecules = count_special_lines('\"ligand id\"', dataset.file_name, invert=True) + dataset.metadata.number_of_molecules = count_special_lines('"ligand id"', dataset.file_name, invert=True) class PHAR(GenericMolFile): """ Pharmacophore database format from silicos-it. """ + file_ext = "phar" def set_peek(self, dataset): @@ -446,8 +455,8 @@ class PHAR(GenericMolFile): dataset.peek = get_file_peek(dataset.file_name) dataset.blurb = "pharmacophore" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" @build_sniff_from_prefix @@ -456,6 +465,7 @@ class PDB(GenericMolFile): Protein Databank format. http://www.wwpdb.org/documentation/format33/v3.3.html """ + file_ext = "pdb" MetadataElement(name="chain_ids", default=[], desc="Chain IDs", readonly=False, visible=True) @@ -471,21 +481,21 @@ class PDB(GenericMolFile): >>> PDB().sniff(fname) False """ - headers = iter_headers(file_prefix, sep=' ', count=300) + headers = iter_headers(file_prefix, sep=" ", count=300) h = t = c = s = k = e = False for line in headers: section_name = line[0].strip() - if section_name == 'HEADER': + if section_name == "HEADER": h = True - elif section_name == 'TITLE': + elif section_name == "TITLE": t = True - elif section_name == 'COMPND': + elif section_name == "COMPND": c = True - elif section_name == 'SOURCE': + elif section_name == "SOURCE": s = True - elif section_name == 'KEYWDS': + elif section_name == "KEYWDS": k = True - elif section_name == 'EXPDTA': + elif section_name == "EXPDTA": e = True if h * t * c * s * k * e: @@ -501,24 +511,24 @@ class PDB(GenericMolFile): chain_ids = set() with open(dataset.file_name) as fh: for line in fh: - if line.startswith('ATOM ') or line.startswith('HETATM'): - if line[21] != ' ': + if line.startswith("ATOM ") or line.startswith("HETATM"): + if line[21] != " ": chain_ids.add(line[21]) dataset.metadata.chain_ids = list(chain_ids) except Exception as e: - log.error('Error finding chain_ids: %s', unicodify(e)) + log.error("Error finding chain_ids: %s", unicodify(e)) raise def set_peek(self, dataset): if not dataset.dataset.purged: atom_numbers = count_special_lines("^ATOM", dataset.file_name) hetatm_numbers = count_special_lines("^HETATM", dataset.file_name) - chain_ids = ','.join(dataset.metadata.chain_ids) if len(dataset.metadata.chain_ids) > 0 else 'None' + chain_ids = ",".join(dataset.metadata.chain_ids) if len(dataset.metadata.chain_ids) > 0 else "None" dataset.peek = get_file_peek(dataset.file_name) dataset.blurb = f"{atom_numbers} atoms and {hetatm_numbers} HET-atoms\nchain_ids: {chain_ids}" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" @build_sniff_from_prefix @@ -527,6 +537,7 @@ class PDBQT(GenericMolFile): PDBQT Autodock and Autodock Vina format http://autodock.scripps.edu/faqs-help/faq/what-is-the-format-of-a-pdbqt-file """ + file_ext = "pdbqt" def sniff_prefix(self, file_prefix: FilePrefix): @@ -541,19 +552,19 @@ class PDBQT(GenericMolFile): >>> PDBQT().sniff(fname) False """ - headers = iter_headers(file_prefix, sep=' ', count=300) + headers = iter_headers(file_prefix, sep=" ", count=300) h = t = c = s = k = False for line in headers: section_name = line[0].strip() - if section_name == 'REMARK': + if section_name == "REMARK": h = True - elif section_name == 'ROOT': + elif section_name == "ROOT": t = True - elif section_name == 'ENDROOT': + elif section_name == "ENDROOT": c = True - elif section_name == 'BRANCH': + elif section_name == "BRANCH": s = True - elif section_name == 'TORSDOF': + elif section_name == "TORSDOF": k = True if h * t * c * s * k: @@ -568,8 +579,8 @@ class PDBQT(GenericMolFile): dataset.peek = get_file_peek(dataset.file_name) dataset.blurb = f"{root_numbers} roots and {branch_numbers} branches" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" @build_sniff_from_prefix @@ -578,6 +589,7 @@ class PQR(GenericMolFile): Protein Databank format. https://apbs-pdb2pqr.readthedocs.io/en/latest/formats/pqr.html """ + file_ext = "pqr" MetadataElement(name="chain_ids", default=[], desc="Chain IDs", readonly=False, visible=True) @@ -604,16 +616,18 @@ class PQR(GenericMolFile): 11: Radius A float which provides the atomic radius (in angstroms). """ - pat = r'(ATOM|HETATM)\s+' +\ - r'(\d+)\s+' +\ - r'([A-Z0-9]+)\s+' +\ - r'([A-Z0-9]+)\s+' +\ - r'(([A-Z]?)\s+)?' +\ - r'([-+]?\d*\.\d+|\d+)\s+' +\ - r'([-+]?\d*\.\d+|\d+)\s+' +\ - r'([-+]?\d*\.\d+|\d+)\s+' +\ - r'([-+]?\d*\.\d+|\d+)\s+' +\ - r'([-+]?\d*\.\d+|\d+)\s+' + pat = ( + r"(ATOM|HETATM)\s+" + + r"(\d+)\s+" + + r"([A-Z0-9]+)\s+" + + r"([A-Z0-9]+)\s+" + + r"(([A-Z]?)\s+)?" + + r"([-+]?\d*\.\d+|\d+)\s+" + + r"([-+]?\d*\.\d+|\d+)\s+" + + r"([-+]?\d*\.\d+|\d+)\s+" + + r"([-+]?\d*\.\d+|\d+)\s+" + + r"([-+]?\d*\.\d+|\d+)\s+" + ) return re.compile(pat) def sniff_prefix(self, file_prefix: FilePrefix): @@ -628,14 +642,14 @@ class PQR(GenericMolFile): False """ prog = self.get_matcher() - headers = iter_headers(file_prefix, sep=None, comment_designator='REMARK 5', count=3000) + headers = iter_headers(file_prefix, sep=None, comment_designator="REMARK 5", count=3000) h = a = False for line in headers: section_name = line[0].strip() - if section_name == 'REMARK': + if section_name == "REMARK": h = True - elif section_name == 'ATOM' or section_name == 'HETATM': - if prog.match(' '.join(line)): + elif section_name == "ATOM" or section_name == "HETATM": + if prog.match(" ".join(line)): a = True break if h * a: @@ -652,26 +666,26 @@ class PQR(GenericMolFile): chain_ids = set() with open(dataset.file_name) as fh: for line in fh: - if line.startswith('REMARK'): + if line.startswith("REMARK"): continue match = prog.match(line.rstrip()) if match and match.groups()[5]: chain_ids.add(match.groups()[5]) dataset.metadata.chain_ids = list(chain_ids) except Exception as e: - log.error('Error finding chain_ids: %s', unicodify(e)) + log.error("Error finding chain_ids: %s", unicodify(e)) raise def set_peek(self, dataset): if not dataset.dataset.purged: atom_numbers = count_special_lines("^ATOM", dataset.file_name) hetatm_numbers = count_special_lines("^HETATM", dataset.file_name) - chain_ids = ','.join(dataset.metadata.chain_ids) if len(dataset.metadata.chain_ids) > 0 else 'None' + chain_ids = ",".join(dataset.metadata.chain_ids) if len(dataset.metadata.chain_ids) > 0 else "None" dataset.peek = get_file_peek(dataset.file_name) dataset.blurb = f"{atom_numbers} atoms and {hetatm_numbers} HET-atoms\nchain_ids: {str(chain_ids)}" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" class grd(Text): @@ -682,8 +696,8 @@ class grd(Text): dataset.peek = get_file_peek(dataset.file_name) dataset.blurb = "grids for docking" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" class grdtgz(Binary): @@ -691,70 +705,35 @@ class grdtgz(Binary): def set_peek(self, dataset): if not dataset.dataset.purged: - dataset.peek = 'binary data' + dataset.peek = "binary data" dataset.blurb = "compressed grids for docking" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" @build_sniff_from_prefix class InChI(Tabular): file_ext = "inchi" - column_names = ['InChI'] + column_names = ["InChI"] MetadataElement(name="columns", default=2, desc="Number of columns", readonly=True, visible=False) - MetadataElement(name="column_types", default=['str'], param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False) - MetadataElement(name="number_of_molecules", default=0, desc="Number of molecules", readonly=True, visible=True, optional=True, no_value=0) - - def set_meta(self, dataset, **kwd): - """ - Set the number of lines of data in dataset. - """ - dataset.metadata.number_of_molecules = self.count_data_lines(dataset) - - def set_peek(self, dataset): - if not dataset.dataset.purged: - if (dataset.metadata.number_of_molecules == 1): - dataset.blurb = "1 molecule" - else: - dataset.blurb = f"{dataset.metadata.number_of_molecules} molecules" - dataset.peek = get_file_peek(dataset.file_name) - else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' - - def sniff_prefix(self, file_prefix: FilePrefix): - """ - Try to guess if the file is a InChI file. - - >>> from galaxy.datatypes.sniff import get_test_fname - >>> fname = get_test_fname('drugbank_drugs.inchi') - >>> InChI().sniff(fname) - True - >>> fname = get_test_fname('drugbank_drugs.cml') - >>> InChI().sniff(fname) - False - """ - inchi_lines = iter_headers(file_prefix, sep=' ', count=10) - found_lines = False - for inchi in inchi_lines: - if not inchi[0].startswith('InChI='): - return False - found_lines = True - return found_lines - - -class SMILES(Tabular): - # It is hard or impossible to sniff a SMILES File. We can try to import the - # first SMILES and check if it is a molecule, but currently it is not - # possible to use external libraries in datatype definition files. - # Moreover it seems impossible to include OpenBabel as Python library - # because OpenBabel is GPL licensed. - file_ext = "smi" - column_names = ['SMILES', 'TITLE'] - MetadataElement(name="columns", default=2, desc="Number of columns", readonly=True, visible=False) - MetadataElement(name="column_types", default=['str', 'str'], param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False) - MetadataElement(name="number_of_molecules", default=0, desc="Number of molecules", readonly=True, visible=True, optional=True, no_value=0) + MetadataElement( + name="column_types", + default=["str"], + param=metadata.ColumnTypesParameter, + desc="Column types", + readonly=True, + visible=False, + ) + MetadataElement( + name="number_of_molecules", + default=0, + desc="Number of molecules", + readonly=True, + visible=True, + optional=True, + no_value=0, + ) def set_meta(self, dataset, **kwd): """ @@ -770,8 +749,73 @@ class SMILES(Tabular): dataset.blurb = f"{dataset.metadata.number_of_molecules} molecules" dataset.peek = get_file_peek(dataset.file_name) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" + + def sniff_prefix(self, file_prefix: FilePrefix): + """ + Try to guess if the file is a InChI file. + + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname('drugbank_drugs.inchi') + >>> InChI().sniff(fname) + True + >>> fname = get_test_fname('drugbank_drugs.cml') + >>> InChI().sniff(fname) + False + """ + inchi_lines = iter_headers(file_prefix, sep=" ", count=10) + found_lines = False + for inchi in inchi_lines: + if not inchi[0].startswith("InChI="): + return False + found_lines = True + return found_lines + + +class SMILES(Tabular): + # It is hard or impossible to sniff a SMILES File. We can try to import the + # first SMILES and check if it is a molecule, but currently it is not + # possible to use external libraries in datatype definition files. + # Moreover it seems impossible to include OpenBabel as Python library + # because OpenBabel is GPL licensed. + file_ext = "smi" + column_names = ["SMILES", "TITLE"] + MetadataElement(name="columns", default=2, desc="Number of columns", readonly=True, visible=False) + MetadataElement( + name="column_types", + default=["str", "str"], + param=metadata.ColumnTypesParameter, + desc="Column types", + readonly=True, + visible=False, + ) + MetadataElement( + name="number_of_molecules", + default=0, + desc="Number of molecules", + readonly=True, + visible=True, + optional=True, + no_value=0, + ) + + def set_meta(self, dataset, **kwd): + """ + Set the number of lines of data in dataset. + """ + dataset.metadata.number_of_molecules = self.count_data_lines(dataset) + + def set_peek(self, dataset): + if not dataset.dataset.purged: + if dataset.metadata.number_of_molecules == 1: + dataset.blurb = "1 molecule" + else: + dataset.blurb = f"{dataset.metadata.number_of_molecules} molecules" + dataset.peek = get_file_peek(dataset.file_name) + else: + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" @build_sniff_from_prefix @@ -780,25 +824,34 @@ class CML(GenericXml): Chemical Markup Language http://cml.sourceforge.net/ """ + file_ext = "cml" - MetadataElement(name="number_of_molecules", default=0, desc="Number of molecules", readonly=True, visible=True, optional=True, no_value=0) + MetadataElement( + name="number_of_molecules", + default=0, + desc="Number of molecules", + readonly=True, + visible=True, + optional=True, + no_value=0, + ) def set_meta(self, dataset, **kwd): """ Set the number of lines of data in dataset. """ - dataset.metadata.number_of_molecules = count_special_lines(r'^\s*>> CML().sniff(fname) True """ - for expected_string in ['', 'http://www.xml-cml.org/schema']: + for expected_string in ['', "http://www.xml-cml.org/schema"]: if expected_string not in file_prefix.contents_header: return False @@ -831,10 +884,10 @@ class CML(GenericXml): input_files = [ds.file_name for ds in input_datasets] chunk_size = None - if split_params['split_mode'] == 'number_of_parts': + if split_params["split_mode"] == "number_of_parts": raise Exception(f"Split mode \"{split_params['split_mode']}\" is currently not implemented for CML-files.") - elif split_params['split_mode'] == 'to_size': - chunk_size = int(split_params['split_size']) + elif split_params["split_mode"] == "to_size": + chunk_size = int(split_params["split_size"]) else: raise Exception(f"Unsupported split mode {split_params['split_mode']}") @@ -842,22 +895,24 @@ class CML(GenericXml): lines = [] with open(filename) as handle: for line in handle: - if line.lstrip().startswith('') or \ - line.lstrip().startswith('') + or line.lstrip().startswith('") + ): continue lines.append(line) - if line.lstrip().startswith(''): + if line.lstrip().startswith(""): yield lines lines = [] header_lines = ['\n', '\n'] - footer_line = ['\n'] + footer_line = ["\n"] def _write_part_cml_file(accumulated_lines): part_dir = subdir_generator_function() part_path = os.path.join(part_dir, os.path.basename(input_files[0])) - with open(part_path, 'w') as part_file: + with open(part_path, "w") as part_file: part_file.writelines(header_lines) part_file.writelines(accumulated_lines) part_file.writelines(footer_line) @@ -873,7 +928,7 @@ class CML(GenericXml): if cml_lines_accumulated: _write_part_cml_file(cml_lines_accumulated) except Exception as e: - log.error('Unable to split files: %s', unicodify(e)) + log.error("Unable to split files: %s", unicodify(e)) raise @staticmethod @@ -885,8 +940,7 @@ class CML(GenericXml): # For one file only, use base class method (move/copy) return Text.merge(split_files, output_file) if not split_files: - raise ValueError("Given no CML files, %r, to merge into %s" - % (split_files, output_file)) + raise ValueError("Given no CML files, %r, to merge into %s" % (split_files, output_file)) with open(output_file, "w") as out: for filename in split_files: with open(filename) as handle: @@ -904,9 +958,9 @@ class CML(GenericXml): molecule_found = False for line in handle.readlines(): # We found two required header lines, the next line should start with - if line.lstrip().startswith(''): + if line.lstrip().startswith(""): continue - if line.lstrip().startswith('>> GRO().sniff_prefix(fname) False """ - headers = get_headers(file_prefix, sep='\n', count=300) + headers = get_headers(file_prefix, sep="\n", count=300) try: int(headers[1][0]) # the second line should just be the number of atoms except ValueError: return False for line in headers[2:-1]: # skip the first, second and last lines - if not re.search(r'^[0-9 ]{5}[a-zA-Z0-9 ]{10}[0-9 ]{5}[0-9 -]{4}\.[0-9]{3}[0-9 -]{4}\.[0-9]{3}[0-9 -]{4}\.[0-9]{3}', line[0]): + if not re.search( + r"^[0-9 ]{5}[a-zA-Z0-9 ]{10}[0-9 ]{5}[0-9 -]{4}\.[0-9]{3}[0-9 -]{4}\.[0-9]{3}[0-9 -]{4}\.[0-9]{3}", + line[0], + ): return False return True def set_peek(self, dataset): if not dataset.dataset.purged: dataset.peek = get_file_peek(dataset.file_name) - atom_number = int(dataset.peek.split('\n')[1]) + atom_number = int(dataset.peek.split("\n")[1]) dataset.blurb = f"{atom_number} atoms" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" diff --git a/lib/galaxy/datatypes/mothur.py b/lib/galaxy/datatypes/mothur.py index 2c7cd917b31..f1ac6c353e6 100644 --- a/lib/galaxy/datatypes/mothur.py +++ b/lib/galaxy/datatypes/mothur.py @@ -11,7 +11,7 @@ from galaxy.datatypes.sniff import ( build_sniff_from_prefix, FilePrefix, get_headers, - iter_headers + iter_headers, ) from galaxy.datatypes.tabular import Tabular from galaxy.util import unicodify @@ -21,7 +21,7 @@ log = logging.getLogger(__name__) @build_sniff_from_prefix class Otu(Text): - file_ext = 'mothur.otu' + file_ext = "mothur.otu" MetadataElement(name="columns", default=0, desc="Number of columns", readonly=True, visible=True, no_value=0) MetadataElement(name="labels", default=[], desc="Label Names", readonly=True, visible=True, no_value=[]) MetadataElement(name="otulabels", default=[], desc="OTU Names", readonly=True, visible=True, no_value=[]) @@ -57,8 +57,8 @@ class Otu(Text): data_lines = 0 comment_lines = 0 - headers = iter_headers(dataset.file_name, sep='\t', count=-1) - first_line = get_headers(dataset.file_name, sep='\t', count=1) + headers = iter_headers(dataset.file_name, sep="\t", count=-1) + first_line = get_headers(dataset.file_name, sep="\t", count=1) if first_line: first_line = first_line[0] # set otulabels @@ -66,7 +66,7 @@ class Otu(Text): otulabel_names = first_line[2:] # set label names and number of lines for line in headers: - if len(line) >= 2 and not line[0].startswith('@'): + if len(line) >= 2 and not line[0].startswith("@"): data_lines += 1 ncols = max(ncols, len(line)) label_names.add(line[0]) @@ -90,10 +90,10 @@ class Otu(Text): >>> Otu().sniff( fname ) False """ - headers = iter_headers(file_prefix, sep='\t') + headers = iter_headers(file_prefix, sep="\t") count = 0 for line in headers: - if not line[0].startswith('@'): + if not line[0].startswith("@"): if len(line) < 2: return False if count >= 1: @@ -111,7 +111,7 @@ class Otu(Text): class Sabund(Otu): - file_ext = 'mothur.sabund' + file_ext = "mothur.sabund" def __init__(self, **kwd): """ @@ -135,10 +135,10 @@ class Sabund(Otu): >>> Sabund().sniff( fname ) False """ - headers = iter_headers(file_prefix, sep='\t') + headers = iter_headers(file_prefix, sep="\t") count = 0 for line in headers: - if not line[0].startswith('@'): + if not line[0].startswith("@"): if len(line) < 2: return False try: @@ -157,7 +157,7 @@ class Sabund(Otu): class GroupAbund(Otu): - file_ext = 'mothur.shared' + file_ext = "mothur.shared" MetadataElement(name="groups", default=[], desc="Group Names", readonly=True, visible=True, no_value=[]) def __init__(self, **kwd): @@ -177,9 +177,9 @@ class GroupAbund(Otu): comment_lines = 0 ncols = 0 - headers = iter_headers(dataset.file_name, sep='\t', count=-1) + headers = iter_headers(dataset.file_name, sep="\t", count=-1) for line in headers: - if line[0] == 'label' and line[1] == 'Group': + if line[0] == "label" and line[1] == "Group": skip = 1 comment_lines += 1 else: @@ -211,13 +211,13 @@ class GroupAbund(Otu): >>> GroupAbund().sniff( fname ) False """ - headers = iter_headers(file_prefix, sep='\t') + headers = iter_headers(file_prefix, sep="\t") count = 0 for line in headers: - if not line[0].startswith('@'): + if not line[0].startswith("@"): if len(line) < 3: return False - if count > 0 or line[0] != 'label': + if count > 0 or line[0] != "label": try: check = int(line[2]) if check + 3 != len(line): @@ -237,12 +237,12 @@ class GroupAbund(Otu): @build_sniff_from_prefix class SecondaryStructureMap(Tabular): - file_ext = 'mothur.map' + file_ext = "mothur.map" def __init__(self, **kwd): """Initialize secondary structure map datatype""" super().__init__(**kwd) - self.column_names = ['Map'] + self.column_names = ["Map"] def sniff_prefix(self, file_prefix: FilePrefix): """ @@ -259,7 +259,7 @@ class SecondaryStructureMap(Tabular): >>> SecondaryStructureMap().sniff( fname ) False """ - headers = iter_headers(file_prefix, sep='\t') + headers = iter_headers(file_prefix, sep="\t") line_num = 0 rowidxmap = {} for line in headers: @@ -281,13 +281,13 @@ class SecondaryStructureMap(Tabular): class AlignCheck(Tabular): - file_ext = 'mothur.align.check' + file_ext = "mothur.align.check" def __init__(self, **kwd): """Initialize AlignCheck datatype""" super().__init__(**kwd) - self.column_names = ['name', 'pound', 'dash', 'plus', 'equal', 'loop', 'tilde', 'total'] - self.column_types = ['str', 'int', 'int', 'int', 'int', 'int', 'int', 'int'] + self.column_names = ["name", "pound", "dash", "plus", "equal", "loop", "tilde", "total"] + self.column_types = ["str", "int", "int", "int", "int", "int", "int", "int"] self.comment_lines = 1 def set_meta(self, dataset, overwrite=True, **kwd): @@ -305,21 +305,44 @@ class AlignReport(Tabular): QueryName QueryLength TemplateName TemplateLength SearchMethod SearchScore AlignmentMethod QueryStart QueryEnd TemplateStart TemplateEnd PairwiseAlignmentLength GapsInQuery GapsInTemplate LongestInsert SimBtwnQuery&Template AY457915 501 82283 1525 kmer 89.07 needleman 5 501 1 499 499 2 0 0 97.6 """ - file_ext = 'mothur.align.report' + + file_ext = "mothur.align.report" def __init__(self, **kwd): """Initialize AlignCheck datatype""" super().__init__(**kwd) - self.column_names = ['QueryName', 'QueryLength', 'TemplateName', 'TemplateLength', 'SearchMethod', 'SearchScore', - 'AlignmentMethod', 'QueryStart', 'QueryEnd', 'TemplateStart', 'TemplateEnd', - 'PairwiseAlignmentLength', 'GapsInQuery', 'GapsInTemplate', 'LongestInsert', 'SimBtwnQuery&Template' - ] + self.column_names = [ + "QueryName", + "QueryLength", + "TemplateName", + "TemplateLength", + "SearchMethod", + "SearchScore", + "AlignmentMethod", + "QueryStart", + "QueryEnd", + "TemplateStart", + "TemplateEnd", + "PairwiseAlignmentLength", + "GapsInQuery", + "GapsInTemplate", + "LongestInsert", + "SimBtwnQuery&Template", + ] class DistanceMatrix(Text): - file_ext = 'mothur.dist' + file_ext = "mothur.dist" - MetadataElement(name="sequence_count", default=0, desc="Number of sequences", readonly=True, visible=True, optional=True, no_value='?') + MetadataElement( + name="sequence_count", + default=0, + desc="Number of sequences", + readonly=True, + visible=True, + optional=True, + no_value="?", + ) def init_meta(self, dataset, copy_from=None): super().init_meta(dataset, copy_from=copy_from) @@ -327,11 +350,11 @@ class DistanceMatrix(Text): def set_meta(self, dataset, overwrite=True, skip=0, **kwd): super().set_meta(dataset, overwrite=overwrite, skip=skip, **kwd) - headers = iter_headers(dataset.file_name, sep='\t') + headers = iter_headers(dataset.file_name, sep="\t") for line in headers: - if not line[0].startswith('@'): + if not line[0].startswith("@"): try: - dataset.metadata.sequence_count = int(''.join(line)) # seq count sometimes preceded by tab + dataset.metadata.sequence_count = int("".join(line)) # seq count sometimes preceded by tab break except Exception as e: if not isinstance(self, PairwiseDistanceMatrix): @@ -340,7 +363,7 @@ class DistanceMatrix(Text): @build_sniff_from_prefix class LowerTriangleDistanceMatrix(DistanceMatrix): - file_ext = 'mothur.lower.dist' + file_ext = "mothur.lower.dist" def __init__(self, **kwd): """Initialize secondary structure map datatype""" @@ -371,17 +394,17 @@ class LowerTriangleDistanceMatrix(DistanceMatrix): False """ numlines = 300 - headers = iter_headers(file_prefix, sep='\t', count=numlines) + headers = iter_headers(file_prefix, sep="\t", count=numlines) line_num = 0 for line in headers: - if not line[0].startswith('@'): + if not line[0].startswith("@"): # first line should contain the number of sequences in the file if line_num == 0: if len(line) > 2: return False else: try: - sequence_count = int(''.join(line)) + sequence_count = int("".join(line)) assert sequence_count > 0 except ValueError: return False @@ -406,7 +429,7 @@ class LowerTriangleDistanceMatrix(DistanceMatrix): @build_sniff_from_prefix class SquareDistanceMatrix(DistanceMatrix): - file_ext = 'mothur.square.dist' + file_ext = "mothur.square.dist" def __init__(self, **kwd): super().__init__(**kwd) @@ -435,16 +458,16 @@ class SquareDistanceMatrix(DistanceMatrix): False """ numlines = 300 - headers = iter_headers(file_prefix, sep='\t', count=numlines) + headers = iter_headers(file_prefix, sep="\t", count=numlines) line_num = 0 for line in headers: - if not line[0].startswith('@'): + if not line[0].startswith("@"): if line_num == 0: if len(line) > 2: return False else: try: - sequence_count = int(''.join(line)) + sequence_count = int("".join(line)) assert sequence_count > 0 except ValueError: return False @@ -469,13 +492,13 @@ class SquareDistanceMatrix(DistanceMatrix): @build_sniff_from_prefix class PairwiseDistanceMatrix(DistanceMatrix, Tabular): - file_ext = 'mothur.pair.dist' + file_ext = "mothur.pair.dist" def __init__(self, **kwd): """Initialize secondary structure map datatype""" super().__init__(**kwd) - self.column_names = ['Sequence', 'Sequence', 'Distance'] - self.column_types = ['str', 'str', 'float'] + self.column_names = ["Sequence", "Sequence", "Distance"] + self.column_types = ["str", "str", "float"] def set_meta(self, dataset, overwrite=True, skip=None, **kwd): super().set_meta(dataset, overwrite=overwrite, skip=skip, **kwd) @@ -493,11 +516,11 @@ class PairwiseDistanceMatrix(DistanceMatrix, Tabular): >>> PairwiseDistanceMatrix().sniff( fname ) False """ - headers = iter_headers(file_prefix, sep='\t') + headers = iter_headers(file_prefix, sep="\t") count = 0 names = [False, False] for line in headers: - if line[0].startswith('@'): + if line[0].startswith("@"): continue if len(line) != 3: return False @@ -530,7 +553,7 @@ class PairwiseDistanceMatrix(DistanceMatrix, Tabular): class Names(Tabular): - file_ext = 'mothur.names' + file_ext = "mothur.names" def __init__(self, **kwd): """ @@ -538,22 +561,22 @@ class Names(Tabular): Name file shows the relationship between a representative sequence(col 1) and the sequences(comma-separated) it represents(col 2) """ super().__init__(**kwd) - self.column_names = ['name', 'representatives'] + self.column_names = ["name", "representatives"] self.columns = 2 class Summary(Tabular): - file_ext = 'mothur.summary' + file_ext = "mothur.summary" def __init__(self, **kwd): """summarizes the quality of sequences in an unaligned or aligned fasta-formatted sequence file""" super().__init__(**kwd) - self.column_names = ['seqname', 'start', 'end', 'nbases', 'ambigs', 'polymer'] + self.column_names = ["seqname", "start", "end", "nbases", "ambigs", "polymer"] self.columns = 6 class Group(Tabular): - file_ext = 'mothur.groups' + file_ext = "mothur.groups" MetadataElement(name="groups", default=[], desc="Group Names", readonly=True, visible=True, no_value=[]) def __init__(self, **kwd): @@ -562,14 +585,14 @@ class Group(Tabular): Group file assigns sequence (col 1) to a group (col 2) """ super().__init__(**kwd) - self.column_names = ['name', 'group'] + self.column_names = ["name", "group"] self.columns = 2 def set_meta(self, dataset, overwrite=True, skip=None, max_data_lines=None, **kwd): super().set_meta(dataset, overwrite, skip, max_data_lines) group_names = set() - headers = iter_headers(dataset.file_name, sep='\t', count=-1) + headers = iter_headers(dataset.file_name, sep="\t", count=-1) for line in headers: if len(line) > 1: group_names.add(line[1]) @@ -577,18 +600,18 @@ class Group(Tabular): class AccNos(Tabular): - file_ext = 'mothur.accnos' + file_ext = "mothur.accnos" def __init__(self, **kwd): """A list of names""" super().__init__(**kwd) - self.column_names = ['name'] + self.column_names = ["name"] self.columns = 1 @build_sniff_from_prefix class Oligos(Text): - file_ext = 'mothur.oligos' + file_ext = "mothur.oligos" def sniff_prefix(self, file_prefix: FilePrefix): """ @@ -603,14 +626,14 @@ class Oligos(Text): >>> Oligos().sniff( fname ) False """ - headers = iter_headers(file_prefix, sep='\t') + headers = iter_headers(file_prefix, sep="\t") count = 0 for line in headers: - if not line[0].startswith('@') and not line[0].startswith('#'): - if len(line) == 2 and line[0] in ['forward', 'reverse']: + if not line[0].startswith("@") and not line[0].startswith("#"): + if len(line) == 2 and line[0] in ["forward", "reverse"]: count += 1 continue - elif len(line) == 3 and line[0] == 'barcode': + elif len(line) == 3 and line[0] == "barcode": count += 1 continue else: @@ -623,13 +646,13 @@ class Oligos(Text): @build_sniff_from_prefix class Frequency(Tabular): - file_ext = 'mothur.freq' + file_ext = "mothur.freq" def __init__(self, **kwd): """A list of names""" super().__init__(**kwd) - self.column_names = ['position', 'frequency'] - self.column_types = ['int', 'float'] + self.column_names = ["position", "frequency"] + self.column_types = ["int", "float"] def sniff_prefix(self, file_prefix: FilePrefix): """ @@ -655,13 +678,13 @@ class Frequency(Tabular): >>> Frequency().sniff( fname ) False """ - headers = iter_headers(file_prefix, sep='\t') + headers = iter_headers(file_prefix, sep="\t") count = 0 for line in headers: - if not line[0].startswith('@'): + if not line[0].startswith("@"): # first line should be # if count == 0: - if not line[0].startswith('#') or len(line) != 1: + if not line[0].startswith("#") or len(line) != 1: return False else: @@ -672,7 +695,7 @@ class Frequency(Tabular): int(line[0]) float(line[1]) - if line[1].find('.') == -1: + if line[1].find(".") == -1: return False except Exception: return False @@ -686,15 +709,29 @@ class Frequency(Tabular): @build_sniff_from_prefix class Quantile(Tabular): - file_ext = 'mothur.quan' - MetadataElement(name="filtered", default=False, no_value=False, optional=True, desc="Quantiles calculated using a mask", readonly=True) - MetadataElement(name="masked", default=False, no_value=False, optional=True, desc="Quantiles calculated using a frequency filter", readonly=True) + file_ext = "mothur.quan" + MetadataElement( + name="filtered", + default=False, + no_value=False, + optional=True, + desc="Quantiles calculated using a mask", + readonly=True, + ) + MetadataElement( + name="masked", + default=False, + no_value=False, + optional=True, + desc="Quantiles calculated using a frequency filter", + readonly=True, + ) def __init__(self, **kwd): """Quantiles for chimera analysis""" super().__init__(**kwd) - self.column_names = ['num', 'ten', 'twentyfive', 'fifty', 'seventyfive', 'ninetyfive', 'ninetynine'] - self.column_types = ['int', 'float', 'float', 'float', 'float', 'float', 'float'] + self.column_names = ["num", "ten", "twentyfive", "fifty", "seventyfive", "ninetyfive", "ninetynine"] + self.column_types = ["int", "float", "float", "float", "float", "float", "float"] def sniff_prefix(self, file_prefix: FilePrefix): """ @@ -715,10 +752,10 @@ class Quantile(Tabular): >>> Quantile().sniff( fname ) False """ - headers = iter_headers(file_prefix, sep='\t') + headers = iter_headers(file_prefix, sep="\t") count = 0 for line in headers: - if not line[0].startswith('@') and not line[0].startswith('#'): + if not line[0].startswith("@") and not line[0].startswith("#"): if len(line) != 7: return False try: @@ -740,7 +777,7 @@ class Quantile(Tabular): @build_sniff_from_prefix class LaneMask(Text): - file_ext = 'mothur.filter' + file_ext = "mothur.filter" def sniff_prefix(self, file_prefix: FilePrefix): """ @@ -754,7 +791,7 @@ class LaneMask(Text): >>> LaneMask().sniff( fname ) False """ - headers = get_headers(file_prefix, sep='\t', count=2) + headers = get_headers(file_prefix, sep="\t", count=2) if len(headers) != 1 or len(headers[0]) != 1: return False @@ -762,7 +799,7 @@ class LaneMask(Text): # these filter files should be relatively big return False - if not re.match('^[01]+$', headers[0][0]): + if not re.match("^[01]+$", headers[0][0]): return False return True @@ -770,7 +807,7 @@ class LaneMask(Text): class CountTable(Tabular): MetadataElement(name="groups", default=[], desc="Group Names", readonly=True, visible=True, no_value=[]) - file_ext = 'mothur.count_table' + file_ext = "mothur.count_table" def __init__(self, **kwd): """ @@ -790,14 +827,14 @@ class CountTable(Tabular): U68647 1 0 1 """ super().__init__(**kwd) - self.column_names = ['name', 'total'] + self.column_names = ["name", "total"] def set_meta(self, dataset, overwrite=True, skip=1, max_data_lines=None, **kwd): super().set_meta(dataset, overwrite=overwrite, **kwd) - headers = get_headers(dataset.file_name, sep='\t', count=1) + headers = get_headers(dataset.file_name, sep="\t", count=1) colnames = headers[0] - dataset.metadata.column_types = ['str'] + (['int'] * (len(headers[0]) - 1)) + dataset.metadata.column_types = ["str"] + (["int"] * (len(headers[0]) - 1)) if len(colnames) > 1: dataset.metadata.columns = len(colnames) if len(colnames) > 2: @@ -810,11 +847,11 @@ class CountTable(Tabular): @build_sniff_from_prefix class RefTaxonomy(Tabular): - file_ext = 'mothur.ref.taxonomy' + file_ext = "mothur.ref.taxonomy" def __init__(self, **kwd): super().__init__(**kwd) - self.column_names = ['name', 'taxonomy'] + self.column_names = ["name", "taxonomy"] def sniff_prefix(self, file_prefix: FilePrefix): """ @@ -851,17 +888,17 @@ class RefTaxonomy(Tabular): >>> RefTaxonomy().sniff( fname ) False """ - headers = iter_headers(file_prefix, sep='\t', count=300) + headers = iter_headers(file_prefix, sep="\t", count=300) count = 0 - pat_prog = re.compile('^([^ \t\n\r\x0c\x0b;]+([(]\\d+[)])?(;[^ \t\n\r\x0c\x0b;]+([(]\\d+[)])?)*(;)?)$') + pat_prog = re.compile("^([^ \t\n\r\x0c\x0b;]+([(]\\d+[)])?(;[^ \t\n\r\x0c\x0b;]+([(]\\d+[)])?)*(;)?)$") found_semicolons = False for line in headers: - if not line[0].startswith('@') and not line[0].startswith('#'): + if not line[0].startswith("@") and not line[0].startswith("#"): if not (2 <= len(line) <= 3): return False if not pat_prog.match(line[1]): return False - if not found_semicolons and line[1].find(';') > -1: + if not found_semicolons and line[1].find(";") > -1: found_semicolons = True if len(line) == 3: try: @@ -878,26 +915,26 @@ class RefTaxonomy(Tabular): class ConsensusTaxonomy(Tabular): - file_ext = 'mothur.cons.taxonomy' + file_ext = "mothur.cons.taxonomy" def __init__(self, **kwd): """A list of names""" super().__init__(**kwd) - self.column_names = ['OTU', 'count', 'taxonomy'] + self.column_names = ["OTU", "count", "taxonomy"] class TaxonomySummary(Tabular): - file_ext = 'mothur.tax.summary' + file_ext = "mothur.tax.summary" def __init__(self, **kwd): """A Summary of taxon classification""" super().__init__(**kwd) - self.column_names = ['taxlevel', 'rankID', 'taxon', 'daughterlevels', 'total'] + self.column_names = ["taxlevel", "rankID", "taxon", "daughterlevels", "total"] @build_sniff_from_prefix class Axes(Tabular): - file_ext = 'mothur.axes' + file_ext = "mothur.axes" def __init__(self, **kwd): """Initialize axes datatype""" @@ -930,7 +967,7 @@ class Axes(Tabular): >>> Axes().sniff( fname ) False """ - headers = iter_headers(file_prefix, sep='\t') + headers = iter_headers(file_prefix, sep="\t") count = 0 col_cnt = None all_integers = True @@ -984,10 +1021,15 @@ class SffFlow(Tabular): GQY1XT001CF5YW 88 1.02 0.02 1.01 0.04 0.06 1.02 0.03 ... """ - file_ext = 'mothur.sff.flow' - MetadataElement(name="flow_values", default="", no_value="", optional=True, desc="Total number of flow values", readonly=True) - MetadataElement(name="flow_order", default="TACG", no_value="TACG", desc="Total number of flow values", readonly=False) + file_ext = "mothur.sff.flow" + + MetadataElement( + name="flow_values", default="", no_value="", optional=True, desc="Total number of flow values", readonly=True + ) + MetadataElement( + name="flow_order", default="TACG", no_value="TACG", desc="Total number of flow values", readonly=False + ) def __init__(self, **kwd): super().__init__(**kwd) @@ -995,7 +1037,7 @@ class SffFlow(Tabular): def set_meta(self, dataset, overwrite=True, skip=1, max_data_lines=None, **kwd): super().set_meta(dataset, overwrite, 1, max_data_lines) - headers = get_headers(dataset.file_name, sep='\t', count=1) + headers = get_headers(dataset.file_name, sep="\t", count=1) try: flow_values = int(headers[0][0]) dataset.metadata.flow_values = flow_values @@ -1010,20 +1052,21 @@ class SffFlow(Tabular): out = '' # Generate column header - out += '' - out += '' - out += '' + out += "" + out += "" + out += "" for i in range(3, dataset.metadata.columns + 1): base = dataset.metadata.flow_order[(i + 1) % 4] - out += '' % (i - 2, base) - out += '' + out += "" % (i - 2, base) + out += "" out += self.make_html_peek_rows(dataset, skipchars=skipchars) - out += '
              1. Name2. Flows
              1. Name2. Flows%d. %s
              %d. %s
              ' + out += "" except Exception as exc: out = f"Can't create peek: {unicodify(exc)}" return out -if __name__ == '__main__': +if __name__ == "__main__": import doctest + doctest.testmod(sys.modules[__name__]) diff --git a/lib/galaxy/datatypes/msa.py b/lib/galaxy/datatypes/msa.py index e187d663f31..c5c8feab069 100644 --- a/lib/galaxy/datatypes/msa.py +++ b/lib/galaxy/datatypes/msa.py @@ -4,7 +4,10 @@ import os import re from galaxy.datatypes.binary import Binary -from galaxy.datatypes.data import get_file_peek, Text +from galaxy.datatypes.data import ( + get_file_peek, + Text, +) from galaxy.datatypes.metadata import MetadataElement from galaxy.datatypes.sniff import ( build_sniff_from_prefix, @@ -18,18 +21,32 @@ from galaxy.util import ( log = logging.getLogger(__name__) -STOCKHOLM_SEARCH_PATTERN = re.compile(r'#\s+STOCKHOLM\s+1\.0') +STOCKHOLM_SEARCH_PATTERN = re.compile(r"#\s+STOCKHOLM\s+1\.0") @build_sniff_from_prefix class InfernalCM(Text): file_ext = "cm" - MetadataElement(name="number_of_models", default=0, desc="Number of covariance models", - readonly=True, visible=True, optional=True, no_value=0) + MetadataElement( + name="number_of_models", + default=0, + desc="Number of covariance models", + readonly=True, + visible=True, + optional=True, + no_value=0, + ) - MetadataElement(name="cm_version", default="1/a", desc="Infernal Covariance Model version", - readonly=True, visible=True, optional=True, no_value=0) + MetadataElement( + name="cm_version", + default="1/a", + desc="Infernal Covariance Model version", + readonly=True, + visible=True, + optional=True, + no_value=0, + ) def set_peek(self, dataset): if not dataset.dataset.purged: @@ -40,8 +57,8 @@ class InfernalCM(Text): dataset.blurb = f"{dataset.metadata.number_of_models} models" dataset.peek = get_file_peek(dataset.file_name) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disc' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disc" def sniff_prefix(self, file_prefix: FilePrefix): """ @@ -59,11 +76,11 @@ class InfernalCM(Text): """ Set the number of models and the version of CM file in dataset. """ - dataset.metadata.number_of_models = generic_util.count_special_lines('^INFERNAL', dataset.file_name) + dataset.metadata.number_of_models = generic_util.count_special_lines("^INFERNAL", dataset.file_name) with open(dataset.file_name) as f: first_line = f.readline() if first_line.startswith("INFERNAL"): - dataset.metadata.cm_version = (first_line.split()[0]).replace('INFERNAL', '') + dataset.metadata.cm_version = (first_line.split()[0]).replace("INFERNAL", "") @build_sniff_from_prefix @@ -76,8 +93,8 @@ class Hmmer(Text): dataset.peek = get_file_peek(dataset.file_name) dataset.blurb = "HMMER Database" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disc' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disc" def display_peek(self, dataset): try: @@ -95,9 +112,8 @@ class Hmmer2(Hmmer): file_ext = "hmm2" def sniff_prefix(self, file_prefix: FilePrefix): - """HMMER2 files start with HMMER2.0 - """ - return file_prefix.startswith('HMMER2.0') + """HMMER2 files start with HMMER2.0""" + return file_prefix.startswith("HMMER2.0") class Hmmer3(Hmmer): @@ -105,15 +121,15 @@ class Hmmer3(Hmmer): file_ext = "hmm3" def sniff_prefix(self, file_prefix: FilePrefix): - """HMMER3 files start with HMMER3/f - """ - return file_prefix.startswith('HMMER3/f') + """HMMER3 files start with HMMER3/f""" + return file_prefix.startswith("HMMER3/f") class HmmerPress(Binary): """Class for hmmpress database files.""" - file_ext = 'hmmpress' - composite_type = 'basic' + + file_ext = "hmmpress" + composite_type = "basic" def set_peek(self, dataset): """Set the peek and blurb text.""" @@ -121,8 +137,8 @@ class HmmerPress(Binary): dataset.peek = "HMMER Binary database" dataset.blurb = "HMMER Binary database" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): """Create HTML content, used for displaying peek.""" @@ -134,13 +150,13 @@ class HmmerPress(Binary): def __init__(self, **kwd): super().__init__(**kwd) # Binary model - self.add_composite_file('model.hmm.h3m', is_binary=True) + self.add_composite_file("model.hmm.h3m", is_binary=True) # SSI index for binary model - self.add_composite_file('model.hmm.h3i', is_binary=True) + self.add_composite_file("model.hmm.h3i", is_binary=True) # Profiles (MSV part) - self.add_composite_file('model.hmm.h3f', is_binary=True) + self.add_composite_file("model.hmm.h3f", is_binary=True) # Profiles (remained) - self.add_composite_file('model.hmm.h3p', is_binary=True) + self.add_composite_file("model.hmm.h3p", is_binary=True) @build_sniff_from_prefix @@ -149,18 +165,26 @@ class Stockholm_1_0(Text): edam_format = "format_1961" file_ext = "stockholm" - MetadataElement(name="number_of_models", default=0, desc="Number of multiple alignments", readonly=True, visible=True, optional=True, no_value=0) + MetadataElement( + name="number_of_models", + default=0, + desc="Number of multiple alignments", + readonly=True, + visible=True, + optional=True, + no_value=0, + ) def set_peek(self, dataset): if not dataset.dataset.purged: - if (dataset.metadata.number_of_models == 1): + if dataset.metadata.number_of_models == 1: dataset.blurb = "1 alignment" else: dataset.blurb = f"{dataset.metadata.number_of_models} alignments" dataset.peek = get_file_peek(dataset.file_name) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disc' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disc" def sniff_prefix(self, file_prefix: FilePrefix): return file_prefix.search(STOCKHOLM_SEARCH_PATTERN) @@ -170,7 +194,9 @@ class Stockholm_1_0(Text): Set the number of models in dataset. """ - dataset.metadata.number_of_models = generic_util.count_special_lines('^#[[:space:]+]STOCKHOLM[[:space:]+]1.0', dataset.file_name) + dataset.metadata.number_of_models = generic_util.count_special_lines( + "^#[[:space:]+]STOCKHOLM[[:space:]+]1.0", dataset.file_name + ) @classmethod def split(cls, input_datasets, subdir_generator_function, split_params): @@ -186,10 +212,12 @@ class Stockholm_1_0(Text): input_files = [ds.file_name for ds in input_datasets] chunk_size = None - if split_params['split_mode'] == 'number_of_parts': - raise Exception(f"Split mode \"{split_params['split_mode']}\" is currently not implemented for STOCKHOLM-files.") - elif split_params['split_mode'] == 'to_size': - chunk_size = int(split_params['split_size']) + if split_params["split_mode"] == "number_of_parts": + raise Exception( + f"Split mode \"{split_params['split_mode']}\" is currently not implemented for STOCKHOLM-files." + ) + elif split_params["split_mode"] == "to_size": + chunk_size = int(split_params["split_size"]) else: raise Exception(f"Unsupported split mode {split_params['split_mode']}") @@ -198,14 +226,14 @@ class Stockholm_1_0(Text): with open(filename) as handle: for line in handle: lines.append(line) - if line.strip() == '//': + if line.strip() == "//": yield lines lines = [] def _write_part_stockholm_file(accumulated_lines): part_dir = subdir_generator_function() part_path = os.path.join(part_dir, os.path.basename(input_files[0])) - with open(part_path, 'w') as part_file: + with open(part_path, "w") as part_file: part_file.writelines(accumulated_lines) try: @@ -220,7 +248,7 @@ class Stockholm_1_0(Text): if stockholm_lines_accumulated: _write_part_stockholm_file(stockholm_lines_accumulated) except Exception as e: - log.error('Unable to split files: %s', unicodify(e)) + log.error("Unable to split files: %s", unicodify(e)) raise @@ -228,24 +256,34 @@ class Stockholm_1_0(Text): class MauveXmfa(Text): file_ext = "xmfa" - MetadataElement(name="number_of_models", default=0, desc="Number of alignmened sequences", readonly=True, visible=True, optional=True, no_value=0) + MetadataElement( + name="number_of_models", + default=0, + desc="Number of alignmened sequences", + readonly=True, + visible=True, + optional=True, + no_value=0, + ) def set_peek(self, dataset): if not dataset.dataset.purged: - if (dataset.metadata.number_of_models == 1): + if dataset.metadata.number_of_models == 1: dataset.blurb = "1 alignment" else: dataset.blurb = f"{dataset.metadata.number_of_models} alignments" dataset.peek = get_file_peek(dataset.file_name) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disc' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disc" def sniff_prefix(self, file_prefix: FilePrefix): - return file_prefix.startswith('#FormatVersion Mauve1') + return file_prefix.startswith("#FormatVersion Mauve1") def set_meta(self, dataset, **kwd): - dataset.metadata.number_of_models = generic_util.count_special_lines('^#Sequence([[:digit:]]+)Entry', dataset.file_name) + dataset.metadata.number_of_models = generic_util.count_special_lines( + "^#Sequence([[:digit:]]+)Entry", dataset.file_name + ) class Msf(Text): @@ -253,6 +291,7 @@ class Msf(Text): Multiple sequence alignment format produced by the Accelrys GCG suite and other programs. """ + edam_data = "data_0863" edam_format = "format_1947" - file_ext = 'msf' + file_ext = "msf" diff --git a/lib/galaxy/datatypes/neo4j.py b/lib/galaxy/datatypes/neo4j.py index c4191706aec..c2706a8917a 100644 --- a/lib/galaxy/datatypes/neo4j.py +++ b/lib/galaxy/datatypes/neo4j.py @@ -26,30 +26,30 @@ class Neo4j(Html): """ # self.regenerate_primary_file(dataset) rval = [ - 'Files for Composite Dataset (%s)

              \ - This composite dataset is composed of the following files:

                ' % ( - self.file_ext)] + "Files for Composite Dataset (%s)

                \ + This composite dataset is composed of the following files:

                  " + % (self.file_ext) + ] for composite_name, composite_file in self.get_composite_files(dataset=dataset).items(): - opt_text = '' + opt_text = "" if composite_file.optional: - opt_text = ' (optional)' - rval.append('
                • %s%s' % - (composite_name, composite_name, opt_text)) - rval.append('
                ') + opt_text = " (optional)" + rval.append('
              • %s%s' % (composite_name, composite_name, opt_text)) + rval.append("
              ") return "\n".join(rval) def get_mime(self): """Returns the mime type of the datatype""" - return 'text/html' + return "text/html" def set_peek(self, dataset): """Set the peek and blurb text""" if not dataset.dataset.purged: - dataset.peek = 'Neo4j database (multiple files)' - dataset.blurb = 'Neo4j database (multiple files)' + dataset.peek = "Neo4j database (multiple files)" + dataset.blurb = "Neo4j database (multiple files)" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): """Create HTML content, used for displaying peek.""" @@ -61,80 +61,80 @@ class Neo4j(Html): class Neo4jDB(Neo4j, Data): """Class for neo4jDB database files.""" - file_ext = 'neostore' - composite_type = 'auto_primary_file' + + file_ext = "neostore" + composite_type = "auto_primary_file" def __init__(self, **kwd): Data.__init__(self, **kwd) - self.add_composite_file('neostore', is_binary=True) - self.add_composite_file('neostore.id', is_binary=True) - self.add_composite_file('neostore.counts.db.a', optional=True, is_binary=True) - self.add_composite_file('neostore.counts.db.b', optional=True, is_binary=True) - self.add_composite_file('neostore.labeltokenstore.db', is_binary=True) - self.add_composite_file( - 'neostore.labeltokenstore.db.id', is_binary=True) - self.add_composite_file( - 'neostore.labeltokenstore.db.names', is_binary=True) - self.add_composite_file( - 'neostore.labeltokenstore.db.names.id', is_binary=True) - self.add_composite_file('neostore.nodestore.db', is_binary=True) - self.add_composite_file('neostore.nodestore.db.id', is_binary=True) - self.add_composite_file('neostore.nodestore.db.labels', is_binary=True) - self.add_composite_file( - 'neostore.nodestore.db.labels.id', is_binary=True) + self.add_composite_file("neostore", is_binary=True) + self.add_composite_file("neostore.id", is_binary=True) + self.add_composite_file("neostore.counts.db.a", optional=True, is_binary=True) + self.add_composite_file("neostore.counts.db.b", optional=True, is_binary=True) + self.add_composite_file("neostore.labeltokenstore.db", is_binary=True) + self.add_composite_file("neostore.labeltokenstore.db.id", is_binary=True) + self.add_composite_file("neostore.labeltokenstore.db.names", is_binary=True) + self.add_composite_file("neostore.labeltokenstore.db.names.id", is_binary=True) + self.add_composite_file("neostore.nodestore.db", is_binary=True) + self.add_composite_file("neostore.nodestore.db.id", is_binary=True) + self.add_composite_file("neostore.nodestore.db.labels", is_binary=True) + self.add_composite_file("neostore.nodestore.db.labels.id", is_binary=True) - self.add_composite_file('neostore.propertystore.db', is_binary=True) - self.add_composite_file('neostore.propertystore.db.id', is_binary=True) - self.add_composite_file( - 'neostore.propertystore.db.arrays', is_binary=True) - self.add_composite_file( - 'neostore.propertystore.db.arrays.id', is_binary=True) - self.add_composite_file( - 'neostore.propertystore.db.index', is_binary=True) - self.add_composite_file( - 'neostore.propertystore.db.index.id', is_binary=True) - self.add_composite_file( - 'neostore.propertystore.db.index.keys', is_binary=True) - self.add_composite_file( - 'neostore.propertystore.db.index.keys.id', is_binary=True) - self.add_composite_file( - 'neostore.propertystore.db.strings', is_binary=True) - self.add_composite_file( - 'neostore.propertystore.db.strings.id', is_binary=True) + self.add_composite_file("neostore.propertystore.db", is_binary=True) + self.add_composite_file("neostore.propertystore.db.id", is_binary=True) + self.add_composite_file("neostore.propertystore.db.arrays", is_binary=True) + self.add_composite_file("neostore.propertystore.db.arrays.id", is_binary=True) + self.add_composite_file("neostore.propertystore.db.index", is_binary=True) + self.add_composite_file("neostore.propertystore.db.index.id", is_binary=True) + self.add_composite_file("neostore.propertystore.db.index.keys", is_binary=True) + self.add_composite_file("neostore.propertystore.db.index.keys.id", is_binary=True) + self.add_composite_file("neostore.propertystore.db.strings", is_binary=True) + self.add_composite_file("neostore.propertystore.db.strings.id", is_binary=True) - self.add_composite_file( - 'neostore.relationshipgroupstore.db', is_binary=True) - self.add_composite_file( - 'neostore.relationshipgroupstore.db.id', is_binary=True) - self.add_composite_file( - 'neostore.relationshipstore.db', is_binary=True) - self.add_composite_file( - 'neostore.relationshipstore.db.id', is_binary=True) - self.add_composite_file( - 'neostore.relationshiptypestore.db.names', is_binary=True) - self.add_composite_file( - 'neostore.relationshiptypestore.db.names.id', is_binary=True) - self.add_composite_file('neostore.schemastore.db', is_binary=True) - self.add_composite_file('neostore.schemastore.db.id', is_binary=True) - self.add_composite_file('neostore.transaction.db.0', is_binary=True) + self.add_composite_file("neostore.relationshipgroupstore.db", is_binary=True) + self.add_composite_file("neostore.relationshipgroupstore.db.id", is_binary=True) + self.add_composite_file("neostore.relationshipstore.db", is_binary=True) + self.add_composite_file("neostore.relationshipstore.db.id", is_binary=True) + self.add_composite_file("neostore.relationshiptypestore.db.names", is_binary=True) + self.add_composite_file("neostore.relationshiptypestore.db.names.id", is_binary=True) + self.add_composite_file("neostore.schemastore.db", is_binary=True) + self.add_composite_file("neostore.schemastore.db.id", is_binary=True) + self.add_composite_file("neostore.transaction.db.0", is_binary=True) class Neo4jDBzip(Neo4j, Data): """Class for neo4jDB database files.""" - MetadataElement(name='reference_name', default='neostore_file', desc='Reference Name', - readonly=True, visible=True, set_in_upload=True, no_value='neostore') - MetadataElement(name="neostore_zip", default=None, desc="Neostore zip", - readonly=True, visible=True, set_in_upload=True, optional=True) + + MetadataElement( + name="reference_name", + default="neostore_file", + desc="Reference Name", + readonly=True, + visible=True, + set_in_upload=True, + no_value="neostore", + ) + MetadataElement( + name="neostore_zip", + default=None, + desc="Neostore zip", + readonly=True, + visible=True, + set_in_upload=True, + optional=True, + ) file_ext = "neostore.zip" - composite_type = 'auto_primary_file' + composite_type = "auto_primary_file" def __init__(self, **kwd): Data.__init__(self, **kwd) - self.add_composite_file('%s.zip', description='neostore zip', substitute_name_with_metadata='reference_name', - is_binary=True) + self.add_composite_file( + "%s.zip", description="neostore zip", substitute_name_with_metadata="reference_name", is_binary=True + ) -if __name__ == '__main__': +if __name__ == "__main__": import doctest + doctest.testmod(sys.modules[__name__]) diff --git a/lib/galaxy/datatypes/ngsindex.py b/lib/galaxy/datatypes/ngsindex.py index e734db4df65..d065005ce29 100644 --- a/lib/galaxy/datatypes/ngsindex.py +++ b/lib/galaxy/datatypes/ngsindex.py @@ -15,17 +15,30 @@ class BowtieIndex(Html): base class for BowtieIndex is subclassed by BowtieColorIndex and BowtieBaseIndex """ - MetadataElement(name="base_name", desc="base name for this index set", default='galaxy_generated_bowtie_index', set_in_upload=True, readonly=True) - MetadataElement(name="sequence_space", desc="sequence_space for this index set", default='unknown', set_in_upload=True, readonly=True) - composite_type = 'auto_primary_file' + MetadataElement( + name="base_name", + desc="base name for this index set", + default="galaxy_generated_bowtie_index", + set_in_upload=True, + readonly=True, + ) + MetadataElement( + name="sequence_space", + desc="sequence_space for this index set", + default="unknown", + set_in_upload=True, + readonly=True, + ) + + composite_type = "auto_primary_file" def generate_primary_file(self, dataset=None): """ This is called only at upload to write the html file cannot rename the datasets here - they come with the default unfortunately """ - return 'AutoGenerated Primary File for Composite Dataset' + return "AutoGenerated Primary File for Composite Dataset" def regenerate_primary_file(self, dataset): """ @@ -33,22 +46,24 @@ class BowtieIndex(Html): """ bn = dataset.metadata.base_name flist = os.listdir(dataset.extra_files_path) - rval = [f'Files for Composite Dataset {bn}

              Comprises the following files:

                '] + rval = [ + f"Files for Composite Dataset {bn}

                Comprises the following files:

                  " + ] for fname in flist: sfname = os.path.split(fname)[-1] rval.append(f'
                • {sfname}') - rval.append('
                ') - with open(dataset.file_name, 'w') as f: + rval.append("
              ") + with open(dataset.file_name, "w") as f: f.write("\n".join(rval)) - f.write('\n') + f.write("\n") def set_peek(self, dataset): if not dataset.dataset.purged: dataset.peek = f"Bowtie index file ({dataset.metadata.sequence_space})" dataset.blurb = f"{dataset.metadata.sequence_space} space" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: @@ -61,15 +76,29 @@ class BowtieColorIndex(BowtieIndex): """ Bowtie color space index """ - MetadataElement(name="sequence_space", desc="sequence_space for this index set", default='color', set_in_upload=True, readonly=True) - file_ext = 'bowtie_color_index' + MetadataElement( + name="sequence_space", + desc="sequence_space for this index set", + default="color", + set_in_upload=True, + readonly=True, + ) + + file_ext = "bowtie_color_index" class BowtieBaseIndex(BowtieIndex): """ Bowtie base space index """ - MetadataElement(name="sequence_space", desc="sequence_space for this index set", default='base', set_in_upload=True, readonly=True) - file_ext = 'bowtie_base_index' + MetadataElement( + name="sequence_space", + desc="sequence_space for this index set", + default="base", + set_in_upload=True, + readonly=True, + ) + + file_ext = "bowtie_base_index" diff --git a/lib/galaxy/datatypes/phylip.py b/lib/galaxy/datatypes/phylip.py index 305d74f0f87..dd0f2798289 100644 --- a/lib/galaxy/datatypes/phylip.py +++ b/lib/galaxy/datatypes/phylip.py @@ -8,7 +8,10 @@ Created on January. 05, 2018 Phylip datatype sniffer """ from galaxy import util -from galaxy.datatypes.data import get_file_peek, Text +from galaxy.datatypes.data import ( + get_file_peek, + Text, +) from galaxy.datatypes.sniff import ( build_sniff_from_prefix, FilePrefix, @@ -20,12 +23,14 @@ from .metadata import MetadataElement @build_sniff_from_prefix class Phylip(Text): """Phylip format stores a multiple sequence alignment""" + edam_data = "data_0863" edam_format = "format_1997" file_ext = "phylip" - MetadataElement(name="sequences", default=0, desc="Number of sequences", readonly=True, - visible=False, optional=True, no_value=0) + MetadataElement( + name="sequences", default=0, desc="Number of sequences", readonly=True, visible=False, optional=True, no_value=0 + ) def set_meta(self, dataset, **kwd): """ @@ -45,8 +50,8 @@ class Phylip(Text): else: dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def sniff_strict_interleaved(self, nb_seq, seq_length, alignment_prefix): found_seq_length = None @@ -70,7 +75,7 @@ class Phylip(Text): # All sequence parts should have the same length return False # Fail if sequence is not ascii - seq.encode('ascii') + seq.encode("ascii") if any(str.isdigit(c) for c in seq): # Could tighten up further by requiring IUPAC strings chars return False @@ -100,7 +105,7 @@ class Phylip(Text): # All sequence parts should have the same length return False # Fail if sequence is not ascii - seq.encode('ascii') + seq.encode("ascii") if any(str.isdigit(c) for c in seq): # Could tighten up further by requiring IUPAC strings chars return False diff --git a/lib/galaxy/datatypes/plant_tribes.py b/lib/galaxy/datatypes/plant_tribes.py index 2d1cda45641..12c37747eee 100644 --- a/lib/galaxy/datatypes/plant_tribes.py +++ b/lib/galaxy/datatypes/plant_tribes.py @@ -1,7 +1,10 @@ import logging import re -from galaxy.datatypes.data import get_file_peek, Text +from galaxy.datatypes.data import ( + get_file_peek, + Text, +) from galaxy.datatypes.metadata import MetadataElement from galaxy.datatypes.sniff import ( build_sniff_from_prefix, @@ -29,8 +32,8 @@ class Smat(Text): dataset.peek = get_file_peek(dataset.file_name) dataset.blurb = "ESTScan scores matrices" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disc' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disc" def sniff_prefix(self, file_prefix: FilePrefix): """ @@ -61,11 +64,11 @@ class Smat(Text): line_no += 1 if line_no > 10000: return True - if line_no == 1 and not line.startswith('FORMAT'): + if line_no == 1 and not line.startswith("FORMAT"): # The first line is always the start of a format section. return False - if not line.startswith('FORMAT'): - if line.find('\t') >= 0: + if not line.startswith("FORMAT"): + if line.find("\t") >= 0: # Smat files are not tabular. return False items = line.split() @@ -101,7 +104,14 @@ class Smat(Text): class PlantTribesKsComponents(Tabular): file_ext = "ptkscmp" - MetadataElement(name="number_comp", default=0, desc="Number of significant components in the Ks distribution", readonly=True, visible=True, no_value=0) + MetadataElement( + name="number_comp", + default=0, + desc="Number of significant components in the Ks distribution", + readonly=True, + visible=True, + no_value=0, + ) def display_peek(self, dataset): try: @@ -134,13 +144,13 @@ class PlantTribesKsComponents(Tabular): def set_peek(self, dataset): if not dataset.dataset.purged: dataset.peek = get_file_peek(dataset.file_name) - if (dataset.metadata.number_comp == 1): + if dataset.metadata.number_comp == 1: dataset.blurb = "1 significant component" else: dataset.blurb = f"{dataset.metadata.number_comp} significant components" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def sniff(self, filename): """ @@ -153,11 +163,12 @@ class PlantTribesKsComponents(Tabular): True """ try: - line_item_str = get_headers(filename, '\\t', 1)[0][0] - return line_item_str == 'species\tn\tnumber_comp\tlnL\tAIC\tBIC\tmean\tvariance\tporportion' + line_item_str = get_headers(filename, "\\t", 1)[0][0] + return line_item_str == "species\tn\tnumber_comp\tlnL\tAIC\tBIC\tmean\tvariance\tporportion" except Exception: return False + # class PlantTribesOrtho(PlantTribes): # """ # PlantTribes sequences classified into precomputed, orthologous gene family diff --git a/lib/galaxy/datatypes/proteomics.py b/lib/galaxy/datatypes/proteomics.py index 93a1dfa2f02..a9878667abd 100644 --- a/lib/galaxy/datatypes/proteomics.py +++ b/lib/galaxy/datatypes/proteomics.py @@ -12,47 +12,55 @@ from galaxy.datatypes.sniff import ( build_sniff_from_prefix, FilePrefix, ) -from galaxy.datatypes.tabular import Tabular, TabularData +from galaxy.datatypes.tabular import ( + Tabular, + TabularData, +) from galaxy.datatypes.xml import GenericXml from galaxy.util import nice_size - log = logging.getLogger(__name__) class Wiff(Binary): """Class for wiff files.""" + edam_data = "data_2536" edam_format = "format_3710" - file_ext = 'wiff' - composite_type = 'auto_primary_file' + file_ext = "wiff" + composite_type = "auto_primary_file" def __init__(self, **kwd): super().__init__(**kwd) self.add_composite_file( - 'wiff', - description='AB SCIEX files in .wiff format. This can contain all needed information or only metadata.', - is_binary=True) + "wiff", + description="AB SCIEX files in .wiff format. This can contain all needed information or only metadata.", + is_binary=True, + ) self.add_composite_file( - 'wiff_scan', - description='AB SCIEX spectra file (wiff.scan), if the corresponding .wiff file only contains metadata.', - optional='True', is_binary=True) + "wiff_scan", + description="AB SCIEX spectra file (wiff.scan), if the corresponding .wiff file only contains metadata.", + optional="True", + is_binary=True, + ) def generate_primary_file(self, dataset=None): - rval = ['Wiff Composite Dataset

              '] - rval.append('

              This composite dataset is composed of the following files:

                ') + rval = ["Wiff Composite Dataset

                "] + rval.append("

                This composite dataset is composed of the following files:

                  ") for composite_name, composite_file in self.get_composite_files(dataset=dataset).items(): fn = composite_name - opt_text = '' + opt_text = "" if composite_file.optional: - opt_text = ' (optional)' - if composite_file.get('description'): - rval.append(f"
                • {fn} ({composite_file.get('description')}){opt_text}
                • ") + opt_text = " (optional)" + if composite_file.get("description"): + rval.append( + f"
                • {fn} ({composite_file.get('description')}){opt_text}
                • " + ) else: rval.append(f'
                • {fn}{opt_text}
                • ') - rval.append('
                ') + rval.append("
              ") return "\n".join(rval) @@ -69,15 +77,18 @@ class MzTab(Text): >>> MzTab().sniff(fname) False """ + edam_data = "data_3681" file_ext = "mztab" # section names (except MTD) _sections = ["PRH", "PRT", "PEH", "PEP", "PSH", "PSM", "SMH", "SML", "COM"] # mandatory metadata fields and list of allowed entries (in lower case) # (or None if everything is allowed) - _man_mtd = {"mzTab-mode": ["complete", "summary"], - "mzTab-type": ['quantification', 'identification'], - "description": None} + _man_mtd = { + "mzTab-mode": ["complete", "summary"], + "mzTab-type": ["quantification", "identification"], + "description": None, + } _version_re = r"(1)(\.[0-9])?(\.[0-9])?" def __init__(self, **kwd): @@ -87,13 +98,13 @@ class MzTab(Text): """Set the peek and blurb text""" if not dataset.dataset.purged: dataset.peek = data.get_file_peek(dataset.file_name) - dataset.blurb = 'mzTab Format' + dataset.blurb = "mzTab Format" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def sniff_prefix(self, file_prefix: FilePrefix): - """ Determines whether the file is the correct type. """ + """Determines whether the file is the correct type.""" has_version = False found_man_mtd = set() contents = file_prefix.string_io() @@ -125,6 +136,7 @@ class MzTab2(MzTab): >>> MzTab2().sniff(fname) False """ + file_ext = "mztab2" _sections = ["SMH", "SML", "SFH", "SMF", "SEH", "SME", "COM"] _version_re = r"(2)(\.[0-9])?(\.[0-9])?-M$" @@ -137,10 +149,10 @@ class MzTab2(MzTab): """Set the peek and blurb text""" if not dataset.dataset.purged: dataset.peek = data.get_file_peek(dataset.file_name) - dataset.blurb = 'mzTab2 Format' + dataset.blurb = "mzTab2 Format" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" @build_sniff_from_prefix @@ -156,11 +168,27 @@ class Kroenik(Tabular): >>> Kroenik().sniff(fname) False """ + file_ext = "kroenik" def __init__(self, **kwd): super().__init__(**kwd) - self.column_names = ["File", "First Scan", "Last Scan", "Num of Scans", "Charge", "Monoisotopic Mass", "Base Isotope Peak", "Best Intensity", "Summed Intensity", "First RTime", "Last RTime", "Best RTime", "Best Correlation", "Modifications"] + self.column_names = [ + "File", + "First Scan", + "Last Scan", + "Num of Scans", + "Charge", + "Monoisotopic Mass", + "Base Isotope Peak", + "Best Intensity", + "Summed Intensity", + "First RTime", + "Last RTime", + "Best RTime", + "Best Correlation", + "Modifications", + ] def display_peek(self, dataset): """Returns formated html of peek""" @@ -194,6 +222,7 @@ class PepList(Tabular): >>> PepList().sniff(fname) False """ + file_ext = "peplist" def __init__(self, **kwd): @@ -230,6 +259,7 @@ class PSMS(Tabular): >>> PSMS().sniff(fname) False """ + file_ext = "psms" def __init__(self, **kwd): @@ -254,6 +284,7 @@ class PEFF(Sequence): PSI Extended FASTA Format https://github.com/HUPO-PSI/PEFF """ + file_ext = "peff" def sniff_prefix(self, file_prefix: FilePrefix): @@ -275,12 +306,25 @@ class PEFF(Sequence): class PepXmlReport(Tabular): """pepxml converted to tabular report""" + edam_data = "data_2536" file_ext = "pepxml.tsv" def __init__(self, **kwd): super().__init__(**kwd) - self.column_names = ['Protein', 'Peptide', 'Assumed Charge', 'Neutral Pep Mass (calculated)', 'Neutral Mass', 'Retention Time', 'Start Scan', 'End Scan', 'Search Engine', 'PeptideProphet Probability', 'Interprophet Probability'] + self.column_names = [ + "Protein", + "Peptide", + "Assumed Charge", + "Neutral Pep Mass (calculated)", + "Neutral Mass", + "Retention Time", + "Start Scan", + "End Scan", + "Search Engine", + "PeptideProphet Probability", + "Interprophet Probability", + ] def display_peek(self, dataset): """Returns formated html of peek""" @@ -289,6 +333,7 @@ class PepXmlReport(Tabular): class ProtXmlReport(Tabular): """protxml converted to tabular report""" + edam_data = "data_2536" file_ext = "protxml.tsv" comment_lines = 1 @@ -296,16 +341,31 @@ class ProtXmlReport(Tabular): def __init__(self, **kwd): super().__init__(**kwd) self.column_names = [ - "Entry Number", "Group Probability", - "Protein", "Protein Link", "Protein Probability", - "Percent Coverage", "Number of Unique Peptides", - "Total Independent Spectra", "Percent Share of Spectrum ID's", - "Description", "Protein Molecular Weight", "Protein Length", - "Is Nondegenerate Evidence", "Weight", "Precursor Ion Charge", - "Peptide sequence", "Peptide Link", "NSP Adjusted Probability", - "Initial Probability", "Number of Total Termini", - "Number of Sibling Peptides Bin", "Number of Instances", - "Peptide Group Designator", "Is Evidence?"] + "Entry Number", + "Group Probability", + "Protein", + "Protein Link", + "Protein Probability", + "Percent Coverage", + "Number of Unique Peptides", + "Total Independent Spectra", + "Percent Share of Spectrum ID's", + "Description", + "Protein Molecular Weight", + "Protein Length", + "Is Nondegenerate Evidence", + "Weight", + "Precursor Ion Charge", + "Peptide sequence", + "Peptide Link", + "NSP Adjusted Probability", + "Initial Probability", + "Number of Total Termini", + "Number of Sibling Peptides Bin", + "Number of Instances", + "Peptide Group Designator", + "Is Evidence?", + ] def display_peek(self, dataset): """Returns formated html of peek""" @@ -318,6 +378,7 @@ class Dta(TabularData): peptide charge state separated by a space. Subsequent lines contain space separated pairs of fragment ion m/z and intensity values. """ + file_ext = "dta" comment_lines = 0 @@ -337,9 +398,9 @@ class Dta(TabularData): # Set metadata dataset.metadata.data_lines = data_lines dataset.metadata.comment_lines = 0 - dataset.metadata.column_types = ['float', 'float'] + dataset.metadata.column_types = ["float", "float"] dataset.metadata.columns = 2 - dataset.metadata.column_names = ['m/z', 'intensity'] + dataset.metadata.column_names = ["m/z", "intensity"] dataset.metadata.delimiter = " " @@ -365,6 +426,7 @@ class Dta2d(TabularData): >>> Dta2d().sniff(fname) False """ + file_ext = "dta2d" comment_lines = 0 @@ -373,7 +435,7 @@ class Dta2d(TabularData): return None line[0] = line[0].lstrip("#") line = [_.strip() for _ in line] - if 'MZ' not in line or 'INT' not in line or ('MIN' not in line and 'SEC' not in line): + if "MZ" not in line or "INT" not in line or ("MIN" not in line and "SEC" not in line): return None return line @@ -410,12 +472,12 @@ class Dta2d(TabularData): dataset.metadata.data_lines = data_lines dataset.metadata.comment_lines = 0 - dataset.metadata.column_types = ['float', 'float', 'float'] + dataset.metadata.column_types = ["float", "float", "float"] dataset.metadata.columns = 3 if dataset.metadata.column_names is None or dataset.metadata.column_names == []: dataset.metadata.comment_lines += 1 dataset.metadata.data_lines -= 1 - dataset.metadata.column_names = ['SEC', 'MZ', 'INT'] + dataset.metadata.column_names = ["SEC", "MZ", "INT"] def sniff_prefix(self, file_prefix: FilePrefix): sep = None @@ -466,6 +528,7 @@ class Edta(TabularData): >>> Edta().sniff(fname) False """ + file_ext = "edta" comment_lines = 0 @@ -585,20 +648,21 @@ class Edta(TabularData): class ProteomicsXml(GenericXml): - """ An enhanced XML datatype used to reuse code across several - proteomic/mass-spec datatypes. """ + """An enhanced XML datatype used to reuse code across several + proteomic/mass-spec datatypes.""" + edam_data = "data_2536" edam_format = "format_2032" root: str def sniff_prefix(self, file_prefix: FilePrefix): - """ Determines whether the file is the correct XML type. """ + """Determines whether the file is the correct XML type.""" for line in file_prefix.line_iterator(): line = line.strip() - if not line.startswith('Spectral Library Composite Dataset

              '] - rval.append('

              This composite dataset is composed of the following files:

                ') + rval = ["Spectral Library Composite Dataset

                "] + rval.append("

                This composite dataset is composed of the following files:

                  ") for composite_name, composite_file in self.get_composite_files(dataset=dataset).items(): fn = composite_name - opt_text = '' + opt_text = "" if composite_file.optional: - opt_text = ' (optional)' - if composite_file.get('description'): - rval.append(f"
                • {fn} ({composite_file.get('description')}){opt_text}
                • ") + opt_text = " (optional)" + if composite_file.get("description"): + rval.append( + f"
                • {fn} ({composite_file.get('description')}){opt_text}
                • " + ) else: rval.append(f'
                • {fn}{opt_text}
                • ') - rval.append('
                ') + rval.append("
              ") return "\n".join(rval) def set_peek(self, dataset): """Set the peek and blurb text""" if not dataset.dataset.purged: dataset.peek = data.get_file_peek(dataset.file_name) - dataset.blurb = 'splib Spectral Library Format' + dataset.blurb = "splib Spectral Library Format" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def sniff_prefix(self, file_prefix: FilePrefix): - """ Determines whether the file is a SpectraST generated file. - """ + """Determines whether the file is a SpectraST generated file.""" contents = file_prefix.string_io() return Msp.next_line_starts_with(contents, "Name:") and Msp.next_line_starts_with(contents, "LibID:") @@ -931,20 +1010,20 @@ class Ms2(Text): file_ext = "ms2" def sniff_prefix(self, file_prefix: FilePrefix): - """ Determines whether the file is a valid ms2 file.""" + """Determines whether the file is a valid ms2 file.""" header_lines = [] for line in file_prefix.line_iterator(): if line.strip() == "": continue - elif line.startswith('H\t'): + elif line.startswith("H\t"): header_lines.append(line) else: break - for header_field in ['CreationDate', 'Extractor', 'ExtractorVersion', 'ExtractorOptions']: + for header_field in ["CreationDate", "Extractor", "ExtractorVersion", "ExtractorOptions"]: found_header = False for header_line in header_lines: - if header_line.startswith(f'H\t{header_field}'): + if header_line.startswith(f"H\t{header_field}"): found_header = True break if not found_header: @@ -955,46 +1034,45 @@ class Ms2(Text): # unsniffable binary format, should do something about this class XHunterAslFormat(Binary): - """ Annotated Spectra in the HLF format http://www.thegpm.org/HUNTER/format_2006_09_15.html """ + """Annotated Spectra in the HLF format http://www.thegpm.org/HUNTER/format_2006_09_15.html""" + file_ext = "hlf" class Sf3(Binary): """Class describing a Scaffold SF3 files""" + file_ext = "sf3" class ImzML(Binary): """ - Class for imzML files. - http://www.imzml.org + Class for imzML files. + http://www.imzml.org """ + edam_format = "format_3682" - file_ext = 'imzml' - composite_type = 'auto_primary_file' + file_ext = "imzml" + composite_type = "auto_primary_file" def __init__(self, **kwd): super().__init__(**kwd) - self.add_composite_file( - 'imzml', - description='The imzML metadata component.', - is_binary=False) + self.add_composite_file("imzml", description="The imzML metadata component.", is_binary=False) - self.add_composite_file( - 'ibd', - description='The mass spectral data component.', - is_binary=True) + self.add_composite_file("ibd", description="The mass spectral data component.", is_binary=True) def generate_primary_file(self, dataset=None): - rval = ['imzML Composite Dataset

              '] - rval.append('

              This composite dataset is composed of the following files:

                ') + rval = ["imzML Composite Dataset

                "] + rval.append("

                This composite dataset is composed of the following files:

                ') + rval.append("
              ") return "\n".join(rval) diff --git a/lib/galaxy/datatypes/qualityscore.py b/lib/galaxy/datatypes/qualityscore.py index 1e987be499d..1ce98a17600 100644 --- a/lib/galaxy/datatypes/qualityscore.py +++ b/lib/galaxy/datatypes/qualityscore.py @@ -7,9 +7,7 @@ from galaxy.datatypes.sniff import ( build_sniff_from_prefix, FilePrefix, ) -from . import ( - data, -) +from . import data log = logging.getLogger(__name__) @@ -18,6 +16,7 @@ class QualityScore(data.Text): """ until we know more about quality score formats """ + edam_data = "data_2048" edam_format = "format_3606" file_ext = "qual" @@ -28,6 +27,7 @@ class QualityScoreSOLiD(QualityScore): """ until we know more about quality score formats """ + edam_format = "format_3610" file_ext = "qualsolid" @@ -46,10 +46,10 @@ class QualityScoreSOLiD(QualityScore): goodblock = 0 for line in fh: line = line.strip() - if not line.startswith('#'): # first non-empty non-comment line - if line.startswith('>'): + if not line.startswith("#"): # first non-empty non-comment line + if line.startswith(">"): line = fh.readline().strip() - if line == '' or line.startswith('>'): + if line == "" or line.startswith(">"): return False try: [int(x) for x in line.split()] @@ -77,6 +77,7 @@ class QualityScore454(QualityScore): """ until we know more about quality score formats """ + edam_format = "format_3611" file_ext = "qual454" @@ -93,10 +94,10 @@ class QualityScore454(QualityScore): fh = file_prefix.string_io() for line in fh: line = line.strip() - if line and not line.startswith('#'): # first non-empty non-comment line - if line.startswith('>'): + if line and not line.startswith("#"): # first non-empty non-comment line + if line.startswith(">"): line = fh.readline().strip() - if line == '' or line.startswith('>'): + if line == "" or line.startswith(">"): break try: [int(x) for x in line.split()] @@ -112,6 +113,7 @@ class QualityScoreSolexa(QualityScore): """ until we know more about quality score formats """ + edam_format = "format_3608" file_ext = "qualsolexa" @@ -120,5 +122,6 @@ class QualityScoreIllumina(QualityScore): """ until we know more about quality score formats """ + edam_format = "format_3609" file_ext = "qualillumina" diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index bb0c508bd19..3cc5f1d4e68 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -6,7 +6,13 @@ import imp import logging import os from string import Template -from typing import Dict, List, Optional, Tuple, TYPE_CHECKING +from typing import ( + Dict, + List, + Optional, + Tuple, + TYPE_CHECKING, +) import yaml @@ -24,7 +30,7 @@ from . import ( tabular, text, tracks, - xml + xml, ) from .display_applications.application import DisplayApplication @@ -37,7 +43,6 @@ class ConfigurationError(Exception): class Registry: - def __init__(self, config=None): self.log = logging.getLogger(__name__) self.log.addHandler(logging.NullHandler()) @@ -85,7 +90,16 @@ class Registry: self.display_sites = {} self.legacy_build_sites = {} - def load_datatypes(self, root_dir=None, config=None, deactivate=False, override=True, use_converters=True, use_display_applications=True, use_build_sites=True): + def load_datatypes( + self, + root_dir=None, + config=None, + deactivate=False, + override=True, + use_converters=True, + use_display_applications=True, + use_build_sites=True, + ): """ Parse a datatypes XML file located at root_dir/config (if processing the Galaxy distributed config) or contained within an installed Tool Shed repository. If deactivate is True, an installed Tool Shed repository that includes custom datatypes @@ -115,63 +129,71 @@ class Registry: root = tree.getroot() # Load datatypes and converters from config if deactivate: - self.log.debug(f'Deactivating datatypes from {config}') + self.log.debug(f"Deactivating datatypes from {config}") else: - self.log.debug(f'Loading datatypes from {config}') + self.log.debug(f"Loading datatypes from {config}") else: root = config - registration = root.find('registration') + registration = root.find("registration") # Set default paths defined in local datatypes_conf.xml. if use_converters: if not self.converters_path: - self.converters_path_attr = registration.get('converters_path', 'lib/galaxy/datatypes/converters') + self.converters_path_attr = registration.get("converters_path", "lib/galaxy/datatypes/converters") self.converters_path = os.path.join(root_dir, self.converters_path_attr) - if self.converters_path_attr == 'lib/galaxy/datatypes/converters' \ - and not os.path.isdir(self.converters_path): + if self.converters_path_attr == "lib/galaxy/datatypes/converters" and not os.path.isdir( + self.converters_path + ): # Deal with the old default of this path being set in # datatypes_conf.xml.sample (this path is not useful in an # "installed Galaxy" world) - self.converters_path_attr = os.path.abspath(os.path.join(os.path.dirname(__file__), 'converters')) + self.converters_path_attr = os.path.abspath( + os.path.join(os.path.dirname(__file__), "converters") + ) self.converters_path = self.converters_path_attr if not os.path.isdir(self.converters_path): raise ConfigurationError(f"Directory does not exist: {self.converters_path}") if use_display_applications: if not self.display_applications_path: - self.display_path_attr = registration.get('display_path', 'display_applications') + self.display_path_attr = registration.get("display_path", "display_applications") self.display_applications_path = os.path.join(root_dir, self.display_path_attr) - if self.display_path_attr == 'display_applications' \ - and not os.path.isdir('display_applications'): + if self.display_path_attr == "display_applications" and not os.path.isdir("display_applications"): # Ditto as with converters_path - self.display_path_attr = os.path.abspath(os.path.join(os.path.dirname(__file__), 'display_applications', 'configs')) + self.display_path_attr = os.path.abspath( + os.path.join(os.path.dirname(__file__), "display_applications", "configs") + ) self.display_applications_path = self.display_path_attr # Proprietary datatype's tag may have special attributes, proprietary_converter_path and proprietary_display_path. - proprietary_converter_path = registration.get('proprietary_converter_path', None) - proprietary_display_path = registration.get('proprietary_display_path', None) - if proprietary_converter_path is not None or proprietary_display_path is not None and not handling_proprietary_datatypes: + proprietary_converter_path = registration.get("proprietary_converter_path", None) + proprietary_display_path = registration.get("proprietary_display_path", None) + if ( + proprietary_converter_path is not None + or proprietary_display_path is not None + and not handling_proprietary_datatypes + ): handling_proprietary_datatypes = True - for elem in registration.findall('datatype'): + for elem in registration.findall("datatype"): # Keep a status of the process steps to enable stopping the process of handling the datatype if necessary. ok = True extension = self.get_extension(elem) - dtype = elem.get('type', None) - type_extension = elem.get('type_extension', None) - auto_compressed_types = galaxy.util.listify(elem.get('auto_compressed_types', '')) + dtype = elem.get("type", None) + type_extension = elem.get("type_extension", None) + auto_compressed_types = galaxy.util.listify(elem.get("auto_compressed_types", "")) sniff_compressed_types = galaxy.util.string_as_bool_or_none(elem.get("sniff_compressed_types", "None")) if sniff_compressed_types is None: sniff_compressed_types = getattr(self.config, "sniff_compressed_dynamic_datatypes_default", True) # Make sure this is set in the elems we write out so the config option is passed to the upload # tool which does not have a config object. elem.set("sniff_compressed_types", str(sniff_compressed_types)) - mimetype = elem.get('mimetype', None) - display_in_upload = galaxy.util.string_as_bool(elem.get('display_in_upload', False)) + mimetype = elem.get("mimetype", None) + display_in_upload = galaxy.util.string_as_bool(elem.get("display_in_upload", False)) # If make_subclass is True, it does not necessarily imply that we are subclassing a datatype that is contained # in the distribution. - make_subclass = galaxy.util.string_as_bool(elem.get('subclass', False)) - edam_format = elem.get('edam_format', None) + make_subclass = galaxy.util.string_as_bool(elem.get("subclass", False)) + edam_format = elem.get("edam_format", None) if edam_format and not make_subclass: self.log.warning("Cannot specify edam_format without setting subclass to True, skipping datatype.") continue - edam_data = elem.get('edam_data', None) + edam_data = elem.get("edam_data", None) if edam_data and not make_subclass: self.log.warning("Cannot specify edam_data without setting subclass to True, skipping datatype.") continue @@ -179,26 +201,34 @@ class Registry: # (proprietary_path and proprietary_datatype_module) if they depend on proprietary datatypes classes. # The value of proprietary_path is the path to the cloned location of the tool shed repository's contained # datatypes_conf.xml file. - proprietary_path = elem.get('proprietary_path', None) - proprietary_datatype_module = elem.get('proprietary_datatype_module', None) - if proprietary_path is not None or proprietary_datatype_module is not None and not handling_proprietary_datatypes: + proprietary_path = elem.get("proprietary_path", None) + proprietary_datatype_module = elem.get("proprietary_datatype_module", None) + if ( + proprietary_path is not None + or proprietary_datatype_module is not None + and not handling_proprietary_datatypes + ): handling_proprietary_datatypes = True if deactivate: # We are deactivating or uninstalling an installed tool shed repository, so eliminate the datatype # elem from the in-memory list of datatype elems. for in_memory_elem in self.datatype_elems: - in_memory_extension = in_memory_elem.get('extension', None) + in_memory_extension = in_memory_elem.get("extension", None) if in_memory_extension == extension: - in_memory_dtype = elem.get('type', None) - in_memory_type_extension = elem.get('type_extension', None) - in_memory_mimetype = elem.get('mimetype', None) - in_memory_display_in_upload = galaxy.util.string_as_bool(elem.get('display_in_upload', False)) - in_memory_make_subclass = galaxy.util.string_as_bool(elem.get('subclass', False)) - if in_memory_dtype == dtype and \ - in_memory_type_extension == type_extension and \ - in_memory_mimetype == mimetype and \ - in_memory_display_in_upload == display_in_upload and \ - in_memory_make_subclass == make_subclass: + in_memory_dtype = elem.get("type", None) + in_memory_type_extension = elem.get("type_extension", None) + in_memory_mimetype = elem.get("mimetype", None) + in_memory_display_in_upload = galaxy.util.string_as_bool( + elem.get("display_in_upload", False) + ) + in_memory_make_subclass = galaxy.util.string_as_bool(elem.get("subclass", False)) + if ( + in_memory_dtype == dtype + and in_memory_type_extension == type_extension + and in_memory_mimetype == mimetype + and in_memory_display_in_upload == display_in_upload + and in_memory_make_subclass == make_subclass + ): self.datatype_elems.remove(in_memory_elem) if extension is not None and extension in self.datatypes_by_extension: # We are deactivating or uninstalling an installed tool shed repository, so eliminate the datatype @@ -218,11 +248,11 @@ class Registry: if can_process_datatype: if dtype is not None: try: - fields = dtype.split(':') + fields = dtype.split(":") datatype_module = fields[0] datatype_class_name = fields[1] except Exception: - self.log.exception('Error parsing datatype definition for dtype %s', str(dtype)) + self.log.exception("Error parsing datatype definition for dtype %s", str(dtype)) ok = False if ok: datatype_class = None @@ -230,36 +260,46 @@ class Registry: # TODO: previously comments suggested this needs to be locked because it modifies # the sys.path, probably true but the previous lock wasn't doing that. try: - imported_module = __import_module(proprietary_path, - proprietary_datatype_module, - datatype_class_name) + imported_module = __import_module( + proprietary_path, proprietary_datatype_module, datatype_class_name + ) if imported_module not in self.imported_modules: self.imported_modules.append(imported_module) if hasattr(imported_module, datatype_class_name): datatype_class = getattr(imported_module, datatype_class_name) except Exception as e: full_path = os.path.join(proprietary_path, proprietary_datatype_module) - self.log.debug("Exception importing proprietary code file %s: %s", full_path, galaxy.util.unicodify(e)) + self.log.debug( + "Exception importing proprietary code file %s: %s", + full_path, + galaxy.util.unicodify(e), + ) # Either the above exception was thrown because the proprietary_datatype_module is not derived from a class # in the repository, or we are loading Galaxy's datatypes. In either case we'll look in the registry. if datatype_class is None: try: # The datatype class name must be contained in one of the datatype modules in the Galaxy distribution. - fields = datatype_module.split('.')[1:] + fields = datatype_module.split(".")[1:] module = __import__(datatype_module) for mod in fields: module = getattr(module, mod) datatype_class = getattr(module, datatype_class_name) - self.log.debug(f'Retrieved datatype module {str(datatype_module)}:{datatype_class_name} from the datatype registry for extension {extension}.') + self.log.debug( + f"Retrieved datatype module {str(datatype_module)}:{datatype_class_name} from the datatype registry for extension {extension}." + ) except Exception: - self.log.exception('Error importing datatype module %s', str(datatype_module)) + self.log.exception("Error importing datatype module %s", str(datatype_module)) ok = False elif type_extension is not None: try: datatype_class = self.datatypes_by_extension[type_extension].__class__ - self.log.debug(f'Retrieved datatype module {str(datatype_class.__name__)} from type_extension {type_extension} for extension {extension}.') + self.log.debug( + f"Retrieved datatype module {str(datatype_class.__name__)} from type_extension {type_extension} for extension {extension}." + ) except Exception: - self.log.exception('Error determining datatype_class for type_extension %s', str(type_extension)) + self.log.exception( + "Error determining datatype_class for type_extension %s", str(type_extension) + ) ok = False if ok: if not deactivate: @@ -269,10 +309,12 @@ class Registry: if extension in self.datatypes_by_extension: # Because of the way that the value of can_process_datatype was set above, we know that the value of # override is True. - self.log.debug("Overriding conflicting datatype with extension '%s', using datatype from %s." % - (str(extension), str(config))) + self.log.debug( + "Overriding conflicting datatype with extension '%s', using datatype from %s." + % (str(extension), str(config)) + ) if make_subclass: - datatype_class = type(datatype_class_name, (datatype_class, ), {}) + datatype_class = type(datatype_class_name, (datatype_class,), {}) if edam_format: datatype_class.edam_format = edam_format if edam_data: @@ -291,30 +333,38 @@ class Registry: if display_in_upload and extension not in self.upload_file_formats: self.upload_file_formats.append(extension) # Max file size cut off for setting optional metadata. - self.datatypes_by_extension[extension].max_optional_metadata_filesize = elem.get('max_optional_metadata_filesize', None) - for converter in elem.findall('converter'): + self.datatypes_by_extension[extension].max_optional_metadata_filesize = elem.get( + "max_optional_metadata_filesize", None + ) + for converter in elem.findall("converter"): # Build the list of datatype converters which will later be loaded into the calling app's toolbox. - converter_config = converter.get('file', None) - target_datatype = converter.get('target_datatype', None) - depends_on = converter.get('depends_on', None) + converter_config = converter.get("file", None) + target_datatype = converter.get("target_datatype", None) + depends_on = converter.get("depends_on", None) if depends_on is not None and target_datatype is not None: if extension not in self.converter_deps: self.converter_deps[extension] = {} - self.converter_deps[extension][target_datatype] = depends_on.split(',') + self.converter_deps[extension][target_datatype] = depends_on.split(",") if converter_config and target_datatype: if proprietary_converter_path: - self.proprietary_converters.append((converter_config, extension, target_datatype)) + self.proprietary_converters.append( + (converter_config, extension, target_datatype) + ) else: self.converters.append((converter_config, extension, target_datatype)) # Add composite files. - for composite_file in elem.findall('composite_file'): - name = composite_file.get('name', None) + for composite_file in elem.findall("composite_file"): + name = composite_file.get("name", None) if name is None: - self.log.warning(f"You must provide a name for your composite_file ({composite_file}).") - optional = composite_file.get('optional', False) - mimetype = composite_file.get('mimetype', None) - self.datatypes_by_extension[extension].add_composite_file(name, optional=optional, mimetype=mimetype) - for _display_app in elem.findall('display'): + self.log.warning( + f"You must provide a name for your composite_file ({composite_file})." + ) + optional = composite_file.get("optional", False) + mimetype = composite_file.get("mimetype", None) + self.datatypes_by_extension[extension].add_composite_file( + name, optional=optional, mimetype=mimetype + ) + for _display_app in elem.findall("display"): if proprietary_display_path: if elem not in self.proprietary_display_app_containers: self.proprietary_display_app_containers.append(elem) @@ -329,7 +379,7 @@ class Registry: } composite_files = datatype_instance.composite_files if composite_files: - datatype_info_dict['composite_files'] = [_.dict() for _ in composite_files.values()] + datatype_info_dict["composite_files"] = [_.dict() for _ in composite_files.values()] self.datatype_info_dicts.append(datatype_info_dict) for auto_compressed_type in auto_compressed_types: @@ -345,7 +395,14 @@ class Registry: raise Exception(f"Unknown auto compression type [{auto_compressed_type}]") attributes["file_ext"] = compressed_extension attributes["uncompressed_datatype_instance"] = datatype_instance - compressed_datatype_class = type(auto_compressed_type_name, (datatype_class, dynamic_parent, ), attributes) + compressed_datatype_class = type( + auto_compressed_type_name, + ( + datatype_class, + dynamic_parent, + ), + attributes, + ) if edam_format: compressed_datatype_class.edam_format = edam_format if edam_data: @@ -354,15 +411,25 @@ class Registry: self.datatypes_by_extension[compressed_extension] = compressed_datatype_instance if display_in_upload and compressed_extension not in self.upload_file_formats: self.upload_file_formats.append(compressed_extension) - self.datatype_info_dicts.append({ - "display_in_upload": display_in_upload, - "extension": compressed_extension, - "description": description, - "description_url": description_url, - }) - if auto_compressed_type == 'gz': - self.converters.append((f"uncompressed_to_{auto_compressed_type}.xml", extension, compressed_extension)) - self.converters.append((f"{auto_compressed_type}_to_uncompressed.xml", compressed_extension, extension)) + self.datatype_info_dicts.append( + { + "display_in_upload": display_in_upload, + "extension": compressed_extension, + "description": description, + "description_url": description_url, + } + ) + if auto_compressed_type == "gz": + self.converters.append( + ( + f"uncompressed_to_{auto_compressed_type}.xml", + extension, + compressed_extension, + ) + ) + self.converters.append( + (f"{auto_compressed_type}_to_uncompressed.xml", compressed_extension, extension) + ) if datatype_class not in compressed_sniffers: compressed_sniffers[datatype_class] = [] if sniff_compressed_types: @@ -377,14 +444,18 @@ class Registry: if not override: # Do not load the datatype since it conflicts with an existing datatype which we are not supposed # to override. - self.log.debug(f"Ignoring conflicting datatype with extension '{extension}' from {config}.") + self.log.debug( + f"Ignoring conflicting datatype with extension '{extension}' from {config}." + ) # Load datatype sniffers from the config - we'll do this even if one or more datatypes were not properly processed in the config # since sniffers are not tightly coupled with datatypes. - self.load_datatype_sniffers(root, - deactivate=deactivate, - handling_proprietary_datatypes=handling_proprietary_datatypes, - override=override, - compressed_sniffers=compressed_sniffers) + self.load_datatype_sniffers( + root, + deactivate=deactivate, + handling_proprietary_datatypes=handling_proprietary_datatypes, + override=override, + compressed_sniffers=compressed_sniffers, + ) self.upload_file_formats.sort() # Load build sites if use_build_sites: @@ -398,23 +469,25 @@ class Registry: # has a sniff() method and was not defined with subclass="true". # Do not add dynamic compressed types - these were carefully added or not # to the sniff order in the proper position above. - if type(datatype) not in sniff_order_classes and \ - hasattr(datatype, 'sniff') and not datatype.is_subclass and \ - not hasattr(datatype, "uncompressed_datatype_instance"): + if ( + type(datatype) not in sniff_order_classes + and hasattr(datatype, "sniff") + and not datatype.is_subclass + and not hasattr(datatype, "uncompressed_datatype_instance") + ): self.sniff_order.append(datatype) append_to_sniff_order() def _load_build_sites(self, root): - def load_build_site(build_site_config): # Take in either an XML element or simple dictionary from YAML and add build site for this. - if not (build_site_config.get('type') and build_site_config.get('file')): + if not (build_site_config.get("type") and build_site_config.get("file")): self.log.exception("Site is missing required 'type' and 'file' attributes") return - site_type = build_site_config.get('type') - path = build_site_config.get('file') + site_type = build_site_config.get("type") + path = build_site_config.get("file") if not os.path.exists(path): sample_path = f"{path}.sample" if os.path.exists(sample_path): @@ -422,19 +495,19 @@ class Registry: path = sample_path self.build_sites[site_type] = path - if site_type in ('ucsc', 'gbrowse'): + if site_type in ("ucsc", "gbrowse"): self.legacy_build_sites[site_type] = galaxy.util.read_build_sites(path) - if build_site_config.get('display', None): - display = build_site_config.get('display') + if build_site_config.get("display", None): + display = build_site_config.get("display") if not isinstance(display, list): - display = [x.strip() for x in display.lower().split(',')] + display = [x.strip() for x in display.lower().split(",")] self.display_sites[site_type] = display self.log.debug("Loaded build site '%s': %s with display sites: %s", site_type, path, display) else: self.log.debug("Loaded build site '%s': %s", site_type, path) - if root.find('build_sites') is not None: - for elem in root.find('build_sites').findall('site'): + if root.find("build_sites") is not None: + for elem in root.find("build_sites").findall("site"): load_build_site(elem) else: build_sites_config_file = getattr(self.config, "build_sites_config_file", None) @@ -453,14 +526,16 @@ class Registry: def get_legacy_sites_by_build(self, site_type, build): sites = [] for site in self.legacy_build_sites.get(site_type, []): - if build in site['builds']: - sites.append((site['name'], site['url'])) + if build in site["builds"]: + sites.append((site["name"], site["url"])) return sites def get_display_sites(self, site_type): return self.display_sites.get(site_type, []) - def load_datatype_sniffers(self, root, deactivate=False, handling_proprietary_datatypes=False, override=False, compressed_sniffers=None): + def load_datatype_sniffers( + self, root, deactivate=False, handling_proprietary_datatypes=False, override=False, compressed_sniffers=None + ): """ Process the sniffers element from a parsed a datatypes XML file located at root_dir/config (if processing the Galaxy distributed config) or contained within an installed Tool Shed repository. If deactivate is True, an installed Tool @@ -469,13 +544,13 @@ class Registry: Since installation is occurring after the datatypes registry has been initialized at server startup, its contents cannot be overridden by newly introduced conflicting sniffers. """ - sniffer_elem_classes = [e.attrib['type'] for e in self.sniffer_elems] - sniffers = root.find('sniffers') + sniffer_elem_classes = [e.attrib["type"] for e in self.sniffer_elems] + sniffers = root.find("sniffers") if sniffers is not None: - for elem in sniffers.findall('sniffer'): + for elem in sniffers.findall("sniffer"): # Keep a status of the process steps to enable stopping the process of handling the sniffer if necessary. ok = True - dtype = elem.get('type', None) + dtype = elem.get("type", None) if dtype is not None: try: fields = dtype.split(":") @@ -483,7 +558,7 @@ class Registry: datatype_class_name = fields[1] module = None except Exception: - self.log.exception('Error determining datatype class or module for dtype %s', str(dtype)) + self.log.exception("Error determining datatype class or module for dtype %s", str(dtype)) ok = False if ok: if handling_proprietary_datatypes: @@ -496,7 +571,7 @@ class Registry: try: # The datatype class name must be contained in one of the datatype modules in the Galaxy distribution. module = __import__(datatype_module) - for comp in datatype_module.split('.')[1:]: + for comp in datatype_module.split(".")[1:]: module = getattr(module, comp) except Exception: self.log.exception("Error importing datatype class for '%s'", str(dtype)) @@ -505,23 +580,29 @@ class Registry: try: aclass = getattr(module, datatype_class_name)() except Exception: - self.log.exception('Error calling method %s from class %s', str(datatype_class_name), str(module)) + self.log.exception( + "Error calling method %s from class %s", str(datatype_class_name), str(module) + ) ok = False if ok: if deactivate: # We are deactivating or uninstalling an installed Tool Shed repository, so eliminate the appropriate sniffers. - sniffer_class = elem.get('type', None) + sniffer_class = elem.get("type", None) if sniffer_class is not None: for index, s_e_c in enumerate(sniffer_elem_classes): if sniffer_class == s_e_c: del self.sniffer_elems[index] - sniffer_elem_classes = [elem.attrib['type'] for elem in self.sniffer_elems] + sniffer_elem_classes = [ + elem.attrib["type"] for elem in self.sniffer_elems + ] self.log.debug(f"Removed sniffer element for datatype '{str(dtype)}'") break for sniffer_class in self.sniff_order: if sniffer_class.__class__ == aclass.__class__: self.sniff_order.remove(sniffer_class) - self.log.debug(f"Removed sniffer class for datatype '{str(dtype)}' from sniff order") + self.log.debug( + f"Removed sniffer class for datatype '{str(dtype)}' from sniff order" + ) break else: # We are loading new sniffer, so see if we have a conflicting sniffer already loaded. @@ -541,14 +622,14 @@ class Registry: self.sniff_order.append(aclass) self.log.debug(f"Loaded sniffer for datatype '{dtype}'") # Processing the new sniffer elem is now complete, so make sure the element defining it is loaded if necessary. - sniffer_class = elem.get('type', None) + sniffer_class = elem.get("type", None) if sniffer_class is not None: if sniffer_class not in sniffer_elem_classes: self.sniffer_elems.append(elem) def is_extension_unsniffable_binary(self, ext): datatype = self.get_datatype_by_extension(ext) - return datatype is not None and isinstance(datatype, binary.Binary) and not hasattr(datatype, 'sniff') + return datatype is not None and isinstance(datatype, binary.Binary) and not hasattr(datatype, "sniff") def get_datatype_class_by_name(self, name): """ @@ -557,7 +638,7 @@ class Registry: """ # TODO: obviously not ideal but some of these base classes that are useful for testing datatypes # aren't loaded into the datatypes registry, so we'd need to test for them here - if name == 'images.Image': + if name == "images.Image": return images.Image # TODO: too inefficient - would be better to generate this once as a map and store in this object @@ -571,14 +652,14 @@ class Registry: def get_available_tracks(self): return self.available_tracks - def get_mimetype_by_extension(self, ext, default='application/octet-stream'): + def get_mimetype_by_extension(self, ext, default="application/octet-stream"): """Returns a mimetype based on an extension""" try: mimetype = self.mimetypes_by_extension[ext] except KeyError: # datatype was never declared mimetype = default - self.log.warning(f'unknown mimetype in data factory {str(ext)}') + self.log.warning(f"unknown mimetype in data factory {str(ext)}") return mimetype def get_datatype_by_extension(self, ext): @@ -612,7 +693,7 @@ class Registry: source_datatype = elem[1] target_datatype = elem[2] if installed_repository_dict: - converter_path = installed_repository_dict['converter_path'] + converter_path = installed_repository_dict["converter_path"] else: converter_path = self.converters_path try: @@ -622,17 +703,17 @@ class Registry: if installed_repository_dict: # If the converter is included in an installed tool shed repository, set the tool # shed related tool attributes. - converter.tool_shed = installed_repository_dict['tool_shed'] - converter.repository_name = installed_repository_dict['repository_name'] - converter.repository_owner = installed_repository_dict['repository_owner'] - converter.installed_changeset_revision = installed_repository_dict['installed_changeset_revision'] + converter.tool_shed = installed_repository_dict["tool_shed"] + converter.repository_name = installed_repository_dict["repository_name"] + converter.repository_owner = installed_repository_dict["repository_owner"] + converter.installed_changeset_revision = installed_repository_dict["installed_changeset_revision"] converter.old_id = converter.id # The converter should be included in the list of tools defined in tool_dicts. - tool_dicts = installed_repository_dict['tool_dicts'] + tool_dicts = installed_repository_dict["tool_dicts"] for tool_dict in tool_dicts: - if tool_dict['id'] == converter.id: - converter.guid = tool_dict['guid'] - converter.id = tool_dict['guid'] + if tool_dict["id"] == converter.id: + converter.guid = tool_dict["guid"] + converter.id = tool_dict["guid"] break if deactivate: toolbox.remove_tool_by_id(converter.id, remove_from_panel=False) @@ -645,7 +726,7 @@ class Registry: if source_datatype not in self.datatype_converters: self.datatype_converters[source_datatype] = {} self.datatype_converters[source_datatype][target_datatype] = converter - if not hasattr(toolbox.app, 'tool_cache') or converter.id in toolbox.app.tool_cache._new_tool_ids: + if not hasattr(toolbox.app, "tool_cache") or converter.id in toolbox.app.tool_cache._new_tool_ids: self.log.debug("Loaded converter: %s", converter.id) except Exception: if deactivate: @@ -667,16 +748,16 @@ class Registry: datatype_elems = self.display_app_containers for elem in datatype_elems: extension = self.get_extension(elem) - for display_app in elem.findall('display'): - display_file = display_app.get('file', None) + for display_app in elem.findall("display"): + display_file = display_app.get("file", None) if installed_repository_dict: - display_path = installed_repository_dict['display_path'] + display_path = installed_repository_dict["display_path"] display_file_head, display_file_tail = os.path.split(display_file) config_path = os.path.join(display_path, display_file_tail) else: config_path = os.path.join(self.display_applications_path, display_file) try: - inherit = galaxy.util.string_as_bool(display_app.get('inherit', 'False')) + inherit = galaxy.util.string_as_bool(display_app.get("inherit", "False")) display_app = DisplayApplication.from_file(config_path, app) if display_app: if display_app.id in self.display_applications: @@ -688,17 +769,19 @@ class Registry: elif installed_repository_dict: # If the display application is included in an installed tool shed repository, # set the tool shed related tool attributes. - display_app.tool_shed = installed_repository_dict['tool_shed'] - display_app.repository_name = installed_repository_dict['repository_name'] - display_app.repository_owner = installed_repository_dict['repository_owner'] - display_app.installed_changeset_revision = installed_repository_dict['installed_changeset_revision'] + display_app.tool_shed = installed_repository_dict["tool_shed"] + display_app.repository_name = installed_repository_dict["repository_name"] + display_app.repository_owner = installed_repository_dict["repository_owner"] + display_app.installed_changeset_revision = installed_repository_dict[ + "installed_changeset_revision" + ] display_app.old_id = display_app.id # The display application should be included in the list of tools defined in tool_dicts. - tool_dicts = installed_repository_dict['tool_dicts'] + tool_dicts = installed_repository_dict["tool_dicts"] for tool_dict in tool_dicts: - if tool_dict['id'] == display_app.id: - display_app.guid = tool_dict['guid'] - display_app.id = tool_dict['guid'] + if tool_dict["id"] == display_app.id: + display_app.guid = tool_dict["guid"] + display_app.id = tool_dict["guid"] break if deactivate: if display_app.id in self.display_applications: @@ -706,15 +789,31 @@ class Registry: if extension in self.datatypes_by_extension: if display_app.id in self.datatypes_by_extension[extension].display_applications: del self.datatypes_by_extension[extension].display_applications[display_app.id] - if inherit and (self.datatypes_by_extension[extension], display_app) in self.inherit_display_application_by_class: - self.inherit_display_application_by_class.remove((self.datatypes_by_extension[extension], display_app)) - self.log.debug(f"Deactivated display application '{display_app.id}' for datatype '{extension}'.") + if ( + inherit + and (self.datatypes_by_extension[extension], display_app) + in self.inherit_display_application_by_class + ): + self.inherit_display_application_by_class.remove( + (self.datatypes_by_extension[extension], display_app) + ) + self.log.debug( + f"Deactivated display application '{display_app.id}' for datatype '{extension}'." + ) else: self.display_applications[display_app.id] = display_app self.datatypes_by_extension[extension].add_display_application(display_app) - if inherit and (self.datatypes_by_extension[extension], display_app) not in self.inherit_display_application_by_class: - self.inherit_display_application_by_class.append((self.datatypes_by_extension[extension], display_app)) - self.log.debug(f"Loaded display application '{display_app.id}' for datatype '{extension}', inherit={inherit}.") + if ( + inherit + and (self.datatypes_by_extension[extension], display_app) + not in self.inherit_display_application_by_class + ): + self.inherit_display_application_by_class.append( + (self.datatypes_by_extension[extension], display_app) + ) + self.log.debug( + f"Loaded display application '{display_app.id}' for datatype '{extension}', inherit={inherit}." + ) except Exception: if deactivate: self.log.exception(f"Error deactivating display application ({config_path})") @@ -744,7 +843,9 @@ class Registry: self.display_applications[display_application_id].reload() reloaded.append(display_application_id) except Exception as e: - self.log.debug('Requested to reload display application "%s", but failed: %s.', display_application_id, e) + self.log.debug( + 'Requested to reload display application "%s", but failed: %s.', display_application_id, e + ) failed.append(display_application_id) return (reloaded, failed) @@ -753,7 +854,9 @@ class Registry: # We need to be able to add a job to the queue to set metadata. The queue will currently only accept jobs with an associated # tool. We'll load a special tool to be used for Auto-Detecting metadata; this is less than ideal, but effective # Properly building a tool without relying on parsing an XML file is near difficult...so we bundle with Galaxy. - set_meta_tool = toolbox.load_hidden_lib_tool(os.path.abspath(os.path.join(os.path.dirname(__file__), "set_metadata_tool.xml"))) + set_meta_tool = toolbox.load_hidden_lib_tool( + os.path.abspath(os.path.join(os.path.dirname(__file__), "set_metadata_tool.xml")) + ) self.set_external_metadata_tool = set_meta_tool self.log.debug("Loaded external metadata tool: %s", self.set_external_metadata_tool.id) @@ -761,80 +864,80 @@ class Registry: # Default values. if not self.datatypes_by_extension: self.datatypes_by_extension = { - 'ab1': binary.Ab1(), - 'axt': sequence.Axt(), - 'bam': binary.Bam(), - 'jp2': binary.JP2(), - 'bed': interval.Bed(), - 'coverage': coverage.LastzCoverage(), - 'customtrack': interval.CustomTrack(), - 'csfasta': sequence.csFasta(), - 'fasta': sequence.Fasta(), - 'eland': tabular.Eland(), - 'fastq': sequence.Fastq(), - 'fastqsanger': sequence.FastqSanger(), - 'gtf': interval.Gtf(), - 'gff': interval.Gff(), - 'gff3': interval.Gff3(), - 'genetrack': tracks.GeneTrack(), - 'h5': binary.H5(), - 'interval': interval.Interval(), - 'laj': images.Laj(), - 'lav': sequence.Lav(), - 'maf': sequence.Maf(), - 'pileup': tabular.Pileup(), - 'qualsolid': qualityscore.QualityScoreSOLiD(), - 'qualsolexa': qualityscore.QualityScoreSolexa(), - 'qual454': qualityscore.QualityScore454(), - 'sam': tabular.Sam(), - 'scf': binary.Scf(), - 'sff': binary.Sff(), - 'tabular': tabular.Tabular(), - 'csv': tabular.CSV(), - 'taxonomy': tabular.Taxonomy(), - 'txt': data.Text(), - 'wig': interval.Wiggle(), - 'xml': xml.GenericXml(), + "ab1": binary.Ab1(), + "axt": sequence.Axt(), + "bam": binary.Bam(), + "jp2": binary.JP2(), + "bed": interval.Bed(), + "coverage": coverage.LastzCoverage(), + "customtrack": interval.CustomTrack(), + "csfasta": sequence.csFasta(), + "fasta": sequence.Fasta(), + "eland": tabular.Eland(), + "fastq": sequence.Fastq(), + "fastqsanger": sequence.FastqSanger(), + "gtf": interval.Gtf(), + "gff": interval.Gff(), + "gff3": interval.Gff3(), + "genetrack": tracks.GeneTrack(), + "h5": binary.H5(), + "interval": interval.Interval(), + "laj": images.Laj(), + "lav": sequence.Lav(), + "maf": sequence.Maf(), + "pileup": tabular.Pileup(), + "qualsolid": qualityscore.QualityScoreSOLiD(), + "qualsolexa": qualityscore.QualityScoreSolexa(), + "qual454": qualityscore.QualityScore454(), + "sam": tabular.Sam(), + "scf": binary.Scf(), + "sff": binary.Sff(), + "tabular": tabular.Tabular(), + "csv": tabular.CSV(), + "taxonomy": tabular.Taxonomy(), + "txt": data.Text(), + "wig": interval.Wiggle(), + "xml": xml.GenericXml(), } self.mimetypes_by_extension = { - 'ab1': 'application/octet-stream', - 'axt': 'text/plain', - 'bam': 'application/octet-stream', - 'jp2': 'application/octet-stream', - 'bed': 'text/plain', - 'customtrack': 'text/plain', - 'csfasta': 'text/plain', - 'eland': 'application/octet-stream', - 'fasta': 'text/plain', - 'fastq': 'text/plain', - 'fastqsanger': 'text/plain', - 'gtf': 'text/plain', - 'gff': 'text/plain', - 'gff3': 'text/plain', - 'h5': 'application/octet-stream', - 'interval': 'text/plain', - 'laj': 'text/plain', - 'lav': 'text/plain', - 'maf': 'text/plain', - 'memexml': 'application/xml', - 'pileup': 'text/plain', - 'qualsolid': 'text/plain', - 'qualsolexa': 'text/plain', - 'qual454': 'text/plain', - 'sam': 'text/plain', - 'scf': 'application/octet-stream', - 'sff': 'application/octet-stream', - 'tabular': 'text/plain', - 'csv': 'text/plain', - 'taxonomy': 'text/plain', - 'txt': 'text/plain', - 'wig': 'text/plain', - 'xml': 'application/xml', + "ab1": "application/octet-stream", + "axt": "text/plain", + "bam": "application/octet-stream", + "jp2": "application/octet-stream", + "bed": "text/plain", + "customtrack": "text/plain", + "csfasta": "text/plain", + "eland": "application/octet-stream", + "fasta": "text/plain", + "fastq": "text/plain", + "fastqsanger": "text/plain", + "gtf": "text/plain", + "gff": "text/plain", + "gff3": "text/plain", + "h5": "application/octet-stream", + "interval": "text/plain", + "laj": "text/plain", + "lav": "text/plain", + "maf": "text/plain", + "memexml": "application/xml", + "pileup": "text/plain", + "qualsolid": "text/plain", + "qualsolexa": "text/plain", + "qual454": "text/plain", + "sam": "text/plain", + "scf": "application/octet-stream", + "sff": "application/octet-stream", + "tabular": "text/plain", + "csv": "text/plain", + "taxonomy": "text/plain", + "txt": "text/plain", + "wig": "text/plain", + "xml": "application/xml", } # super supertype fix for input steps in workflows. - if 'data' not in self.datatypes_by_extension: - self.datatypes_by_extension['data'] = data.Data() - self.mimetypes_by_extension['data'] = 'application/octet-stream' + if "data" not in self.datatypes_by_extension: + self.datatypes_by_extension["data"] = data.Data() + self.mimetypes_by_extension["data"] = "application/octet-stream" # Default values - the order in which we attempt to determine data types is critical # because some formats are much more flexibly defined than others. if len(self.sniff_order) < 1: @@ -865,7 +968,7 @@ class Registry: interval.Interval(), tabular.Sam(), tabular.Eland(), - tabular.CSV() + tabular.CSV(), ] def get_converters_by_datatype(self, ext): @@ -905,13 +1008,17 @@ class Registry: ext = dataset_or_ext dataset = None - if self.get_datatype_by_extension(ext) is not None and self.get_datatype_by_extension(ext).matches_any(accepted_formats): + if self.get_datatype_by_extension(ext) is not None and self.get_datatype_by_extension(ext).matches_any( + accepted_formats + ): return True, None, None for convert_ext in self.get_converters_by_datatype(ext): convert_ext_datatype = self.get_datatype_by_extension(convert_ext) if convert_ext_datatype is None: - self.log.warning(f"Datatype class not found for extension '{convert_ext}', which is used as target for conversion from datatype '{dataset.ext}'") + self.log.warning( + f"Datatype class not found for extension '{convert_ext}', which is used as target for conversion from datatype '{dataset.ext}'" + ) elif convert_ext_datatype.matches_any(accepted_formats): converted_dataset = dataset and dataset.get_converted_files_by_type(convert_ext) if converted_dataset: @@ -936,31 +1043,32 @@ class Registry: help_txt = meta_spec.desc if not help_txt or help_txt == meta_name: help_txt = "" - inputs.append(f'') + inputs.append( + f'' + ) rval[ext] = "\n".join(inputs) - if 'auto' not in rval and 'txt' in rval: # need to manually add 'auto' datatype - rval['auto'] = rval['txt'] + if "auto" not in rval and "txt" in rval: # need to manually add 'auto' datatype + rval["auto"] = rval["txt"] return rval @property def edam_formats(self): - """ - """ + """ """ if not self._edam_formats_mapping: self._edam_formats_mapping = {k: v.edam_format for k, v in self.datatypes_by_extension.items()} return self._edam_formats_mapping @property def edam_data(self): - """ - """ + """ """ if not self._edam_data_mapping: self._edam_data_mapping = {k: v.edam_data for k, v in self.datatypes_by_extension.items()} return self._edam_data_mapping def to_xml_file(self, path): if not self._registry_xml_string: - registry_string_template = Template(""" + registry_string_template = Template( + """ $datatype_elems @@ -969,16 +1077,19 @@ class Registry: $sniffer_elems - """) - converters_path = self.converters_path_attr or '' - display_path = self.display_path_attr or '' + """ + ) + converters_path = self.converters_path_attr or "" + display_path = self.display_path_attr or "" datatype_elems = "".join(galaxy.util.xml_to_string(elem) for elem in self.datatype_elems) sniffer_elems = "".join(galaxy.util.xml_to_string(elem) for elem in self.sniffer_elems) - self._registry_xml_string = registry_string_template.substitute(converters_path=converters_path, - display_path=display_path, - datatype_elems=datatype_elems, - sniffer_elems=sniffer_elems) - with open(os.path.abspath(path), 'w') as registry_xml: + self._registry_xml_string = registry_string_template.substitute( + converters_path=converters_path, + display_path=display_path, + datatype_elems=datatype_elems, + sniffer_elems=sniffer_elems, + ) + with open(os.path.abspath(path), "w") as registry_xml: os.chmod(path, RW_R__R__) registry_xml.write(self._registry_xml_string) @@ -988,11 +1099,14 @@ class Registry: :param elem: :return extension: """ - extension = elem.get('extension', None) + extension = elem.get("extension", None) # If extension is not None and is uppercase or mixed case, we need to lowercase it if extension is not None and not extension.islower(): - self.log.debug("%s is not lower case, that could cause troubles in the future. \ - Please change it to lower case" % extension) + self.log.debug( + "%s is not lower case, that could cause troubles in the future. \ + Please change it to lower case" + % extension + ) extension = extension.lower() return extension diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index 4e5e6028e2d..ad02b1239fd 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -16,11 +16,12 @@ from markupsafe import escape from galaxy import util from galaxy.datatypes import metadata -from galaxy.datatypes.binary import ( - Binary -) +from galaxy.datatypes.binary import Binary from galaxy.datatypes.data import DatatypeValidation -from galaxy.datatypes.metadata import DictParameter, MetadataElement +from galaxy.datatypes.metadata import ( + DictParameter, + MetadataElement, +) from galaxy.datatypes.sniff import ( build_sniff_from_prefix, FilePrefix, @@ -29,11 +30,9 @@ from galaxy.datatypes.sniff import ( ) from galaxy.util import ( compression_utils, - nice_size -) -from galaxy.util.checkers import ( - is_gzip + nice_size, ) +from galaxy.util.checkers import is_gzip from galaxy.util.image_util import check_image_type from . import data @@ -54,6 +53,7 @@ class SequenceSplitLocations(data.Text): ]} """ + file_ext = "fqtoc" def set_peek(self, dataset): @@ -62,21 +62,21 @@ class SequenceSplitLocations(data.Text): parsed_data = json.load(open(dataset.file_name)) # dataset.peek = json.dumps(data, sort_keys=True, indent=4) dataset.peek = data.get_file_peek(dataset.file_name) - dataset.blurb = '%d sections' % len(parsed_data['sections']) + dataset.blurb = "%d sections" % len(parsed_data["sections"]) except Exception: - dataset.peek = 'Not FQTOC file' - dataset.blurb = 'Not FQTOC file' + dataset.peek = "Not FQTOC file" + dataset.blurb = "Not FQTOC file" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def sniff_prefix(self, file_prefix: FilePrefix): if file_prefix.file_size < 50000 and not file_prefix.truncated: try: data = json.loads(file_prefix.contents_header) - sections = data['sections'] + sections = data["sections"] for section in sections: - if 'start' not in section or 'end' not in section or 'sequences' not in section: + if "start" not in section or "end" not in section or "sequences" not in section: return False return True except Exception: @@ -86,9 +86,12 @@ class SequenceSplitLocations(data.Text): class Sequence(data.Text): """Class describing a sequence""" + edam_data = "data_2044" - MetadataElement(name="sequences", default=0, desc="Number of sequences", readonly=True, visible=False, optional=True, no_value=0) + MetadataElement( + name="sequences", default=0, desc="Number of sequences", readonly=True, visible=False, optional=True, no_value=0 + ) def set_meta(self, dataset, **kwd): """ @@ -99,10 +102,10 @@ class Sequence(data.Text): with compression_utils.get_fileobj(dataset.file_name) as fh: for line in fh: line = line.strip() - if line and line.startswith('#'): + if line and line.startswith("#"): # We don't count comment lines for sequence data types continue - if line and line.startswith('>'): + if line and line.startswith(">"): sequences += 1 data_lines += 1 else: @@ -118,20 +121,20 @@ class Sequence(data.Text): else: dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" @staticmethod def get_sequences_per_file(total_sequences, split_params): - if split_params['split_mode'] == 'number_of_parts': + if split_params["split_mode"] == "number_of_parts": # legacy basic mode - split into a specified number of parts - parts = int(split_params['split_size']) + parts = int(split_params["split_size"]) sequences_per_file = [total_sequences / parts for i in range(parts)] for i in range(total_sequences % parts): sequences_per_file[i] += 1 - elif split_params['split_mode'] == 'to_size': + elif split_params["split_mode"] == "to_size": # loop through the sections and calculate the number of sequences - chunk_size = int(split_params['split_size']) + chunk_size = int(split_params["split_size"]) rem = total_sequences % chunk_size sequences_per_file = [chunk_size for i in range(total_sequences / chunk_size)] # TODO: Should we invest the time in a better way to handle small remainders? @@ -158,10 +161,10 @@ class Sequence(data.Text): @classmethod def do_fast_split(cls, input_datasets, toc_file_datasets, subdir_generator_function, split_params): data = json.load(open(toc_file_datasets[0].file_name)) - sections = data['sections'] + sections = data["sections"] total_sequences = int(0) for section in sections: - total_sequences += int(section['sequences']) + total_sequences += int(section["sequences"]) sequences_per_file = cls.get_sequences_per_file(total_sequences, split_params) return cls.write_split_files(input_datasets, toc_file_datasets, subdir_generator_function, sequences_per_file) @@ -186,14 +189,16 @@ class Sequence(data.Text): ds = input_datasets[ds_no] base_name = os.path.basename(ds.file_name) part_path = os.path.join(dir, base_name) - split_data = dict(class_name=f'{cls.__module__}.{cls.__name__}', - output_name=part_path, - input_name=ds.file_name, - args=dict(start_sequence=start_sequence, num_sequences=sequences_per_file[part_no])) + split_data = dict( + class_name=f"{cls.__module__}.{cls.__name__}", + output_name=part_path, + input_name=ds.file_name, + args=dict(start_sequence=start_sequence, num_sequences=sequences_per_file[part_no]), + ) if toc_file_datasets is not None: toc = toc_file_datasets[ds_no] - split_data['args']['toc_file'] = toc.file_name - with open(os.path.join(dir, f'split_info_{base_name}.json'), 'w') as f: + split_data["args"]["toc_file"] = toc.file_name + with open(os.path.join(dir, f"split_info_{base_name}.json"), "w") as f: json.dump(split_data, f) start_sequence += sequences_per_file[part_no] return directories @@ -223,40 +228,49 @@ class Sequence(data.Text): >>> Sequence.get_split_commands_with_toc('./input.gz', './output.gz', dict(sections=three_sections), start_sequence=5, sequence_count=20) ['(dd bs=1 skip=0 count=74 if=./input.gz 2> /dev/null )| zcat | ( tail -n +21 2> /dev/null) | head -20 | gzip -c >> ./output.gz', 'dd bs=1 skip=74 count=74 if=./input.gz 2> /dev/null >> ./output.gz', '(dd bs=1 skip=148 count=76 if=./input.gz 2> /dev/null )| zcat | ( tail -n +1 2> /dev/null) | head -20 | gzip -c >> ./output.gz'] """ - sections = toc_file['sections'] + sections = toc_file["sections"] result = [] current_sequence = int(0) i = 0 # skip to the section that contains my starting sequence - while i < len(sections) and start_sequence >= current_sequence + int(sections[i]['sequences']): - current_sequence += int(sections[i]['sequences']) + while i < len(sections) and start_sequence >= current_sequence + int(sections[i]["sequences"]): + current_sequence += int(sections[i]["sequences"]) i += 1 if i == len(sections): # bad input data! - raise Exception(f'No FQTOC section contains starting sequence {start_sequence}') + raise Exception(f"No FQTOC section contains starting sequence {start_sequence}") # These two variables act as an accumulator for consecutive entire blocks that # can be copied verbatim (without decompressing) start_chunk = int(-1) end_chunk = int(-1) - copy_chunk_cmd = 'dd bs=1 skip=%s count=%s if=%s 2> /dev/null >> %s' + copy_chunk_cmd = "dd bs=1 skip=%s count=%s if=%s 2> /dev/null >> %s" while sequence_count > 0 and i < len(sections): # we need to extract partial data. So, find the byte offsets of the chunks that contain the data we need # use a combination of dd (to pull just the right sections out) tail (to skip lines) and head (to get the # right number of lines - sequences = int(sections[i]['sequences']) + sequences = int(sections[i]["sequences"]) skip_sequences = start_sequence - current_sequence sequences_to_extract = min(sequence_count, sequences - skip_sequences) - start_copy = int(sections[i]['start']) - end_copy = int(sections[i]['end']) + start_copy = int(sections[i]["start"]) + end_copy = int(sections[i]["end"]) if sequences_to_extract < sequences: if start_chunk > -1: result.append(copy_chunk_cmd % (start_chunk, end_chunk - start_chunk, input_name, output_name)) start_chunk = -1 # extract, unzip, trim, recompress - result.append('(dd bs=1 skip=%s count=%s if=%s 2> /dev/null )| zcat | ( tail -n +%s 2> /dev/null) | head -%s | gzip -c >> %s' % - (start_copy, end_copy - start_copy, input_name, skip_sequences * 4 + 1, sequences_to_extract * 4, output_name)) + result.append( + "(dd bs=1 skip=%s count=%s if=%s 2> /dev/null )| zcat | ( tail -n +%s 2> /dev/null) | head -%s | gzip -c >> %s" + % ( + start_copy, + end_copy - start_copy, + input_name, + skip_sequences * 4 + 1, + sequences_to_extract * 4, + output_name, + ) + ) else: # whole section - add it to the start_chunk/end_chunk accumulator if start_chunk == -1: start_chunk = start_copy @@ -269,7 +283,7 @@ class Sequence(data.Text): result.append(copy_chunk_cmd % (start_chunk, end_chunk - start_chunk, input_name, output_name)) if sequence_count > 0: - raise Exception(f'{sequence_count} sequences not found in file') + raise Exception(f"{sequence_count} sequences not found in file") return result @@ -296,9 +310,12 @@ class Sequence(data.Text): class Alignment(data.Text): """Class describing an alignment""" + edam_data = "data_0863" - MetadataElement(name="species", desc="Species", default=[], param=metadata.SelectParameter, multiple=True, readonly=True) + MetadataElement( + name="species", desc="Species", default=[], param=metadata.SelectParameter, multiple=True, readonly=True + ) def split(cls, input_datasets, subdir_generator_function, split_params): """Split a generic alignment file (not sensible or possible, see subclasses).""" @@ -310,6 +327,7 @@ class Alignment(data.Text): @build_sniff_from_prefix class Fasta(Sequence): """Class representing a FASTA sequence""" + edam_format = "format_1929" file_ext = "fasta" @@ -351,17 +369,17 @@ class Fasta(Sequence): for line in fh: line = line.strip() if line: # first non-empty line - if line.startswith('>'): + if line.startswith(">"): # The next line.strip() must not be '', nor startwith '>' line = fh.readline().strip() - if line == '' or line.startswith('>'): + if line == "" or line.startswith(">"): return False # If there is a third line, and it isn't a header line, it may not contain chars like '()[].' otherwise it's most likely a DotBracket file line = fh.readline() if not line: return True - if not line.startswith('>') and re.search(r"[\(\)\[\]\.]", line): + if not line.startswith(">") and re.search(r"[\(\)\[\]\.]", line): return False return True else: @@ -385,10 +403,10 @@ class Fasta(Sequence): input_file = input_datasets[0].file_name # Counting chunk size as number of sequences. - if 'split_mode' not in split_params: - raise Exception('Tool does not define a split mode') - elif split_params['split_mode'] == 'number_of_parts': - split_size = int(split_params['split_size']) + if "split_mode" not in split_params: + raise Exception("Tool does not define a split mode") + elif split_params["split_mode"] == "number_of_parts": + split_size = int(split_params["split_size"]) log.debug("Split %s into %i parts..." % (input_file, split_size)) # if split_mode = number_of_parts, and split_size = 10, and # we know the number of sequences (say 1234), then divide by @@ -404,10 +422,10 @@ class Fasta(Sequence): # the file size. chunk_size = os.path.getsize(input_file) // split_size cls._size_split(input_file, chunk_size, subdir_generator_function) - elif split_params['split_mode'] == 'to_size': + elif split_params["split_mode"] == "to_size": # Split the input file into as many sub-files as required, # each containing to_size many sequences - batch_size = int(split_params['split_size']) + batch_size = int(split_params["split_size"]) log.debug("Split %s into batches of %i records..." % (input_file, batch_size)) cls._count_split(input_file, batch_size, subdir_generator_function) else: @@ -428,10 +446,10 @@ class Fasta(Sequence): # produce just one sub-file which will be a copy of it. part_dir = subdir_generator_function() part_path = os.path.join(part_dir, os.path.basename(input_file)) - part_file = open(part_path, 'w') + part_file = open(part_path, "w") log.debug(f"Writing {input_file} part to {part_path}") start_offset = 0 - for line in iter(f.readline, ''): + for line in iter(f.readline, ""): offset = f.tell() if not line: break @@ -440,12 +458,12 @@ class Fasta(Sequence): part_file.close() part_dir = subdir_generator_function() part_path = os.path.join(part_dir, os.path.basename(input_file)) - part_file = open(part_path, 'w') + part_file = open(part_path, "w") log.debug(f"Writing {input_file} part to {part_path}") start_offset = f.tell() part_file.write(line) except Exception as e: - log.error('Unable to size split FASTA file: %s', util.unicodify(e)) + log.error("Unable to size split FASTA file: %s", util.unicodify(e)) raise finally: if part_file: @@ -462,7 +480,7 @@ class Fasta(Sequence): # produce just one sub-file which will be a copy of it. part_dir = subdir_generator_function() part_path = os.path.join(part_dir, os.path.basename(input_file)) - part_file = open(part_path, 'w') + part_file = open(part_path, "w") log.debug(f"Writing {input_file} part to {part_path}") rec_count = 0 for line in f: @@ -475,12 +493,12 @@ class Fasta(Sequence): part_file.close() part_dir = subdir_generator_function() part_path = os.path.join(part_dir, os.path.basename(input_file)) - part_file = open(part_path, 'w') + part_file = open(part_path, "w") log.debug(f"Writing {input_file} part to {part_path}") rec_count = 1 part_file.write(line) except Exception as e: - log.error('Unable to count split FASTA file: %s', util.unicodify(e)) + log.error("Unable to count split FASTA file: %s", util.unicodify(e)) raise finally: if part_file: @@ -489,7 +507,8 @@ class Fasta(Sequence): @build_sniff_from_prefix class csFasta(Sequence): - """ Class representing the SOLID Color-Space sequence ( csfasta ) """ + """Class representing the SOLID Color-Space sequence ( csfasta )""" + edam_format = "format_3589" file_ext = "csfasta" @@ -510,14 +529,14 @@ class csFasta(Sequence): fh = file_prefix.string_io() for line in fh: line = line.strip() - if line and not line.startswith('#'): # first non-empty non-comment line - if line.startswith('>'): + if line and not line.startswith("#"): # first non-empty non-comment line + if line.startswith(">"): line = fh.readline().strip() - if line == '' or line.startswith('>'): + if line == "" or line.startswith(">"): break elif line[0] not in string.ascii_uppercase: return False - elif len(line) > 1 and not re.search(r'^[\d.]+$', line[1:]): + elif len(line) > 1 and not re.search(r"^[\d.]+$", line[1:]): return False return True else: @@ -534,13 +553,24 @@ class csFasta(Sequence): @build_sniff_from_prefix class Fastg(Sequence): - """ Class representing a FASTG sequence - http://fastg.sourceforge.net/FASTG_Spec_v1.00.pdf """ + """Class representing a FASTG sequence + http://fastg.sourceforge.net/FASTG_Spec_v1.00.pdf""" + edam_format = "format_3823" file_ext = "fastg" - MetadataElement(name="version", default='1.0', desc="FASTG format version", readonly=True, visible=True, no_value='1.0') - MetadataElement(name="properties", default={}, param=DictParameter, desc="FASTG properites", readonly=True, visible=True, no_value={}) + MetadataElement( + name="version", default="1.0", desc="FASTG format version", readonly=True, visible=True, no_value="1.0" + ) + MetadataElement( + name="properties", + default={}, + param=DictParameter, + desc="FASTG properites", + readonly=True, + visible=True, + no_value={}, + ) def sniff_prefix(self, file_prefix: FilePrefix): """FASTG must begin with lines: @@ -577,13 +607,13 @@ class Fastg(Sequence): break # EOF line = line.strip() if i == 0: - if not line.startswith('#FASTG:begin'): + if not line.startswith("#FASTG:begin"): break - elif line and not line.startswith('#'): # first non-empty non-comment line - if line.startswith('>'): + elif line and not line.startswith("#"): # first non-empty non-comment line + if line.startswith(">"): # The next line.strip() must not be '', nor startwith '>' line = fh.readline().strip() - if line == '' or line.startswith('>'): + if line == "" or line.startswith(">"): break return True else: @@ -597,14 +627,17 @@ class Fastg(Sequence): break # EOF line = line.strip() if i == 0: - if not line.startswith('#FASTG:begin'): + if not line.startswith("#FASTG:begin"): break - if line.startswith('#FASTG'): - props = {x.split('=')[0][1:]: x.split('=')[1] for x in re.findall(':[a-zA-Z0-9_]+=[a-zA-Z0-9_().,\" ]+', line)} + if line.startswith("#FASTG"): + props = { + x.split("=")[0][1:]: x.split("=")[1] + for x in re.findall(':[a-zA-Z0-9_]+=[a-zA-Z0-9_().," ]+', line) + } dataset.metadata.properties.update(props) - if 'version' in props: - dataset.metadata.version = props['version'] - if line and line.startswith('>'): + if "version" in props: + dataset.metadata.version = props["version"] + if line and line.startswith(">"): break if self.max_optional_metadata_filesize >= 0 and dataset.get_size() > self.max_optional_metadata_filesize: dataset.metadata.data_lines = None @@ -619,18 +652,19 @@ class Fastg(Sequence): dataset.blurb = f"{util.commaify(str(dataset.metadata.sequences))} sequences" else: dataset.blurb = nice_size(dataset.get_size()) - dataset.blurb += f'\nversion={dataset.metadata.version}' + dataset.blurb += f"\nversion={dataset.metadata.version}" for k, v in dataset.metadata.properties.items(): - if k != 'version': - dataset.blurb += f'\n{k}={v}' + if k != "version": + dataset.blurb += f"\n{k}={v}" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" @build_sniff_from_prefix class BaseFastq(Sequence): """Base class for FastQ sequences""" + edam_format = "format_1930" file_ext = "fastq" bases_regexp = re.compile(r"^[NGTAC 0123\.]*$", re.IGNORECASE) @@ -647,16 +681,16 @@ class BaseFastq(Sequence): return data_lines = 0 sequences = 0 - seq_counter = 0 # blocks should be 4 lines long + seq_counter = 0 # blocks should be 4 lines long with compression_utils.get_fileobj(dataset.file_name) as in_file: for line in in_file: line = line.strip() - if line and line.startswith('#') and not data_lines: + if line and line.startswith("#") and not data_lines: # We don't count comment lines for sequence data types continue seq_counter += 1 data_lines += 1 - if line and line.startswith('@'): + if line and line.startswith("@"): if seq_counter >= 4: # count previous block # blocks should be 4 lines long @@ -707,7 +741,7 @@ class BaseFastq(Sequence): compressed = file_prefix.compressed_format is not None if compressed and not isinstance(self, Binary): return False - headers = iter_headers(file_prefix, sep='\n', count=1000) + headers = iter_headers(file_prefix, sep="\n", count=1000) # check to see if the base qualities match if not self.quality_check(headers): return False @@ -722,9 +756,12 @@ class BaseFastq(Sequence): mime = "text/plain" self._clean_and_set_mime_type(trans, mime, headers) return fh.read(), headers - return trans.fill_template_mako("/dataset/large_file.mako", - truncated_data=fh.read(max_peek_size), - data=dataset), headers + return ( + trans.fill_template_mako( + "/dataset/large_file.mako", truncated_data=fh.read(max_peek_size), data=dataset + ), + headers, + ) else: return Sequence.display_data(self, trans, dataset, preview, filename, to_ext, **kwd) @@ -743,7 +780,7 @@ class BaseFastq(Sequence): tmp_ds = ds fqtoc_file = None while fqtoc_file is None and tmp_ds is not None: - fqtoc_file = tmp_ds.get_converted_files_by_type('fqtoc') + fqtoc_file = tmp_ds.get_converted_files_by_type("fqtoc") tmp_ds = tmp_ds.copied_from_library_dataset_dataset_association if fqtoc_file is not None: @@ -760,18 +797,22 @@ class BaseFastq(Sequence): to create the input files for the Task. The parameters: data - a dict containing the contents of the split file """ - args = data['args'] - input_name = data['input_name'] - output_name = data['output_name'] - start_sequence = int(args['start_sequence']) - sequence_count = int(args['num_sequences']) + args = data["args"] + input_name = data["input_name"] + output_name = data["output_name"] + start_sequence = int(args["start_sequence"]) + sequence_count = int(args["num_sequences"]) - if 'toc_file' in args: - with open(args['toc_file']) as f: + if "toc_file" in args: + with open(args["toc_file"]) as f: toc_file = json.load(f) - commands = Sequence.get_split_commands_with_toc(input_name, output_name, toc_file, start_sequence, sequence_count) + commands = Sequence.get_split_commands_with_toc( + input_name, output_name, toc_file, start_sequence, sequence_count + ) else: - commands = Sequence.get_split_commands_sequential(is_gzip(input_name), input_name, output_name, start_sequence, sequence_count) + commands = Sequence.get_split_commands_sequential( + is_gzip(input_name), input_name, output_name, start_sequence, sequence_count + ) for cmd in commands: subprocess.check_call(cmd, shell=True) return True @@ -783,12 +824,19 @@ class BaseFastq(Sequence): @classmethod def check_first_block(cls, file_prefix: FilePrefix): # check that first block looks like a fastq block - block = get_headers(file_prefix, sep='\n', count=4) + block = get_headers(file_prefix, sep="\n", count=4) return cls.check_block(block) @classmethod def check_block(cls, block): - if len(block) == 4 and block[0][0] and block[0][0][0] == "@" and block[2][0] and block[2][0][0] == "+" and block[1][0]: + if ( + len(block) == 4 + and block[0][0] + and block[0][0][0] == "@" + and block[2][0] + and block[2][0][0] == "+" + and block[1][0] + ): # Check the sequence line, make sure it contains only G/C/A/T/N match = cls.bases_regexp.match(block[1][0]) if match: @@ -798,12 +846,12 @@ class BaseFastq(Sequence): return False def validate(self, dataset, **kwd): - headers = iter_headers(dataset.file_name, sep='\n', count=-1) + headers = iter_headers(dataset.file_name, sep="\n", count=-1) # check to see if the base qualities match if not self.quality_check(headers): return DatatypeValidation.invalid("Invalid quality score(s) found for this fastq datatype.") - headers = iter_headers(dataset.file_name, sep='\n', count=-1) + headers = iter_headers(dataset.file_name, sep="\n", count=-1) while True: block = list(islice(headers, 4)) if len(block) == 0: @@ -816,6 +864,7 @@ class BaseFastq(Sequence): class Fastq(BaseFastq): """Class representing a generic FASTQ sequence""" + edam_format = "format_1930" file_ext = "fastq" @@ -825,6 +874,7 @@ class FastqSanger(Fastq): phred scored quality values 0:50 represented by ASCII 33:83 """ + edam_format = "format_1932" file_ext = "fastqsanger" bases_regexp = re.compile("^[NGTAC]*$", re.IGNORECASE) @@ -836,7 +886,7 @@ class FastqSanger(Fastq): return True if the qualities are compatible with sanger encoding """ for line in islice(lines, 3, None, 4): - if not all(q >= '!' and q <= 'S' for q in line[0]): + if not all(q >= "!" and q <= "S" for q in line[0]): return False return True @@ -846,6 +896,7 @@ class FastqSolexa(Fastq): solexa scored quality values -5:40 represented by ASCII 59:104 """ + edam_format = "format_1933" file_ext = "fastqsolexa" @@ -856,7 +907,7 @@ class FastqSolexa(Fastq): return True if the qualities are compatible with sanger encoding """ for line in islice(lines, 3, None, 4): - if not all(q >= ';' and q <= 'h' for q in line[0]): + if not all(q >= ";" and q <= "h" for q in line[0]): return False return True @@ -871,6 +922,7 @@ class FastqIllumina(Fastq): phred scored quality values 0:40 represented by ASCII 64:104 """ + edam_format = "format_1931" file_ext = "fastqillumina" @@ -881,7 +933,7 @@ class FastqIllumina(Fastq): return True if the qualities are compatible with sanger encoding """ for line in islice(lines, 3, None, 4): - if not all(q >= '@' and q <= 'h' for q in line[0]): + if not all(q >= "@" and q <= "h" for q in line[0]): return False return True @@ -897,6 +949,7 @@ class FastqCSSanger(Fastq): sequence in in color space phred scored quality values 0:93 represented by ASCII 33:126 """ + file_ext = "fastqcssanger" bases_regexp = re.compile(r"^[NGTAC][0123\.]*$", re.IGNORECASE) @@ -904,13 +957,30 @@ class FastqCSSanger(Fastq): @build_sniff_from_prefix class Maf(Alignment): """Class describing a Maf alignment""" + edam_format = "format_3008" file_ext = "maf" # Readonly and optional, users can't unset it, but if it is not set, we are generally ok; if required use a metadata validator in the tool definition - MetadataElement(name="blocks", default=0, desc="Number of blocks", readonly=True, optional=True, visible=False, no_value=0) - MetadataElement(name="species_chromosomes", desc="Species Chromosomes", param=metadata.FileParameter, readonly=True, visible=False, optional=True) - MetadataElement(name="maf_index", desc="MAF Index File", param=metadata.FileParameter, readonly=True, visible=False, optional=True) + MetadataElement( + name="blocks", default=0, desc="Number of blocks", readonly=True, optional=True, visible=False, no_value=0 + ) + MetadataElement( + name="species_chromosomes", + desc="Species Chromosomes", + param=metadata.FileParameter, + readonly=True, + visible=False, + optional=True, + ) + MetadataElement( + name="maf_index", + desc="MAF Index File", + param=metadata.FileParameter, + readonly=True, + visible=False, + optional=True, + ) def init_meta(self, dataset, copy_from=None): Alignment.init_meta(self, dataset, copy_from=copy_from) @@ -922,6 +992,7 @@ class Maf(Alignment): # these metadata values are not accessable by users, always overwrite # Imported here to avoid circular dependency from galaxy.tools.util.maf_utilities import build_maf_index_species_chromosomes + indexes, species, species_chromosomes, blocks = build_maf_index_species_chromosomes(dataset.file_name) if indexes is None: return # this is not a MAF file @@ -931,16 +1002,16 @@ class Maf(Alignment): # write species chromosomes to a file chrom_file = dataset.metadata.species_chromosomes if not chrom_file: - chrom_file = dataset.metadata.spec['species_chromosomes'].param.new_file(dataset=dataset) - with open(chrom_file.file_name, 'w') as chrom_out: + chrom_file = dataset.metadata.spec["species_chromosomes"].param.new_file(dataset=dataset) + with open(chrom_file.file_name, "w") as chrom_out: for spec, chroms in species_chromosomes.items(): chrom_out.write("{}\t{}\n".format(spec, "\t".join(chroms))) dataset.metadata.species_chromosomes = chrom_file index_file = dataset.metadata.maf_index if not index_file: - index_file = dataset.metadata.spec['maf_index'].param.new_file(dataset=dataset) - indexes.write(open(index_file.file_name, 'wb')) + index_file = dataset.metadata.spec["maf_index"].param.new_file(dataset=dataset) + indexes.write(open(index_file.file_name, "wb")) dataset.metadata.maf_index = index_file def set_peek(self, dataset): @@ -954,8 +1025,8 @@ class Maf(Alignment): # needed to set metadata dataset.blurb = "? blocks" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): """Returns formated html of peek""" @@ -966,10 +1037,10 @@ class Maf(Alignment): skipchars = skipchars or [] out = [''] try: - out.append('') + out.append(f"{species} ") + out.append("") if not dataset.peek: dataset.set_peek() data = dataset.peek @@ -978,8 +1049,8 @@ class Maf(Alignment): line = line.strip() if not line: continue - out.append(f'') - out.append('
              Species: ') + out.append("
              Species: ") for species in dataset.metadata.species: - out.append(f'{species} ') - out.append('
              {escape(line)}
              ') + out.append(f"{escape(line)}") + out.append("") out = "".join(out) except Exception as exc: out = f"Can't create peek {exc}" @@ -1021,9 +1092,21 @@ class Maf(Alignment): class MafCustomTrack(data.Text): file_ext = "mafcustomtrack" - MetadataElement(name="vp_chromosome", default='chr1', desc="Viewport Chromosome", readonly=True, optional=True, visible=False, no_value='') - MetadataElement(name="vp_start", default='1', desc="Viewport Start", readonly=True, optional=True, visible=False, no_value='') - MetadataElement(name="vp_end", default='100', desc="Viewport End", readonly=True, optional=True, visible=False, no_value='') + MetadataElement( + name="vp_chromosome", + default="chr1", + desc="Viewport Chromosome", + readonly=True, + optional=True, + visible=False, + no_value="", + ) + MetadataElement( + name="vp_start", default="1", desc="Viewport Start", readonly=True, optional=True, visible=False, no_value="" + ) + MetadataElement( + name="vp_end", default="100", desc="Viewport End", readonly=True, optional=True, visible=False, no_value="" + ) def set_meta(self, dataset, overwrite=True, **kwd): """ @@ -1059,6 +1142,7 @@ class MafCustomTrack(data.Text): @build_sniff_from_prefix class Axt(data.Text): """Class describing an axt alignment""" + # gvk- 11/19/09 - This is really an alignment, but we no longer have tools that use this data type, and it is # here simply for backward compatibility ( although it is still in the datatypes registry ). Subclassing # from data.Text eliminates managing metadata elements inherited from the Alignemnt class. @@ -1116,6 +1200,7 @@ class Axt(data.Text): @build_sniff_from_prefix class Lav(data.Text): """Class describing a LAV alignment""" + # gvk- 11/19/09 - This is really an alignment, but we no longer have tools that use this data type, and it is # here simply for backward compatibility ( although it is still in the datatypes registry ). Subclassing # from data.Text eliminates managing metadata elements inherited from the Alignment class. @@ -1143,7 +1228,7 @@ class Lav(data.Text): """ headers = get_headers(file_prefix, None) try: - if len(headers) > 1 and headers[0][0] and headers[0][0].startswith('#:lav'): + if len(headers) > 1 and headers[0][0] and headers[0][0].startswith("#:lav"): return True else: return False @@ -1157,15 +1242,15 @@ class RNADotPlotMatrix(data.Data): def set_peek(self, dataset): if not dataset.dataset.purged: - dataset.peek = 'RNA Dot Plot format (Postscript derivative)' + dataset.peek = "RNA Dot Plot format (Postscript derivative)" dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def sniff(self, filename): """Determine if the file is in RNA dot plot format.""" - if check_image_type(filename, ['EPS']): + if check_image_type(filename, ["EPS"]): seq = False coor = False pairs = False @@ -1173,11 +1258,11 @@ class RNADotPlotMatrix(data.Data): for line in handle: line = line.strip() if line: - if line.startswith('/sequence'): + if line.startswith("/sequence"): seq = True - elif line.startswith('/coor'): + elif line.startswith("/coor"): coor = True - elif line.startswith('/pairs'): + elif line.startswith("/pairs"): pairs = True if seq and coor and pairs: return True @@ -1211,7 +1296,7 @@ class DotBracket(Sequence): line = line.strip() data_lines += 1 - if line and line.startswith('>'): + if line and line.startswith(">"): sequences += 1 dataset.metadata.data_lines = data_lines @@ -1245,7 +1330,7 @@ class DotBracket(Sequence): * Sniffing is only applied on the first entry. * Empty lines are allowed. - """ + """ state = 0 @@ -1255,7 +1340,7 @@ class DotBracket(Sequence): if line: # header line if state == 0: - if(line[0] != '>'): + if line[0] != ">": return False else: state = 1 @@ -1270,10 +1355,13 @@ class DotBracket(Sequence): # dot-bracket structure line elif state == 2: - if sequence_size != len(line) or not self.structure_regexp.match(line) or \ - line.count('(') != line.count(')') or \ - line.count('[') != line.count(']') or \ - line.count('{') != line.count('}'): + if ( + sequence_size != len(line) + or not self.structure_regexp.match(line) + or line.count("(") != line.count(")") + or line.count("[") != line.count("]") + or line.count("{") != line.count("}") + ): return False else: return True @@ -1285,6 +1373,7 @@ class DotBracket(Sequence): @build_sniff_from_prefix class Genbank(data.Text): """Class representing a Genbank sequence""" + edam_format = "format_1936" edam_data = "data_0849" file_ext = "genbank" @@ -1302,12 +1391,13 @@ class Genbank(data.Text): compressed = file_prefix.compressed_format if compressed and not isinstance(self, Binary): return False - return 'LOCUS ' == file_prefix.contents_header[0:6] + return "LOCUS " == file_prefix.contents_header[0:6] @build_sniff_from_prefix class MemePsp(Sequence): """Class representing MEME Position Specific Priors""" + file_ext = "memepsp" def sniff_prefix(self, file_prefix: FilePrefix): @@ -1327,6 +1417,7 @@ class MemePsp(Sequence): >>> MemePsp().sniff(fname) False """ + def floats_verified(line): for item in line.split(): try: @@ -1338,6 +1429,7 @@ class MemePsp(Sequence): except ValueError: return True return False + num_lines = 0 fh = file_prefix.string_io() got_header = False @@ -1350,11 +1442,11 @@ class MemePsp(Sequence): num_lines += 1 line = line.strip() if line: - if line.startswith('>'): + if line.startswith(">"): got_header = True # The line must not be blank, nor start with '>' line = fh.readline().strip() - if line == '' or line.startswith('>'): + if line == "" or line.startswith(">"): return False # All items within the line must be floats. if not floats_verified(line): diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 40341e0bc6b..840b499dd60 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -29,7 +29,7 @@ from galaxy.files import ConfiguredFileSources from galaxy.util import ( compression_utils, file_reader, - stream_to_open_named_file + stream_to_open_named_file, ) from galaxy.util.checkers import ( check_binary, @@ -40,13 +40,13 @@ from galaxy.util.checkers import ( log = logging.getLogger(__name__) -SNIFF_PREFIX_BYTES = int(os.environ.get("GALAXY_SNIFF_PREFIX_BYTES", None) or 2 ** 20) +SNIFF_PREFIX_BYTES = int(os.environ.get("GALAXY_SNIFF_PREFIX_BYTES", None) or 2**20) def get_test_fname(fname): """Returns test data filename""" path, name = os.path.split(__file__) - full_path = os.path.join(path, 'test', fname) + full_path = os.path.join(path, "test", fname) return full_path @@ -67,12 +67,16 @@ def stream_url_to_file(path: str, file_sources: Optional[ConfiguredFileSources] file_source_path.file_source.realize_to(file_source_path.path, temp_name) return temp_name else: - page = urllib.request.urlopen(path, timeout=util.DEFAULT_SOCKET_TIMEOUT) # page will be .close()ed in stream_to_file - temp_name = stream_to_file(page, prefix=prefix, source_encoding=util.get_charset_from_http_headers(page.headers)) + page = urllib.request.urlopen( + path, timeout=util.DEFAULT_SOCKET_TIMEOUT + ) # page will be .close()ed in stream_to_file + temp_name = stream_to_file( + page, prefix=prefix, source_encoding=util.get_charset_from_http_headers(page.headers) + ) return temp_name -def stream_to_file(stream, suffix='', prefix='', dir=None, text=False, **kwd): +def stream_to_file(stream, suffix="", prefix="", dir=None, text=False, **kwd): """Writes a stream to a temporary file, returns the temporary file's name""" fd, temp_name = tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=dir, text=text) return stream_to_open_named_file(stream, fd, temp_name, **kwd) @@ -80,7 +84,7 @@ def stream_to_file(stream, suffix='', prefix='', dir=None, text=False, **kwd): def handle_composite_file(datatype, src_path, extra_files, name, is_binary, tmp_dir, tmp_prefix, upload_opts): if not is_binary: - if upload_opts.get('space_to_tab'): + if upload_opts.get("space_to_tab"): convert_newlines_sep2tabs(src_path, tmp_dir=tmp_dir, tmp_prefix=tmp_prefix) else: convert_newlines(src_path, tmp_dir=tmp_dir, tmp_prefix=tmp_prefix) @@ -101,12 +105,20 @@ class ConvertResult(NamedTuple): class ConvertFunction(Protocol): - - def __call__(self, fname: str, in_place: bool = True, tmp_dir: Optional[str] = None, tmp_prefix: Optional[str] = "gxupload") -> ConvertResult: + def __call__( + self, fname: str, in_place: bool = True, tmp_dir: Optional[str] = None, tmp_prefix: Optional[str] = "gxupload" + ) -> ConvertResult: ... -def convert_newlines(fname: str, in_place: bool = True, tmp_dir: Optional[str] = None, tmp_prefix: Optional[str] = "gxupload", block_size: int = 128 * 1024, regexp=None) -> ConvertResult: +def convert_newlines( + fname: str, + in_place: bool = True, + tmp_dir: Optional[str] = None, + tmp_prefix: Optional[str] = "gxupload", + block_size: int = 128 * 1024, + regexp=None, +) -> ConvertResult: """ Converts in place a file from universal line endings to Posix line endings. @@ -116,7 +128,9 @@ def convert_newlines(fname: str, in_place: bool = True, tmp_dir: Optional[str] = converted_regex = False NEWLINE_BYTE = 10 CR_BYTE = 13 - with tempfile.NamedTemporaryFile(mode='wb', prefix=tmp_prefix, dir=tmp_dir, delete=False) as fp, open(fname, mode='rb') as fi: + with tempfile.NamedTemporaryFile(mode="wb", prefix=tmp_prefix, dir=tmp_dir, delete=False) as fp, open( + fname, mode="rb" + ) as fi: last_char = None block = fi.read(block_size) last_block = b"" @@ -151,16 +165,24 @@ def convert_newlines(fname: str, in_place: bool = True, tmp_dir: Optional[str] = return ConvertResult(i, fp.name, converted_newlines, converted_regex) -def convert_sep2tabs(fname: str, in_place: bool = True, tmp_dir: Optional[str] = None, tmp_prefix: Optional[str] = "gxupload", block_size: int = 128 * 1024): +def convert_sep2tabs( + fname: str, + in_place: bool = True, + tmp_dir: Optional[str] = None, + tmp_prefix: Optional[str] = "gxupload", + block_size: int = 128 * 1024, +): """ Transforms in place a 'sep' separated file to a tab separated one """ - patt: bytes = br"[^\S\r\n]+" + patt: bytes = rb"[^\S\r\n]+" regexp = re.compile(patt) i = 0 converted_newlines = False converted_regex = False - with tempfile.NamedTemporaryFile(mode='wb', prefix=tmp_prefix, dir=tmp_dir, delete=False) as fp, open(fname, mode='rb') as fi: + with tempfile.NamedTemporaryFile(mode="wb", prefix=tmp_prefix, dir=tmp_dir, delete=False) as fp, open( + fname, mode="rb" + ) as fi: block = fi.read(block_size) while block: if block: @@ -179,11 +201,13 @@ def convert_sep2tabs(fname: str, in_place: bool = True, tmp_dir: Optional[str] = return ConvertResult(i, fp.name, converted_newlines, converted_regex) -def convert_newlines_sep2tabs(fname: str, in_place: bool = True, tmp_dir: Optional[str] = None, tmp_prefix: Optional[str] = "gxupload") -> ConvertResult: +def convert_newlines_sep2tabs( + fname: str, in_place: bool = True, tmp_dir: Optional[str] = None, tmp_prefix: Optional[str] = "gxupload" +) -> ConvertResult: """ Converts newlines in a file to posix newlines and replaces spaces with tabs. """ - patt: bytes = br"[^\S\n]+" + patt: bytes = rb"[^\S\n]+" regexp = re.compile(patt) return convert_newlines(fname, in_place, tmp_dir, tmp_prefix, regexp=regexp) @@ -195,8 +219,8 @@ def iter_headers(fname_or_file_prefix, sep, count=60, comment_designator=None): else: file_iterator = compression_utils.get_fileobj(fname_or_file_prefix) for line in file_iterator: - line = line.rstrip('\n\r') - if comment_designator is not None and comment_designator != '' and line.startswith(comment_designator): + line = line.rstrip("\n\r") + if comment_designator is not None and comment_designator != "" and line.startswith(comment_designator): continue yield line.split(sep) idx += 1 @@ -221,10 +245,14 @@ def get_headers(fname_or_file_prefix, sep, count=60, comment_designator=None): >>> get_headers(fname, '\\t', count=5, comment_designator='#') == [[''], ['chr7', 'bed2gff', 'AR', '26731313', '26731437', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731491', '26731536', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731541', '26731649', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731659', '26731841', '.', '+', '.', 'score']] True """ - return list(iter_headers(fname_or_file_prefix=fname_or_file_prefix, sep=sep, count=count, comment_designator=comment_designator)) + return list( + iter_headers( + fname_or_file_prefix=fname_or_file_prefix, sep=sep, count=count, comment_designator=comment_designator + ) + ) -def is_column_based(fname_or_file_prefix, sep='\t', skip=0): +def is_column_based(fname_or_file_prefix, sep="\t", skip=0): """ Checks whether the file is column based with respect to a separator (defaults to tab separator). @@ -256,14 +284,14 @@ def is_column_based(fname_or_file_prefix, sep='\t', skip=0): return False try: - headers = get_headers(fname_or_file_prefix, sep, comment_designator='#')[skip:] + headers = get_headers(fname_or_file_prefix, sep, comment_designator="#")[skip:] except UnicodeDecodeError: return False count = 0 if not headers: return False for hdr in headers: - if hdr and hdr != ['']: + if hdr and hdr != [""]: if count: if len(hdr) != count: return False @@ -492,26 +520,25 @@ def guess_ext(fname, sniff_order, is_binary=False): # Ugly hack for tsv vs tabular sniffing, we want to prefer tabular # to tsv but it doesn't have a sniffer - is TSV was sniffed just check # if it is an okay tabular and use that instead. - if file_ext == 'tsv': - if is_column_based(file_prefix, '\t', 1): - file_ext = 'tabular' + if file_ext == "tsv": + if is_column_based(file_prefix, "\t", 1): + file_ext = "tabular" if file_ext is not None: return file_ext # skip header check if data is already known to be binary if is_binary: - return file_ext or 'binary' + return file_ext or "binary" try: get_headers(file_prefix, None) except UnicodeDecodeError: - return 'data' # default data type file extension - if is_column_based(file_prefix, '\t', 1): - return 'tabular' # default tabular data type file extension - return 'txt' # default text data type file extension + return "data" # default data type file extension + if is_column_based(file_prefix, "\t", 1): + return "tabular" # default tabular data type file extension + return "txt" # default text data type file extension class FilePrefix: - def __init__(self, filename): non_utf8_error = None compressed_format = None @@ -564,7 +591,7 @@ class FilePrefix: def line_iterator(self): s = self.string_io() s_len = len(s.getvalue()) - for line in iter(s.readline, ''): + for line in iter(s.readline, ""): if line.endswith("\n") or line.endswith("\r"): yield line elif s.tell() == s_len and not self.truncated: @@ -600,8 +627,7 @@ def _get_file_prefix(filename_or_file_prefix: Union[str, FilePrefix]) -> FilePre def run_sniffers_raw(filename_or_file_prefix: Union[str, FilePrefix], sniff_order, is_binary=False): - """Run through sniffers specified by sniff_order, return None of None match. - """ + """Run through sniffers specified by sniff_order, return None of None match.""" file_prefix = _get_file_prefix(filename_or_file_prefix) fname = file_prefix.filename file_ext = None @@ -643,7 +669,7 @@ def run_sniffers_raw(filename_or_file_prefix: Union[str, FilePrefix], sniff_orde def zip_single_fileobj(path): z = zipfile.ZipFile(path) for name in z.namelist(): - if not name.endswith('/'): + if not name.endswith("/"): return z.open(name) @@ -684,14 +710,14 @@ class HandleCompressedFileResponse(NamedTuple): def handle_compressed_file( - filename: str, - datatypes_registry, - ext: str = 'auto', - tmp_prefix: Optional[str] = 'sniff_uncompress_', - tmp_dir: Optional[str] = None, - in_place: bool = False, - check_content: bool = True, - auto_decompress: bool = True, + filename: str, + datatypes_registry, + ext: str = "auto", + tmp_prefix: Optional[str] = "sniff_uncompress_", + tmp_dir: Optional[str] = None, + in_place: bool = False, + check_content: bool = True, + auto_decompress: bool = True, ) -> HandleCompressedFileResponse: """ Check uploaded files for compression, check compressed file contents, and uncompress if necessary. @@ -708,7 +734,7 @@ def handle_compressed_file( in the case of a zip file), this is so lengthy decompression can be bypassed if there is invalid content in the first 32KB. Otherwise the caller should be checking content. """ - CHUNK_SIZE = 2 ** 20 # 1Mb + CHUNK_SIZE = 2**20 # 1Mb is_compressed = False compressed_type = None keep_compressed = False @@ -723,14 +749,14 @@ def handle_compressed_file( if is_compressed and is_valid: if ext in AUTO_DETECT_EXTENSIONS: # attempt to sniff for a keep-compressed datatype (observing the sniff order) - sniff_datatypes = filter(lambda d: getattr(d, 'compressed', False), datatypes_registry.sniff_order) + sniff_datatypes = filter(lambda d: getattr(d, "compressed", False), datatypes_registry.sniff_order) sniffed_ext = run_sniffers_raw(filename, sniff_datatypes) if sniffed_ext: ext = sniffed_ext keep_compressed = True else: datatype = datatypes_registry.get_datatype_by_extension(ext) - keep_compressed = getattr(datatype, 'compressed', False) + keep_compressed = getattr(datatype, "compressed", False) # don't waste time decompressing if we sniff invalid contents if is_compressed and is_valid and auto_decompress and not keep_compressed: assert compressed_type # Tell type checker is_compressed will only be true if compressed_type is also set. @@ -744,7 +770,11 @@ def handle_compressed_file( uncompressed.write(chunk) except OSError as e: os.remove(uncompressed.name) - raise OSError('Problem uncompressing {} data, please try retrieving the data uncompressed: {}'.format(compressed_type, util.unicodify(e))) + raise OSError( + "Problem uncompressing {} data, please try retrieving the data uncompressed: {}".format( + compressed_type, util.unicodify(e) + ) + ) uncompressed_path = uncompressed.name if in_place: # Replace the compressed file with the uncompressed file @@ -780,18 +810,18 @@ def convert_function(convert_to_posix_lines, convert_spaces_to_tabs) -> ConvertF def handle_uploaded_dataset_file_internal( - filename: str, - datatypes_registry, - ext: str = 'auto', - tmp_prefix: Optional[str] = 'sniff_upload_', - tmp_dir: Optional[str] = None, - in_place: bool = False, - check_content: bool = True, - is_binary: Optional[bool] = None, - auto_decompress: bool = True, - uploaded_file_ext: Optional[str] = None, - convert_to_posix_lines: Optional[bool] = None, - convert_spaces_to_tabs: Optional[bool] = None, + filename: str, + datatypes_registry, + ext: str = "auto", + tmp_prefix: Optional[str] = "sniff_upload_", + tmp_dir: Optional[str] = None, + in_place: bool = False, + check_content: bool = True, + is_binary: Optional[bool] = None, + auto_decompress: bool = True, + uploaded_file_ext: Optional[str] = None, + convert_to_posix_lines: Optional[bool] = None, + convert_spaces_to_tabs: Optional[bool] = None, ) -> HandleUploadedDatasetFileInternalResponse: is_valid, ext, converted_path, compressed_type = handle_compressed_file( filename, @@ -808,8 +838,8 @@ def handle_uploaded_dataset_file_internal( try: if not is_valid: if is_tar(converted_path): - raise InappropriateDatasetContentError('TAR file uploads are not supported') - raise InappropriateDatasetContentError('The uploaded compressed file contains invalid content') + raise InappropriateDatasetContentError("TAR file uploads are not supported") + raise InappropriateDatasetContentError("The uploaded compressed file contains invalid content") # This needs to be checked again after decompression is_binary = check_binary(converted_path) @@ -825,7 +855,9 @@ def handle_uploaded_dataset_file_internal( if not is_binary and (convert_to_posix_lines or convert_spaces_to_tabs): # Convert universal line endings to Posix line endings, spaces to tabs (if desired) convert_fxn = convert_function(convert_to_posix_lines, convert_spaces_to_tabs) - line_count, _converted_path, converted_newlines, converted_spaces = convert_fxn(converted_path, in_place=in_place, tmp_dir=tmp_dir, tmp_prefix=tmp_prefix) + line_count, _converted_path, converted_newlines, converted_spaces = convert_fxn( + converted_path, in_place=in_place, tmp_dir=tmp_dir, tmp_prefix=tmp_prefix + ) if not in_place: if converted_path and filename != converted_path: os.unlink(converted_path) @@ -837,19 +869,20 @@ def handle_uploaded_dataset_file_internal( ext = guessed_ext if not is_binary and check_content and check_html(converted_path): - raise InappropriateDatasetContentError('The uploaded file contains invalid HTML content') + raise InappropriateDatasetContentError("The uploaded file contains invalid HTML content") except Exception: if filename != converted_path: os.unlink(converted_path) raise - return HandleUploadedDatasetFileInternalResponse(ext, converted_path, compressed_type, converted_newlines, converted_spaces) + return HandleUploadedDatasetFileInternalResponse( + ext, converted_path, compressed_type, converted_newlines, converted_spaces + ) -AUTO_DETECT_EXTENSIONS = ['auto'] # should 'data' also cause auto detect? +AUTO_DETECT_EXTENSIONS = ["auto"] # should 'data' also cause auto detect? class Decompress(Protocol): - def __call__(self, path: str) -> IO[bytes]: ... @@ -861,6 +894,7 @@ class InappropriateDatasetContentError(Exception): pass -if __name__ == '__main__': +if __name__ == "__main__": import doctest + doctest.testmod(sys.modules[__name__]) diff --git a/lib/galaxy/datatypes/spaln.py b/lib/galaxy/datatypes/spaln.py index 19ad30c083d..528def8473b 100644 --- a/lib/galaxy/datatypes/spaln.py +++ b/lib/galaxy/datatypes/spaln.py @@ -54,12 +54,8 @@ class _SpalnDb(Data): def generate_primary_file(self, dataset=None): rval = ["Spaln Database

              "] - rval.append( - "

              This composite dataset is composed of the following files:

                " - ) - for composite_name, composite_file in self.get_composite_files( - dataset=dataset - ).items(): + rval.append("
                This composite dataset is composed of the following files:

                  ") + for composite_name, composite_file in self.get_composite_files(dataset=dataset).items(): fn = composite_name opt_text = "" if composite_file.get("description"): @@ -68,10 +64,7 @@ class _SpalnDb(Data): % (fn, fn, composite_file.get("description"), opt_text) ) else: - rval.append( - '
                • %s%s
                • ' - % (fn, fn, opt_text) - ) + rval.append('
                • %s%s
                • ' % (fn, fn, opt_text)) rval.append("
                ") return "\n".join(rval) @@ -110,17 +103,7 @@ class _SpalnDb(Data): except Exception: return "spaln database (multiple files)" - def display_data( - self, - trans, - data, - preview=False, - filename=None, - to_ext=None, - size=None, - offset=None, - **kwd - ): + def display_data(self, trans, data, preview=False, filename=None, to_ext=None, size=None, offset=None, **kwd): """ If preview is `True` allows us to format the data shown in the central pane via the "eye" icon. If preview is `False` triggers download. @@ -136,7 +119,7 @@ class _SpalnDb(Data): size=size, offset=offset, headers=headers, - **kwd + **kwd, ) if self.file_ext == "spalndbn": title = "This is a nucleotide-query spaln database" @@ -157,10 +140,10 @@ class _SpalnDb(Data): if not msg: msg = title # Galaxy assumes HTML for the display of composite datatypes, - return smart_str( - "%s
                %s
                " - % (title, msg) - ), headers + return ( + smart_str("%s
                %s
                " % (title, msg)), + headers, + ) def merge(split_files, output_file): """Merge spaln databases (not implemented).""" diff --git a/lib/galaxy/datatypes/speech.py b/lib/galaxy/datatypes/speech.py index b45eee0336f..6be6a96fc94 100644 --- a/lib/galaxy/datatypes/speech.py +++ b/lib/galaxy/datatypes/speech.py @@ -1,4 +1,7 @@ -from galaxy.datatypes.metadata import ListParameter, MetadataElement +from galaxy.datatypes.metadata import ( + ListParameter, + MetadataElement, +) from galaxy.datatypes.sniff import get_headers from galaxy.datatypes.text import Text @@ -21,7 +24,16 @@ class TextGrid(Text): blurb = "Praat TextGrid file" - MetadataElement(name="annotations", default=[], desc="Annotation types", param=ListParameter, readonly=True, visible=True, optional=True, no_value=[]) + MetadataElement( + name="annotations", + default=[], + desc="Annotation types", + param=ListParameter, + readonly=True, + visible=True, + optional=True, + no_value=[], + ) def sniff(self, filename): @@ -49,9 +61,41 @@ class BPF(Text): file_ext = "par" - MetadataElement(name="annotations", default=[], desc="Annotation types", param=ListParameter, readonly=True, visible=True, optional=True, no_value=[]) - mandatory_headers = ['LHD', 'REP', 'SNB', 'SAM', 'SBF', 'SSB', 'NCH', 'SPN', 'LBD'] - optional_headers = ['FIL', 'TYP', 'DBN', 'VOL', 'DIR', 'SRC', 'BEG', 'END', 'RED', 'RET', 'RCC', 'CMT', 'SPI', 'PCF', 'PCN', 'EXP', 'SYS', 'DAT', 'SPA', 'MAO', 'GPO', 'SAO'] + MetadataElement( + name="annotations", + default=[], + desc="Annotation types", + param=ListParameter, + readonly=True, + visible=True, + optional=True, + no_value=[], + ) + mandatory_headers = ["LHD", "REP", "SNB", "SAM", "SBF", "SSB", "NCH", "SPN", "LBD"] + optional_headers = [ + "FIL", + "TYP", + "DBN", + "VOL", + "DIR", + "SRC", + "BEG", + "END", + "RED", + "RET", + "RCC", + "CMT", + "SPI", + "PCF", + "PCN", + "EXP", + "SYS", + "DAT", + "SPA", + "MAO", + "GPO", + "SAO", + ] def set_meta(self, dataset, overwrite=True, **kwd): """Set the metadata for this dataset from the file contents""" @@ -59,7 +103,7 @@ class BPF(Text): with open(dataset.dataset.file_name) as fd: for line in fd: # Split the line on a colon rather than regexing it - parts = line.split(':') + parts = line.split(":") # And if the first part is a 3 character string, then it's # interesting. @@ -74,12 +118,12 @@ class BPF(Text): # We loop over 30 as there are 9 mandatory headers (the last should be # `LBD:`), while there are 21 optional headers that can be # interspersed. - seen_headers = [line[0] for line in get_headers(filename, sep=':', count=40)] + seen_headers = [line[0] for line in get_headers(filename, sep=":", count=40)] # We cut everything after LBD, where the headers end and contents # start. We choose not to validate contents. - if 'LBD' in seen_headers: - seen_headers = seen_headers[0:seen_headers.index('LBD') + 1] + if "LBD" in seen_headers: + seen_headers = seen_headers[0 : seen_headers.index("LBD") + 1] # Check that every mandatory header is present in the seen headers for header in self.mandatory_headers: diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 2bf9a7a6de0..4fbb2100493 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -17,7 +17,11 @@ import pysam from markupsafe import escape from galaxy import util -from galaxy.datatypes import binary, data, metadata +from galaxy.datatypes import ( + binary, + data, + metadata, +) from galaxy.datatypes.binary import _BamOrSam from galaxy.datatypes.metadata import ( MetadataElement, @@ -41,18 +45,41 @@ MAX_DATA_LINES = 100000 @dataproviders.decorators.has_dataproviders class TabularData(data.Text): """Generic tabular data""" + edam_format = "format_3475" # All tabular data is chunkable. CHUNKABLE = True data_line_offset = 0 max_peek_columns = 50 - MetadataElement(name="comment_lines", default=0, desc="Number of comment lines", readonly=False, optional=True, no_value=0) - MetadataElement(name="data_lines", default=0, desc="Number of data lines", readonly=True, visible=False, optional=True, no_value=0) + MetadataElement( + name="comment_lines", default=0, desc="Number of comment lines", readonly=False, optional=True, no_value=0 + ) + MetadataElement( + name="data_lines", + default=0, + desc="Number of data lines", + readonly=True, + visible=False, + optional=True, + no_value=0, + ) MetadataElement(name="columns", default=0, desc="Number of columns", readonly=True, visible=False, no_value=0) - MetadataElement(name="column_types", default=[], desc="Column types", param=metadata.ColumnTypesParameter, readonly=True, visible=False, no_value=[]) - MetadataElement(name="column_names", default=[], desc="Column names", readonly=True, visible=False, optional=True, no_value=[]) - MetadataElement(name="delimiter", default='\t', desc="Data delimiter", readonly=True, visible=False, optional=True, no_value=[]) + MetadataElement( + name="column_types", + default=[], + desc="Column types", + param=metadata.ColumnTypesParameter, + readonly=True, + visible=False, + no_value=[], + ) + MetadataElement( + name="column_names", default=[], desc="Column names", readonly=True, visible=False, optional=True, no_value=[] + ) + MetadataElement( + name="delimiter", default="\t", desc="Data delimiter", readonly=True, visible=False, optional=True, no_value=[] + ) @abc.abstractmethod def set_meta(self, dataset, **kwd): @@ -65,10 +92,12 @@ class TabularData(data.Text): def displayable(self, dataset): try: - return dataset.has_data() \ - and dataset.state == dataset.states.OK \ - and dataset.metadata.columns > 0 \ + return ( + dataset.has_data() + and dataset.state == dataset.states.OK + and dataset.metadata.columns > 0 and dataset.metadata.data_lines != 0 + ) except Exception: return False @@ -76,16 +105,19 @@ class TabularData(data.Text): with compression_utils.get_fileobj(dataset.file_name) as f: f.seek(offset) ck_data = f.read(ck_size or trans.app.config.display_chunk_size) - if ck_data and ck_data[-1] != '\n': + if ck_data and ck_data[-1] != "\n": cursor = f.read(1) - while cursor and cursor != '\n': + while cursor and cursor != "\n": ck_data += cursor cursor = f.read(1) last_read = f.tell() - return dumps({'ck_data': util.unicodify(ck_data), - 'offset': last_read, - 'data_line_offset': self.data_line_offset, - }) + return dumps( + { + "ck_data": util.unicodify(ck_data), + "offset": last_read, + "data_line_offset": self.data_line_offset, + } + ) def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, offset=None, ck_size=None, **kwd): headers = kwd.get("headers", {}) @@ -102,30 +134,40 @@ class TabularData(data.Text): max_peek_size = 1000000 # 1 MB if os.stat(dataset.file_name).st_size < max_peek_size: self._clean_and_set_mime_type(trans, dataset.get_mime(), headers) - return open(dataset.file_name, mode='rb'), headers + return open(dataset.file_name, mode="rb"), headers else: headers["content-type"] = "text/html" - return trans.fill_template_mako("/dataset/large_file.mako", - truncated_data=open(dataset.file_name).read(max_peek_size), - data=dataset), headers + return ( + trans.fill_template_mako( + "/dataset/large_file.mako", + truncated_data=open(dataset.file_name).read(max_peek_size), + data=dataset, + ), + headers, + ) else: - column_names = 'null' + column_names = "null" if dataset.metadata.column_names: column_names = dataset.metadata.column_names - elif hasattr(dataset.datatype, 'column_names'): + elif hasattr(dataset.datatype, "column_names"): column_names = dataset.datatype.column_names column_types = dataset.metadata.column_types if not column_types: column_types = [] column_number = dataset.metadata.columns if column_number is None: - column_number = 'null' - return trans.fill_template("/dataset/tabular_chunked.mako", - dataset=dataset, - chunk=self.get_chunk(trans, dataset, 0), - column_number=column_number, - column_names=column_names, - column_types=column_types), headers + column_number = "null" + return ( + trans.fill_template( + "/dataset/tabular_chunked.mako", + dataset=dataset, + chunk=self.get_chunk(trans, dataset, 0), + column_number=column_number, + column_names=column_names, + column_types=column_types, + ), + headers, + ) def display_as_markdown(self, dataset_instance, markdown_format_helpers): with open(dataset_instance.file_name) as f: @@ -141,13 +183,21 @@ class TabularData(data.Text): try: out.append(self.make_html_peek_header(dataset, **kwargs)) out.append(self.make_html_peek_rows(dataset, **kwargs)) - out.append('') + out.append("") out = "".join(out) except Exception as exc: out = f"Can't create peek: {util.unicodify(exc)}" return out - def make_html_peek_header(self, dataset, skipchars=None, column_names=None, column_number_format='%s', column_parameter_alias=None, **kwargs): + def make_html_peek_header( + self, + dataset, + skipchars=None, + column_names=None, + column_number_format="%s", + column_parameter_alias=None, + **kwargs, + ): if skipchars is None: skipchars = [] if column_names is None: @@ -180,17 +230,17 @@ class TabularData(data.Text): if 0 <= i < columns and column_headers[i] is None: column_headers[i] = column_parameter_alias.get(name, name) - out.append('') + out.append("") for i, header in enumerate(column_headers): - out.append('') + out.append("") if header is None: out.append(column_number_format % str(i + 1)) else: - out.append(f'{str(i + 1)}.{escape(header)}') - out.append('') - out.append('') + out.append(f"{str(i + 1)}.{escape(header)}") + out.append("") + out.append("") except Exception as exc: - log.exception('make_html_peek_header failed on HDA %s', dataset.id) + log.exception("make_html_peek_header failed on HDA %s", dataset.id) raise Exception(f"Can't create peek header: {util.unicodify(exc)}") return "".join(out) @@ -214,20 +264,20 @@ class TabularData(data.Text): out.append(f'{escape(line)}') elif line: elems = line.split(dataset.metadata.delimiter) - elems = elems[:min(len(elems), self.max_peek_columns)] + elems = elems[: min(len(elems), self.max_peek_columns)] # pad shortened elems, since lines could have been truncated by width if len(elems) < columns: - elems.extend([''] * (columns - len(elems))) + elems.extend([""] * (columns - len(elems))) # we may have an invalid comment line or invalid data if len(elems) != columns: out.append(f'{escape(line)}') else: - out.append('') + out.append("") for elem in elems: - out.append(f'{escape(elem)}') - out.append('') + out.append(f"{escape(elem)}") + out.append("") except Exception as exc: - log.exception('make_html_peek_rows failed on HDA %s', dataset.id) + log.exception("make_html_peek_rows failed on HDA %s", dataset.id) raise Exception(f"Can't create peek rows: {util.unicodify(exc)}") return "".join(out) @@ -236,28 +286,27 @@ class TabularData(data.Text): return self.make_html_table(dataset) # ------------- Dataproviders - @dataproviders.decorators.dataprovider_factory('column', dataproviders.column.ColumnarDataProvider.settings) + @dataproviders.decorators.dataprovider_factory("column", dataproviders.column.ColumnarDataProvider.settings) def column_dataprovider(self, dataset, **settings): """Uses column settings that are passed in""" dataset_source = dataproviders.dataset.DatasetDataProvider(dataset) delimiter = dataset.metadata.delimiter return dataproviders.column.ColumnarDataProvider(dataset_source, deliminator=delimiter, **settings) - @dataproviders.decorators.dataprovider_factory('dataset-column', - dataproviders.column.ColumnarDataProvider.settings) + @dataproviders.decorators.dataprovider_factory("dataset-column", dataproviders.column.ColumnarDataProvider.settings) def dataset_column_dataprovider(self, dataset, **settings): """Attempts to get column settings from dataset.metadata""" delimiter = dataset.metadata.delimiter return dataproviders.dataset.DatasetColumnarDataProvider(dataset, deliminator=delimiter, **settings) - @dataproviders.decorators.dataprovider_factory('dict', dataproviders.column.DictDataProvider.settings) + @dataproviders.decorators.dataprovider_factory("dict", dataproviders.column.DictDataProvider.settings) def dict_dataprovider(self, dataset, **settings): """Uses column settings that are passed in""" dataset_source = dataproviders.dataset.DatasetDataProvider(dataset) delimiter = dataset.metadata.delimiter return dataproviders.column.DictDataProvider(dataset_source, deliminator=delimiter, **settings) - @dataproviders.decorators.dataprovider_factory('dataset-dict', dataproviders.column.DictDataProvider.settings) + @dataproviders.decorators.dataprovider_factory("dataset-dict", dataproviders.column.DictDataProvider.settings) def dataset_dict_dataprovider(self, dataset, **settings): """Attempts to get column settings from dataset.metadata""" delimiter = dataset.metadata.delimiter @@ -267,12 +316,15 @@ class TabularData(data.Text): @dataproviders.decorators.has_dataproviders class Tabular(TabularData): """Tab delimited data""" + file_ext = "tabular" def get_column_names(self, first_line=None): return None - def set_meta(self, dataset, overwrite=True, skip=None, max_data_lines=MAX_DATA_LINES, max_guess_type_data_lines=None, **kwd): + def set_meta( + self, dataset, overwrite=True, skip=None, max_data_lines=MAX_DATA_LINES, max_guess_type_data_lines=None, **kwd + ): """ Tries to determine the number of columns as well as those columns that contain numerical values in the dataset. A skip parameter is used @@ -301,7 +353,7 @@ class Tabular(TabularData): requested_skip = skip if skip is None: skip = 0 - column_type_set_order = ['int', 'float', 'list', 'str'] # Order to set column types in + column_type_set_order = ["int", "float", "list", "str"] # Order to set column types in default_column_type = column_type_set_order[-1] # Default column type is lowest in list column_type_compare_order = list(column_type_set_order) # Order to compare column types column_type_compare_order.reverse() @@ -321,7 +373,7 @@ class Tabular(TabularData): def is_int(column_text): # Don't allow underscores in numeric literals (PEP 515) - if '_' in column_text: + if "_" in column_text: return False try: int(column_text) @@ -331,13 +383,13 @@ class Tabular(TabularData): def is_float(column_text): # Don't allow underscores in numeric literals (PEP 515) - if '_' in column_text: + if "_" in column_text: return False try: float(column_text) return True except ValueError: - if column_text.strip().lower() == 'na': + if column_text.strip().lower() == "na": return True # na is special cased to be a float return False @@ -369,19 +421,21 @@ class Tabular(TabularData): # NOTE: if skip > num_check_lines, we won't detect any metadata, and will use default with compression_utils.get_fileobj(dataset.file_name) as dataset_fh: i = 0 - for line in iter(dataset_fh.readline, ''): - line = line.rstrip('\r\n') + for line in iter(dataset_fh.readline, ""): + line = line.rstrip("\r\n") if i == 0: column_names = self.get_column_names(first_line=line) - if i < skip or not line or line.startswith('#'): + if i < skip or not line or line.startswith("#"): # We'll call blank lines comments comment_lines += 1 else: data_lines += 1 if max_guess_type_data_lines is None or data_lines <= max_guess_type_data_lines: - fields = line.split('\t') + fields = line.split("\t") for field_count, field in enumerate(fields): - if field_count >= len(column_types): # found a previously unknown column, we append None + if field_count >= len( + column_types + ): # found a previously unknown column, we append None column_types.append(None) column_type = guess_column_type(field) if type_overrules_type(column_type, column_types[field_count]): @@ -412,7 +466,7 @@ class Tabular(TabularData): # we error on the larger number of columns # first we pad our column_types by using data from first line if len(first_line_column_types) > len(column_types): - for column_type in first_line_column_types[len(column_types):]: + for column_type in first_line_column_types[len(column_types) :]: column_types.append(column_type) # Now we fill any unknown (None) column_types with data from first line for i in range(len(column_types)): @@ -426,20 +480,21 @@ class Tabular(TabularData): dataset.metadata.comment_lines = comment_lines dataset.metadata.column_types = column_types dataset.metadata.columns = len(column_types) - dataset.metadata.delimiter = '\t' + dataset.metadata.delimiter = "\t" if column_names is not None: dataset.metadata.column_names = column_names def as_gbrowse_display_file(self, dataset, **kwd): - return open(dataset.file_name, 'rb') + return open(dataset.file_name, "rb") def as_ucsc_display_file(self, dataset, **kwd): - return open(dataset.file_name, 'rb') + return open(dataset.file_name, "rb") class SraManifest(Tabular): """A manifest received from the sra_source tool.""" - file_ext = 'sra_manifest.tabular' + + file_ext = "sra_manifest.tabular" data_line_offset = 1 def set_meta(self, dataset, **kwds): @@ -447,7 +502,7 @@ class SraManifest(Tabular): dataset.metadata.comment_lines = 1 def get_column_names(self, first_line): - return first_line.strip().split('\t') + return first_line.strip().split("\t") class Taxonomy(Tabular): @@ -456,11 +511,32 @@ class Taxonomy(Tabular): def __init__(self, **kwd): """Initialize taxonomy datatype""" super().__init__(**kwd) - self.column_names = ['Name', 'TaxId', 'Root', 'Superkingdom', 'Kingdom', 'Subkingdom', - 'Superphylum', 'Phylum', 'Subphylum', 'Superclass', 'Class', 'Subclass', - 'Superorder', 'Order', 'Suborder', 'Superfamily', 'Family', 'Subfamily', - 'Tribe', 'Subtribe', 'Genus', 'Subgenus', 'Species', 'Subspecies' - ] + self.column_names = [ + "Name", + "TaxId", + "Root", + "Superkingdom", + "Kingdom", + "Subkingdom", + "Superphylum", + "Phylum", + "Subphylum", + "Superclass", + "Class", + "Subclass", + "Superorder", + "Order", + "Suborder", + "Superfamily", + "Family", + "Subfamily", + "Tribe", + "Subtribe", + "Genus", + "Subgenus", + "Species", + "Subspecies", + ] def display_peek(self, dataset): """Returns formated html of peek""" @@ -472,23 +548,86 @@ class Taxonomy(Tabular): class Sam(Tabular, _BamOrSam): edam_format = "format_2573" edam_data = "data_0863" - file_ext = 'sam' + file_ext = "sam" track_type = "ReadTrack" data_sources = {"data": "bam", "index": "bigwig"} - MetadataElement(name="bam_version", default=None, desc="BAM Version", param=MetadataParameter, readonly=True, visible=False, optional=True) - MetadataElement(name="sort_order", default=None, desc="Sort Order", param=MetadataParameter, readonly=True, visible=False, optional=True) - MetadataElement(name="read_groups", default=[], desc="Read Groups", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value=[]) - MetadataElement(name="reference_names", default=[], desc="Chromosome Names", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value=[]) - MetadataElement(name="reference_lengths", default=[], desc="Chromosome Lengths", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value=[]) - MetadataElement(name="bam_header", default={}, desc="Dictionary of BAM Headers", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value={}) + MetadataElement( + name="bam_version", + default=None, + desc="BAM Version", + param=MetadataParameter, + readonly=True, + visible=False, + optional=True, + ) + MetadataElement( + name="sort_order", + default=None, + desc="Sort Order", + param=MetadataParameter, + readonly=True, + visible=False, + optional=True, + ) + MetadataElement( + name="read_groups", + default=[], + desc="Read Groups", + param=MetadataParameter, + readonly=True, + visible=False, + optional=True, + no_value=[], + ) + MetadataElement( + name="reference_names", + default=[], + desc="Chromosome Names", + param=MetadataParameter, + readonly=True, + visible=False, + optional=True, + no_value=[], + ) + MetadataElement( + name="reference_lengths", + default=[], + desc="Chromosome Lengths", + param=MetadataParameter, + readonly=True, + visible=False, + optional=True, + no_value=[], + ) + MetadataElement( + name="bam_header", + default={}, + desc="Dictionary of BAM Headers", + param=MetadataParameter, + readonly=True, + visible=False, + optional=True, + no_value={}, + ) def __init__(self, **kwd): """Initialize sam datatype""" super().__init__(**kwd) - self.column_names = ['QNAME', 'FLAG', 'RNAME', 'POS', 'MAPQ', 'CIGAR', - 'MRNM', 'MPOS', 'ISIZE', 'SEQ', 'QUAL', 'OPT' - ] + self.column_names = [ + "QNAME", + "FLAG", + "RNAME", + "POS", + "MAPQ", + "CIGAR", + "MRNM", + "MPOS", + "ISIZE", + "SEQ", + "QUAL", + "OPT", + ] def display_peek(self, dataset): """Returns formated html of peek""" @@ -527,8 +666,8 @@ class Sam(Tabular, _BamOrSam): for line in file_prefix.line_iterator(): line = line.strip() if line: - if line[0] != '@': - line_pieces = line.split('\t') + if line[0] != "@": + line_pieces = line.split("\t") if len(line_pieces) < 11: return False try: @@ -570,10 +709,13 @@ class Sam(Tabular, _BamOrSam): if dataset.has_data(): with open(dataset.file_name) as dataset_fh: comment_lines = 0 - if self.max_optional_metadata_filesize >= 0 and dataset.get_size() > self.max_optional_metadata_filesize: + if ( + self.max_optional_metadata_filesize >= 0 + and dataset.get_size() > self.max_optional_metadata_filesize + ): # If the dataset is larger than optional_metadata, just count comment lines. for line in dataset_fh: - if line.startswith('@'): + if line.startswith("@"): comment_lines += 1 else: # No more comments, and the file is too big to look at the whole thing. Give up. @@ -582,12 +724,25 @@ class Sam(Tabular, _BamOrSam): else: # Otherwise, read the whole thing and set num data lines. for i, l in enumerate(dataset_fh): # noqa: B007 - if l.startswith('@'): + if l.startswith("@"): comment_lines += 1 dataset.metadata.data_lines = i + 1 - comment_lines dataset.metadata.comment_lines = comment_lines dataset.metadata.columns = 12 - dataset.metadata.column_types = ['str', 'int', 'str', 'int', 'int', 'str', 'str', 'int', 'int', 'str', 'str', 'str'] + dataset.metadata.column_types = [ + "str", + "int", + "str", + "int", + "int", + "str", + "str", + "int", + "int", + "str", + "str", + "str", + ] _BamOrSam().set_meta(dataset) @@ -600,66 +755,67 @@ class Sam(Tabular, _BamOrSam): shutil.move(split_files[0], output_file) if len(split_files) > 1: - cmd = ['egrep', '-v', '-h', '^@'] + split_files[1:] + ['>>', output_file] + cmd = ["egrep", "-v", "-h", "^@"] + split_files[1:] + [">>", output_file] subprocess.check_call(cmd, shell=True) # Dataproviders # sam does not use '#' to indicate comments/headers - we need to strip out those headers from the std. providers # TODO:?? seems like there should be an easier way to do this - metadata.comment_char? - @dataproviders.decorators.dataprovider_factory('line', dataproviders.line.FilteredLineDataProvider.settings) + @dataproviders.decorators.dataprovider_factory("line", dataproviders.line.FilteredLineDataProvider.settings) def line_dataprovider(self, dataset, **settings): - settings['comment_char'] = '@' + settings["comment_char"] = "@" return super().line_dataprovider(dataset, **settings) - @dataproviders.decorators.dataprovider_factory('regex-line', dataproviders.line.RegexLineDataProvider.settings) + @dataproviders.decorators.dataprovider_factory("regex-line", dataproviders.line.RegexLineDataProvider.settings) def regex_line_dataprovider(self, dataset, **settings): - settings['comment_char'] = '@' + settings["comment_char"] = "@" return super().regex_line_dataprovider(dataset, **settings) - @dataproviders.decorators.dataprovider_factory('column', dataproviders.column.ColumnarDataProvider.settings) + @dataproviders.decorators.dataprovider_factory("column", dataproviders.column.ColumnarDataProvider.settings) def column_dataprovider(self, dataset, **settings): - settings['comment_char'] = '@' + settings["comment_char"] = "@" return super().column_dataprovider(dataset, **settings) - @dataproviders.decorators.dataprovider_factory('dataset-column', - dataproviders.column.ColumnarDataProvider.settings) + @dataproviders.decorators.dataprovider_factory("dataset-column", dataproviders.column.ColumnarDataProvider.settings) def dataset_column_dataprovider(self, dataset, **settings): - settings['comment_char'] = '@' + settings["comment_char"] = "@" return super().dataset_column_dataprovider(dataset, **settings) - @dataproviders.decorators.dataprovider_factory('dict', dataproviders.column.DictDataProvider.settings) + @dataproviders.decorators.dataprovider_factory("dict", dataproviders.column.DictDataProvider.settings) def dict_dataprovider(self, dataset, **settings): - settings['comment_char'] = '@' + settings["comment_char"] = "@" return super().dict_dataprovider(dataset, **settings) - @dataproviders.decorators.dataprovider_factory('dataset-dict', dataproviders.column.DictDataProvider.settings) + @dataproviders.decorators.dataprovider_factory("dataset-dict", dataproviders.column.DictDataProvider.settings) def dataset_dict_dataprovider(self, dataset, **settings): - settings['comment_char'] = '@' + settings["comment_char"] = "@" return super().dataset_dict_dataprovider(dataset, **settings) - @dataproviders.decorators.dataprovider_factory('header', dataproviders.line.RegexLineDataProvider.settings) + @dataproviders.decorators.dataprovider_factory("header", dataproviders.line.RegexLineDataProvider.settings) def header_dataprovider(self, dataset, **settings): dataset_source = dataproviders.dataset.DatasetDataProvider(dataset) - headers_source = dataproviders.line.RegexLineDataProvider(dataset_source, regex_list=['^@']) + headers_source = dataproviders.line.RegexLineDataProvider(dataset_source, regex_list=["^@"]) return dataproviders.line.RegexLineDataProvider(headers_source, **settings) - @dataproviders.decorators.dataprovider_factory('id-seq-qual', dict_dataprovider.settings) + @dataproviders.decorators.dataprovider_factory("id-seq-qual", dict_dataprovider.settings) def id_seq_qual_dataprovider(self, dataset, **settings): # provided as an example of a specified column dict (w/o metadata) - settings['indeces'] = [0, 9, 10] - settings['column_names'] = ['id', 'seq', 'qual'] + settings["indeces"] = [0, 9, 10] + settings["column_names"] = ["id", "seq", "qual"] return self.dict_dataprovider(dataset, **settings) - @dataproviders.decorators.dataprovider_factory('genomic-region', - dataproviders.dataset.GenomicRegionDataProvider.settings) + @dataproviders.decorators.dataprovider_factory( + "genomic-region", dataproviders.dataset.GenomicRegionDataProvider.settings + ) def genomic_region_dataprovider(self, dataset, **settings): - settings['comment_char'] = '@' + settings["comment_char"] = "@" return dataproviders.dataset.GenomicRegionDataProvider(dataset, 2, 3, 3, **settings) - @dataproviders.decorators.dataprovider_factory('genomic-region-dict', - dataproviders.dataset.GenomicRegionDataProvider.settings) + @dataproviders.decorators.dataprovider_factory( + "genomic-region-dict", dataproviders.dataset.GenomicRegionDataProvider.settings + ) def genomic_region_dict_dataprovider(self, dataset, **settings): - settings['comment_char'] = '@' + settings["comment_char"] = "@" return dataproviders.dataset.GenomicRegionDataProvider(dataset, 2, 3, 3, True, **settings) # @dataproviders.decorators.dataprovider_factory( 'samtools' ) @@ -672,6 +828,7 @@ class Sam(Tabular, _BamOrSam): @build_sniff_from_prefix class Pileup(Tabular): """Tab delimited data in pileup (6- or 10-column) format""" + edam_format = "format_3015" file_ext = "pileup" line_class = "genomic coordinate" @@ -687,7 +844,9 @@ class Pileup(Tabular): def display_peek(self, dataset): """Returns formated html of peek""" - return self.make_html_table(dataset, column_parameter_alias={'chromCol': 'Chrom', 'startCol': 'Start', 'baseCol': 'Base'}) + return self.make_html_table( + dataset, column_parameter_alias={"chromCol": "Chrom", "startCol": "Start", "baseCol": "Base"} + ) def repair_methods(self, dataset): """Return options for removing errors along with a description""" @@ -723,54 +882,74 @@ class Pileup(Tabular): """ found_non_comment_lines = False try: - headers = iter_headers(file_prefix, '\t') + headers = iter_headers(file_prefix, "\t") for hdr in headers: - if hdr and not hdr[0].startswith('#'): + if hdr and not hdr[0].startswith("#"): if len(hdr) < 5: return False # chrom start in column 1 (with 0-based columns) # and reference base is in column 2 chrom = int(hdr[1]) assert chrom >= 0 - assert hdr[2] in ['A', 'C', 'G', 'T', 'N', 'a', 'c', 'g', 't', 'n'] + assert hdr[2] in ["A", "C", "G", "T", "N", "a", "c", "g", "t", "n"] found_non_comment_lines = True except Exception: return False return found_non_comment_lines # Dataproviders - @dataproviders.decorators.dataprovider_factory('genomic-region', - dataproviders.dataset.GenomicRegionDataProvider.settings) + @dataproviders.decorators.dataprovider_factory( + "genomic-region", dataproviders.dataset.GenomicRegionDataProvider.settings + ) def genomic_region_dataprovider(self, dataset, **settings): return dataproviders.dataset.GenomicRegionDataProvider(dataset, **settings) - @dataproviders.decorators.dataprovider_factory('genomic-region-dict', - dataproviders.dataset.GenomicRegionDataProvider.settings) + @dataproviders.decorators.dataprovider_factory( + "genomic-region-dict", dataproviders.dataset.GenomicRegionDataProvider.settings + ) def genomic_region_dict_dataprovider(self, dataset, **settings): - settings['named_columns'] = True + settings["named_columns"] = True return self.genomic_region_dataprovider(dataset, **settings) @dataproviders.decorators.has_dataproviders @build_sniff_from_prefix class BaseVcf(Tabular): - """ Variant Call Format for describing SNPs and other simple genome variations. """ + """Variant Call Format for describing SNPs and other simple genome variations.""" + edam_format = "format_3016" track_type = "VariantTrack" data_sources = {"data": "tabix", "index": "bigwig"} - column_names = ['Chrom', 'Pos', 'ID', 'Ref', 'Alt', 'Qual', 'Filter', 'Info', 'Format', 'data'] + column_names = ["Chrom", "Pos", "ID", "Ref", "Alt", "Qual", "Filter", "Info", "Format", "data"] MetadataElement(name="columns", default=10, desc="Number of columns", readonly=True, visible=False) - MetadataElement(name="column_types", default=['str', 'int', 'str', 'str', 'str', 'int', 'str', 'list', 'str', 'str'], param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False) - MetadataElement(name="viz_filter_cols", desc="Score column for visualization", default=[5], param=metadata.ColumnParameter, optional=True, multiple=True, visible=False) - MetadataElement(name="sample_names", default=[], desc="Sample names", readonly=True, visible=False, optional=True, no_value=[]) + MetadataElement( + name="column_types", + default=["str", "int", "str", "str", "str", "int", "str", "list", "str", "str"], + param=metadata.ColumnTypesParameter, + desc="Column types", + readonly=True, + visible=False, + ) + MetadataElement( + name="viz_filter_cols", + desc="Score column for visualization", + default=[5], + param=metadata.ColumnParameter, + optional=True, + multiple=True, + visible=False, + ) + MetadataElement( + name="sample_names", default=[], desc="Sample names", readonly=True, visible=False, optional=True, no_value=[] + ) def _sniff(self, fname_or_file_prefix): # Because this sniffer is run on compressed files that might be BGZF (due to the VcfGz subclass), we should # handle unicode decode errors. This should ultimately be done in get_headers(), but guess_ext() currently # relies on get_headers() raising this exception. - headers = get_headers(fname_or_file_prefix, '\n', count=1) + headers = get_headers(fname_or_file_prefix, "\n", count=1) return headers[0][0].startswith("##fileformat=VCF") def display_peek(self, dataset): @@ -783,10 +962,10 @@ class BaseVcf(Tabular): with compression_utils.get_fileobj(dataset.file_name) as fh: # Skip comments. for line in fh: - if not line.startswith('##'): + if not line.startswith("##"): break - if line and line.startswith('#'): + if line and line.startswith("#"): # Found header line, get sample names. dataset.metadata.sample_names = line.split()[9:] @@ -796,7 +975,7 @@ class BaseVcf(Tabular): stderr_name = stderr_f.name command = ["bcftools", "concat"] + split_files + ["-o", output_file] log.info(f"Merging vcf files with command [{' '.join(command)}]") - exit_code = subprocess.call(args=command, stderr=open(stderr_name, 'wb')) + exit_code = subprocess.call(args=command, stderr=open(stderr_name, "wb")) with open(stderr_name, "rb") as f: stderr = f.read().strip() # Did merge succeed? @@ -807,24 +986,27 @@ class BaseVcf(Tabular): def validate_row(row): if len(row) < 8: raise Exception("Not enough columns in row %s" % row.join("\t")) - validate_tabular(dataset.file_name, sep='\t', validate_row=validate_row, comment_designator="#") + + validate_tabular(dataset.file_name, sep="\t", validate_row=validate_row, comment_designator="#") return data.DatatypeValidation.validated() # Dataproviders - @dataproviders.decorators.dataprovider_factory('genomic-region', - dataproviders.dataset.GenomicRegionDataProvider.settings) + @dataproviders.decorators.dataprovider_factory( + "genomic-region", dataproviders.dataset.GenomicRegionDataProvider.settings + ) def genomic_region_dataprovider(self, dataset, **settings): return dataproviders.dataset.GenomicRegionDataProvider(dataset, 0, 1, 1, **settings) - @dataproviders.decorators.dataprovider_factory('genomic-region-dict', - dataproviders.dataset.GenomicRegionDataProvider.settings) + @dataproviders.decorators.dataprovider_factory( + "genomic-region-dict", dataproviders.dataset.GenomicRegionDataProvider.settings + ) def genomic_region_dict_dataprovider(self, dataset, **settings): - settings['named_columns'] = True + settings["named_columns"] = True return self.genomic_region_dataprovider(dataset, **settings) class Vcf(BaseVcf): - file_ext = 'vcf' + file_ext = "vcf" def sniff_prefix(self, file_prefix: FilePrefix): return self._sniff(file_prefix) @@ -832,12 +1014,20 @@ class Vcf(BaseVcf): class VcfGz(BaseVcf, binary.Binary): # This class name is a misnomer, should be VcfBgzip - file_ext = 'vcf_bgzip' - file_ext_export_alias = 'vcf.gz' + file_ext = "vcf_bgzip" + file_ext_export_alias = "vcf.gz" compressed = True compressed_format = "gzip" - MetadataElement(name="tabix_index", desc="Vcf Index File", param=metadata.FileParameter, file_ext="tbi", readonly=True, visible=False, optional=True) + MetadataElement( + name="tabix_index", + desc="Vcf Index File", + param=metadata.FileParameter, + file_ext="tbi", + readonly=True, + visible=False, + optional=True, + ) def sniff(self, filename): if not self._sniff(filename): @@ -845,10 +1035,10 @@ class VcfGz(BaseVcf, binary.Binary): # Check that the file is compressed with bgzip (not gzip), i.e. the # compressed format is BGZF, as explained in # http://samtools.github.io/hts-specs/SAMv1.pdf - with open(filename, 'rb') as fh: + with open(filename, "rb") as fh: fh.seek(-28, 2) last28 = fh.read() - return binascii.hexlify(last28) == b'1f8b08040000000000ff0600424302001b0003000000000000000000' + return binascii.hexlify(last28) == b"1f8b08040000000000ff0600424302001b0003000000000000000000" def set_meta(self, dataset, **kwd): super().set_meta(dataset, **kwd) @@ -856,37 +1046,98 @@ class VcfGz(BaseVcf, binary.Binary): # These metadata values are not accessible by users, always overwrite index_file = dataset.metadata.tabix_index if not index_file: - index_file = dataset.metadata.spec['tabix_index'].param.new_file(dataset=dataset) + index_file = dataset.metadata.spec["tabix_index"].param.new_file(dataset=dataset) try: - pysam.tabix_index(dataset.file_name, index=index_file.file_name, preset='vcf', keep_original=True, force=True) + pysam.tabix_index( + dataset.file_name, index=index_file.file_name, preset="vcf", keep_original=True, force=True + ) except Exception as e: - raise Exception(f'Error setting VCF.gz metadata: {util.unicodify(e)}') + raise Exception(f"Error setting VCF.gz metadata: {util.unicodify(e)}") dataset.metadata.tabix_index = index_file @build_sniff_from_prefix class Eland(Tabular): """Support for the export.txt.gz file used by Illumina's ELANDv2e aligner""" + compressed = True compressed_format = "gzip" - file_ext = '_export.txt.gz' + file_ext = "_export.txt.gz" MetadataElement(name="columns", default=0, desc="Number of columns", readonly=True, visible=False) - MetadataElement(name="column_types", default=[], param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False, no_value=[]) + MetadataElement( + name="column_types", + default=[], + param=metadata.ColumnTypesParameter, + desc="Column types", + readonly=True, + visible=False, + no_value=[], + ) MetadataElement(name="comment_lines", default=0, desc="Number of comments", readonly=True, visible=False) - MetadataElement(name="tiles", default=[], param=metadata.ListParameter, desc="Set of tiles", readonly=True, visible=False, no_value=[]) - MetadataElement(name="reads", default=[], param=metadata.ListParameter, desc="Set of reads", readonly=True, visible=False, no_value=[]) - MetadataElement(name="lanes", default=[], param=metadata.ListParameter, desc="Set of lanes", readonly=True, visible=False, no_value=[]) - MetadataElement(name="barcodes", default=[], param=metadata.ListParameter, desc="Set of barcodes", readonly=True, visible=False, no_value=[]) + MetadataElement( + name="tiles", + default=[], + param=metadata.ListParameter, + desc="Set of tiles", + readonly=True, + visible=False, + no_value=[], + ) + MetadataElement( + name="reads", + default=[], + param=metadata.ListParameter, + desc="Set of reads", + readonly=True, + visible=False, + no_value=[], + ) + MetadataElement( + name="lanes", + default=[], + param=metadata.ListParameter, + desc="Set of lanes", + readonly=True, + visible=False, + no_value=[], + ) + MetadataElement( + name="barcodes", + default=[], + param=metadata.ListParameter, + desc="Set of barcodes", + readonly=True, + visible=False, + no_value=[], + ) def __init__(self, **kwd): """Initialize eland datatype""" super().__init__(**kwd) - self.column_names = ['MACHINE', 'RUN_NO', 'LANE', 'TILE', 'X', 'Y', - 'INDEX', 'READ_NO', 'SEQ', 'QUAL', 'CHROM', 'CONTIG', - 'POSITION', 'STRAND', 'DESC', 'SRAS', 'PRAS', 'PART_CHROM' - 'PART_CONTIG', 'PART_OFFSET', 'PART_STRAND', 'FILT' - ] + self.column_names = [ + "MACHINE", + "RUN_NO", + "LANE", + "TILE", + "X", + "Y", + "INDEX", + "READ_NO", + "SEQ", + "QUAL", + "CHROM", + "CONTIG", + "POSITION", + "STRAND", + "DESC", + "SRAS", + "PRAS", + "PART_CHROM" "PART_CONTIG", + "PART_OFFSET", + "PART_STRAND", + "FILT", + ] def make_html_table(self, dataset, skipchars=None, peek=None): """Create HTML table, used for displaying peek""" @@ -895,16 +1146,16 @@ class Eland(Tabular): out = [''] try: # Generate column header - out.append('') + out.append("") for i, name in enumerate(self.column_names): - out.append(f'') + out.append(f"") # This data type requires at least 11 columns in the data if dataset.metadata.columns - len(self.column_names) > 0: for i in range(len(self.column_names), max(dataset.metadata.columns, self.max_peek_columns)): - out.append(f'') - out.append('') + out.append(f"") + out.append("") out.append(self.make_html_peek_rows(dataset, skipchars=skipchars, peek=peek)) - out.append('
                {str(i + 1)}.{name}{str(i + 1)}.{name}{str(i + 1)}
                {str(i + 1)}
                ') + out.append("") out = "".join(out) except Exception as exc: out = f"Can't create peek {exc}" @@ -929,15 +1180,15 @@ class Eland(Tabular): if not line: break # Had a EOF comment previously, but this does not indicate EOF. I assume empty lines are not valid and this was intentional. if line: - line_pieces = line.split('\t') + line_pieces = line.split("\t") if len(line_pieces) != 22: return False if int(line_pieces[1]) < 0: - raise Exception('Out of range') + raise Exception("Out of range") if int(line_pieces[2]) < 0: - raise Exception('Out of range') + raise Exception("Out of range") if int(line_pieces[3]) < 0: - raise Exception('Out of range') + raise Exception("Out of range") int(line_pieces[4]) int(line_pieces[5]) # can get a lot more specific @@ -949,7 +1200,7 @@ class Eland(Tabular): def set_meta(self, dataset, overwrite=True, skip=None, max_data_lines=5, **kwd): if dataset.has_data(): - with compression_utils.get_fileobj(dataset.file_name, compressed_formats=['gzip']) as dataset_fh: + with compression_utils.get_fileobj(dataset.file_name, compressed_formats=["gzip"]) as dataset_fh: lanes = {} tiles = {} barcodes = {} @@ -962,9 +1213,9 @@ class Eland(Tabular): # Otherwise, read the whole thing and set num data lines. for i, line in enumerate(dataset_fh): if line: - line_pieces = line.split('\t') + line_pieces = line.split("\t") if len(line_pieces) != 22: - raise Exception('%s:%d:Corrupt line!' % (dataset.file_name, i)) + raise Exception("%s:%d:Corrupt line!" % (dataset.file_name, i)) lanes[line_pieces[2]] = 1 tiles[line_pieces[3]] = 1 barcodes[line_pieces[6]] = 1 @@ -972,16 +1223,40 @@ class Eland(Tabular): dataset.metadata.data_lines = i + 1 dataset.metadata.comment_lines = 0 dataset.metadata.columns = 21 - dataset.metadata.column_types = ['str', 'int', 'int', 'int', 'int', 'int', 'str', 'int', 'str', 'str', 'str', 'str', 'str', 'str', 'str', 'str', 'str', 'str', 'str', 'str', 'str'] + dataset.metadata.column_types = [ + "str", + "int", + "int", + "int", + "int", + "int", + "str", + "int", + "str", + "str", + "str", + "str", + "str", + "str", + "str", + "str", + "str", + "str", + "str", + "str", + "str", + ] dataset.metadata.lanes = list(lanes.keys()) dataset.metadata.tiles = ["%04d" % int(t) for t in tiles.keys()] - dataset.metadata.barcodes = [_ for _ in barcodes.keys() if _ != '0'] + ['NoIndex' for _ in barcodes.keys() if _ == '0'] + dataset.metadata.barcodes = [_ for _ in barcodes.keys() if _ != "0"] + [ + "NoIndex" for _ in barcodes.keys() if _ == "0" + ] dataset.metadata.reads = list(reads.keys()) @build_sniff_from_prefix class ElandMulti(Tabular): - file_ext = 'elandmulti' + file_ext = "elandmulti" def sniff_prefix(self, file_prefix: FilePrefix): return False @@ -991,9 +1266,18 @@ class FeatureLocationIndex(Tabular): """ An index that stores feature locations in tabular format. """ - file_ext = 'fli' + + file_ext = "fli" MetadataElement(name="columns", default=2, desc="Number of columns", readonly=True, visible=False) - MetadataElement(name="column_types", default=['str', 'str'], param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False, no_value=[]) + MetadataElement( + name="column_types", + default=["str", "str"], + param=metadata.ColumnTypesParameter, + desc="Column types", + readonly=True, + visible=False, + no_value=[], + ) @dataproviders.decorators.has_dataproviders @@ -1005,13 +1289,14 @@ class BaseCSV(TabularData): Must be extended to define the dialect to use, strict_width and file_ext. See the Python module csv for documentation of dialect settings """ - delimiter = ',' + + delimiter = "," peek_size = 1024 # File chunk used for sniffing CSV dialect big_peek_size = 10240 # Large File chunk used for sniffing CSV dialect def is_int(self, column_text): # Don't allow underscores in numeric literals (PEP 515) - if '_' in column_text: + if "_" in column_text: return False try: int(column_text) @@ -1021,28 +1306,28 @@ class BaseCSV(TabularData): def is_float(self, column_text): # Don't allow underscores in numeric literals (PEP 515) - if '_' in column_text: + if "_" in column_text: return False try: float(column_text) return True except ValueError: - if column_text.strip().lower() == 'na': + if column_text.strip().lower() == "na": return True # na is special cased to be a float return False def guess_type(self, text): if self.is_int(text): - return 'int' + return "int" if self.is_float(text): - return 'float' + return "float" else: - return 'str' + return "str" def sniff(self, filename): - """ Return True if if recognizes dialect and header. """ + """Return True if if recognizes dialect and header.""" # check the dialect works - with open(filename, newline='') as f: + with open(filename, newline="") as f: reader = csv.reader(f, self.dialect) # Check we can read header and get columns header_row = next(reader) @@ -1075,9 +1360,9 @@ class BaseCSV(TabularData): with open(filename) as f: big_peek = f.read(self.big_peek_size) auto_dialect = csv.Sniffer().sniff(big_peek) - if (auto_dialect.delimiter != self.dialect.delimiter): + if auto_dialect.delimiter != self.dialect.delimiter: return False - if (auto_dialect.quotechar != self.dialect.quotechar): + if auto_dialect.quotechar != self.dialect.quotechar: return False # Not checking for other dialect options # They may be mis detected from just the sample. @@ -1097,7 +1382,7 @@ class BaseCSV(TabularData): data_row = [] data_lines = 0 if dataset.has_data(): - with open(dataset.file_name, newline='') as csvfile: + with open(dataset.file_name, newline="") as csvfile: # Parse file with the correct dialect reader = csv.reader(csvfile, self.dialect) try: @@ -1108,7 +1393,7 @@ class BaseCSV(TabularData): except StopIteration: pass except csv.Error as e: - raise Exception('CSV reader error - line %d: %s' % (reader.line_num, e)) + raise Exception("CSV reader error - line %d: %s" % (reader.line_num, e)) else: data_lines = reader.line_num - 1 @@ -1131,7 +1416,8 @@ class CSV(BaseCSV): Comma-separated table data. Only sniffs comma-separated files with at least 2 rows and 2 columns. """ - file_ext = 'csv' + + file_ext = "csv" dialect = csv.excel # This is the default strict_width = False # Previous csv type did not check column width @@ -1150,7 +1436,8 @@ class TSV(BaseCSV): column less to indicate first column is row names. This kind of file is handled fine by the tabular datatype. """ - file_ext = 'tsv' + + file_ext = "tsv" dialect = csv.excel_tab strict_width = True # Leave files with different width to tabular @@ -1161,13 +1448,15 @@ class ConnectivityTable(Tabular): file_ext = "ct" header_regexp = re.compile("^[0-9]+(?: |[ ]+).*?(?:ENERGY|energy|dG)[ ].*?=") - structure_regexp = re.compile("^[0-9]+(?: |[ ]+)[ACGTURYKMSWBDHVN]+(?: |[ ]+)[^ ]+(?: |[ ]+)[^ ]+(?: |[ ]+)[^ ]+(?: |[ ]+)[^ ]+") + structure_regexp = re.compile( + "^[0-9]+(?: |[ ]+)[ACGTURYKMSWBDHVN]+(?: |[ ]+)[^ ]+(?: |[ ]+)[^ ]+(?: |[ ]+)[^ ]+(?: |[ ]+)[^ ]+" + ) def __init__(self, **kwd): super().__init__(**kwd) self.columns = 6 - self.column_names = ['base_index', 'base', 'neighbor_left', 'neighbor_right', 'partner', 'natural_numbering'] - self.column_types = ['int', 'str', 'int', 'int', 'int', 'int'] + self.column_names = ["base_index", "base", "neighbor_left", "neighbor_right", "partner", "natural_numbering"] + self.column_types = ["int", "str", "int", "int", "int", "int"] def set_meta(self, dataset, **kwd): data_lines = 0 @@ -1222,12 +1511,12 @@ class ConnectivityTable(Tabular): if not self.header_regexp.match(line): return False else: - length = int(re.split(r'\W+', line, 1)[0]) + length = int(re.split(r"\W+", line, 1)[0]) else: if not self.structure_regexp.match(line.upper()): return False else: - if j != int(re.split(r'\W+', line, 1)[0]): + if j != int(re.split(r"\W+", line, 1)[0]): return False elif j == length: # Last line of first sequence has been reached return True @@ -1243,22 +1532,22 @@ class ConnectivityTable(Tabular): # If we aren't at the start of the file, seek to next newline. Do this better eventually. if f.tell() != 0: cursor = f.read(1) - while cursor and cursor != '\n': + while cursor and cursor != "\n": cursor = f.read(1) ck_data = f.read(trans.app.config.display_chunk_size) cursor = f.read(1) - while cursor and ck_data[-1] != '\n': + while cursor and ck_data[-1] != "\n": ck_data += cursor cursor = f.read(1) # The ConnectivityTable format has several derivatives of which one is delimited by (multiple) spaces. # By converting these spaces back to tabs, chucks can still be interpreted by tab delimited file parsers - ck_data_header, ck_data_body = ck_data.split('\n', 1) - ck_data_header = re.sub('^([0-9]+)[ ]+', r'\1\t', ck_data_header) - ck_data_body = re.sub('\n[ \t]+', '\n', ck_data_body) - ck_data_body = re.sub('[ ]+', '\t', ck_data_body) + ck_data_header, ck_data_body = ck_data.split("\n", 1) + ck_data_header = re.sub("^([0-9]+)[ ]+", r"\1\t", ck_data_header) + ck_data_body = re.sub("\n[ \t]+", "\n", ck_data_body) + ck_data_body = re.sub("[ ]+", "\t", ck_data_body) - return dumps({'ck_data': util.unicodify(f"{ck_data_header}\n{ck_data_body}"), 'ck_index': ck_index + 1}) + return dumps({"ck_data": util.unicodify(f"{ck_data_header}\n{ck_data_body}"), "ck_index": ck_index + 1}) @build_sniff_from_prefix @@ -1296,39 +1585,43 @@ class MatrixMarket(TabularData): >>> MatrixMarket().sniff( get_test_fname( '3.mtx' ) ) True """ + file_ext = "mtx" def __init__(self, **kwd): super().__init__(**kwd) def sniff_prefix(self, file_prefix: FilePrefix): - return file_prefix.startswith('%%MatrixMarket matrix coordinate') + return file_prefix.startswith("%%MatrixMarket matrix coordinate") def set_meta(self, dataset, overwrite=True, skip=None, max_data_lines=5, **kwd): if dataset.has_data(): # If the dataset is larger than optional_metadata, just count comment lines. with open(dataset.file_name) as dataset_fh: - line = '' + line = "" data_lines = 0 comment_lines = 0 # If the dataset is larger than optional_metadata, just count comment lines. - count_comments_only = self.max_optional_metadata_filesize >= 0 and dataset.get_size() > self.max_optional_metadata_filesize + count_comments_only = ( + self.max_optional_metadata_filesize >= 0 + and dataset.get_size() > self.max_optional_metadata_filesize + ) for line in dataset_fh: - if line.startswith('%'): + if line.startswith("%"): comment_lines += 1 elif count_comments_only: data_lines = None break else: data_lines += 1 - if ' ' in line: - dataset.metadata.delimiter = ' ' + if " " in line: + dataset.metadata.delimiter = " " else: - dataset.metadata.delimiter = '\t' + dataset.metadata.delimiter = "\t" dataset.metadata.comment_lines = comment_lines dataset.metadata.data_lines = data_lines dataset.metadata.columns = 3 - dataset.metadata.column_types = ['int', 'int', 'float'] + dataset.metadata.column_types = ["int", "int", "float"] @build_sniff_from_prefix @@ -1346,22 +1639,79 @@ class CMAP(TabularData): 182 58474736.7 10235 1 1 58820.9 35.4 13.5 13.5 -1.00 -1.00 -1.00 3.63 0.00 0.00 -1.00 0 182 58474736.7 10235 1 1 58820.9 35.4 13.5 13.5 -1.00 -1.00 -1.00 3.63 0.00 0.00 -1.00 0 """ + file_ext = "cmap" - MetadataElement(name="cmap_version", default='0.2', desc="version of cmap", readonly=True, visible=True, optional=False, no_value='0.2') - MetadataElement(name="label_channels", default=1, desc="the number of label channels", readonly=True, visible=True, optional=False, no_value=1) - MetadataElement(name="nickase_recognition_site_1", default=[], desc="comma separated list of label motif recognition sequences for channel 1", readonly=True, visible=True, optional=False, no_value=[]) - MetadataElement(name="number_of_consensus_nanomaps", default=0, desc="the total number of consensus genome maps in the CMAP file", readonly=True, visible=True, optional=False, no_value=0) - MetadataElement(name="nickase_recognition_site_2", default=[], desc="comma separated list of label motif recognition sequences for channel 2", readonly=True, visible=True, optional=True, no_value=[]) - MetadataElement(name="channel_1_color", default=[], desc="channel 1 color", readonly=True, visible=True, optional=True, no_value=[]) - MetadataElement(name="channel_2_color", default=[], desc="channel 2 color", readonly=True, visible=True, optional=True, no_value=[]) + MetadataElement( + name="cmap_version", + default="0.2", + desc="version of cmap", + readonly=True, + visible=True, + optional=False, + no_value="0.2", + ) + MetadataElement( + name="label_channels", + default=1, + desc="the number of label channels", + readonly=True, + visible=True, + optional=False, + no_value=1, + ) + MetadataElement( + name="nickase_recognition_site_1", + default=[], + desc="comma separated list of label motif recognition sequences for channel 1", + readonly=True, + visible=True, + optional=False, + no_value=[], + ) + MetadataElement( + name="number_of_consensus_nanomaps", + default=0, + desc="the total number of consensus genome maps in the CMAP file", + readonly=True, + visible=True, + optional=False, + no_value=0, + ) + MetadataElement( + name="nickase_recognition_site_2", + default=[], + desc="comma separated list of label motif recognition sequences for channel 2", + readonly=True, + visible=True, + optional=True, + no_value=[], + ) + MetadataElement( + name="channel_1_color", + default=[], + desc="channel 1 color", + readonly=True, + visible=True, + optional=True, + no_value=[], + ) + MetadataElement( + name="channel_2_color", + default=[], + desc="channel 2 color", + readonly=True, + visible=True, + optional=True, + no_value=[], + ) def sniff_prefix(self, file_prefix: FilePrefix): handle = file_prefix.string_io() for line in handle: - if not line.startswith('#'): + if not line.startswith("#"): return False - if line.startswith('# CMAP File Version:'): + if line.startswith("# CMAP File Version:"): return True return False @@ -1373,49 +1723,55 @@ class CMAP(TabularData): cleaned_column_types = None number_of_columns = 0 for i, line in enumerate(dataset_fh): - line = line.strip('\n') - if line.startswith('#'): + line = line.strip("\n") + if line.startswith("#"): - if line.startswith('#h'): + if line.startswith("#h"): column_headers = line.split("\t")[1:] - elif line.startswith('#f'): + elif line.startswith("#f"): cleaned_column_types = [] - for column_type in line.split('\t')[1:]: - if column_type == 'Hex': - cleaned_column_types.append('str') + for column_type in line.split("\t")[1:]: + if column_type == "Hex": + cleaned_column_types.append("str") else: cleaned_column_types.append(column_type) comment_lines += 1 - fields = line.split('\t') + fields = line.split("\t") if len(fields) == 2: - if fields[0] == '# CMAP File Version:': + if fields[0] == "# CMAP File Version:": dataset.metadata.cmap_version = fields[1] - elif fields[0] == '# Label Channels:': + elif fields[0] == "# Label Channels:": dataset.metadata.label_channels = int(fields[1]) - elif fields[0] == '# Nickase Recognition Site 1:': - fields2 = fields[1].split(';') + elif fields[0] == "# Nickase Recognition Site 1:": + fields2 = fields[1].split(";") if len(fields2) == 2: dataset.metadata.channel_1_color = fields2[1] - dataset.metadata.nickase_recognition_site_1 = fields2[0].split(',') - elif fields[0] == '# Number of Consensus Maps:': + dataset.metadata.nickase_recognition_site_1 = fields2[0].split(",") + elif fields[0] == "# Number of Consensus Maps:": dataset.metadata.number_of_consensus_nanomaps = int(fields[1]) - elif fields[0] == '# Nickase Recognition Site 2:': - fields2 = fields[1].split(';') + elif fields[0] == "# Nickase Recognition Site 2:": + fields2 = fields[1].split(";") if len(fields2) == 2: dataset.metadata.channel_2_color = fields2[1] - dataset.metadata.nickase_recognition_site_2 = fields2[0].split(',') - elif self.max_optional_metadata_filesize >= 0 and dataset.get_size() > self.max_optional_metadata_filesize: + dataset.metadata.nickase_recognition_site_2 = fields2[0].split(",") + elif ( + self.max_optional_metadata_filesize >= 0 + and dataset.get_size() > self.max_optional_metadata_filesize + ): # If the dataset is larger than optional_metadata, just count comment lines. # No more comments, and the file is too big to look at the whole thing. Give up. dataset.metadata.data_lines = None break elif i == comment_lines + 1: - number_of_columns = len(line.split('\t')) - if not (self.max_optional_metadata_filesize >= 0 and dataset.get_size() > self.max_optional_metadata_filesize): + number_of_columns = len(line.split("\t")) + if not ( + self.max_optional_metadata_filesize >= 0 + and dataset.get_size() > self.max_optional_metadata_filesize + ): dataset.metadata.data_lines = i + 1 - comment_lines dataset.metadata.comment_lines = comment_lines dataset.metadata.column_names = column_headers dataset.metadata.column_types = cleaned_column_types dataset.metadata.columns = number_of_columns - dataset.metadata.delimiter = '\t' + dataset.metadata.delimiter = "\t" diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index a5bdb09dcb4..c4f84555202 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -12,8 +12,15 @@ import tempfile import yaml -from galaxy.datatypes.data import get_file_peek, Headers, Text -from galaxy.datatypes.metadata import MetadataElement, MetadataParameter +from galaxy.datatypes.data import ( + get_file_peek, + Headers, + Text, +) +from galaxy.datatypes.metadata import ( + MetadataElement, + MetadataParameter, +) from galaxy.datatypes.sniff import ( build_sniff_from_prefix, FilePrefix, @@ -31,6 +38,7 @@ log = logging.getLogger(__name__) @build_sniff_from_prefix class Html(Text): """Class describing an html file""" + edam_format = "format_2331" file_ext = "html" @@ -39,12 +47,12 @@ class Html(Text): dataset.peek = "HTML file" dataset.blurb = nice_size(dataset.get_size()) else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def get_mime(self): """Returns the mime type of the datatype""" - return 'text/html' + return "text/html" def sniff_prefix(self, file_prefix: FilePrefix): """ @@ -60,7 +68,7 @@ class Html(Text): """ headers = iter_headers(file_prefix, None) for hdr in headers: - if hdr and hdr[0].lower().find('') >= 0: + if hdr and hdr[0].lower().find("") >= 0: return True return False @@ -75,16 +83,16 @@ class Json(Text): dataset.peek = get_file_peek(dataset.file_name) dataset.blurb = "JavaScript Object Notation (JSON)" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disc' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disc" def get_mime(self): """Returns the mime type of the datatype""" - return 'application/json' + return "application/json" def sniff_prefix(self, file_prefix: FilePrefix): """ - Try to load the string with the json module. If successful it's a json file. + Try to load the string with the json module. If successful it's a json file. """ return self._looks_like_json(file_prefix) @@ -115,14 +123,15 @@ class Json(Text): class ExpressionJson(Json): - """ Represents the non-data input or output to a tool or workflow. - """ + """Represents the non-data input or output to a tool or workflow.""" + file_ext = "json" - MetadataElement(name="json_type", default=None, desc="JavaScript or JSON type of expression", readonly=True, visible=True) + MetadataElement( + name="json_type", default=None, desc="JavaScript or JSON type of expression", readonly=True, visible=True + ) def set_meta(self, dataset, **kwd): - """ - """ + """ """ if dataset.has_data(): json_type = "null" file_path = dataset.file_name @@ -153,18 +162,18 @@ class Ipynb(Json): dataset.peek = get_file_peek(dataset.file_name) dataset.blurb = "Jupyter Notebook" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disc' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disc" def sniff_prefix(self, file_prefix: FilePrefix): """ - Try to load the string with the json module. If successful it's a json file. + Try to load the string with the json module. If successful it's a json file. """ if self._looks_like_json(file_prefix): try: with open(file_prefix.filename) as f: ipynb = json.load(f) - if ipynb.get('nbformat', False) is not False and ipynb.get('metadata', False): + if ipynb.get("nbformat", False) is not False and ipynb.get("metadata", False): return True else: return False @@ -174,11 +183,15 @@ class Ipynb(Json): def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, **kwd): headers = kwd.get("headers", {}) config = trans.app.config - trust = getattr(config, 'trust_jupyter_notebook_conversion', False) + trust = getattr(config, "trust_jupyter_notebook_conversion", False) if trust: - return self._display_data_trusted(trans, dataset, preview=preview, filename=filename, to_ext=to_ext, headers=headers, **kwd) + return self._display_data_trusted( + trans, dataset, preview=preview, filename=filename, to_ext=to_ext, headers=headers, **kwd + ) else: - return super().display_data(trans, dataset, preview=preview, filename=filename, to_ext=to_ext, headers=headers, **kwd) + return super().display_data( + trans, dataset, preview=preview, filename=filename, to_ext=to_ext, headers=headers, **kwd + ) def _display_data_trusted(self, trans, dataset, preview=False, filename=None, to_ext=None, **kwd): headers = kwd.get("headers", {}) @@ -189,13 +202,26 @@ class Ipynb(Json): with tempfile.NamedTemporaryFile(delete=False) as ofile_handle: ofilename = ofile_handle.name try: - cmd = ['jupyter', 'nbconvert', '--to', 'html', '--template', 'full', dataset.file_name, '--output', ofilename] + cmd = [ + "jupyter", + "nbconvert", + "--to", + "html", + "--template", + "full", + dataset.file_name, + "--output", + ofilename, + ] subprocess.check_call(cmd) - ofilename = f'{ofilename}.html' + ofilename = f"{ofilename}.html" except subprocess.CalledProcessError: ofilename = dataset.file_name - log.exception('Command "%s" failed. Could not convert the Jupyter Notebook to HTML, defaulting to plain text.', ' '.join(map(shlex.quote, cmd))) - return open(ofilename, mode='rb'), headers + log.exception( + 'Command "%s" failed. Could not convert the Jupyter Notebook to HTML, defaulting to plain text.', + " ".join(map(shlex.quote, cmd)), + ) + return open(ofilename, mode="rb"), headers def set_meta(self, dataset, **kwd): """ @@ -206,24 +232,132 @@ class Ipynb(Json): @build_sniff_from_prefix class Biom1(Json): """ - BIOM version 1.0 file format description - http://biom-format.org/documentation/format_versions/biom-1.0.html + BIOM version 1.0 file format description + http://biom-format.org/documentation/format_versions/biom-1.0.html """ + file_ext = "biom1" edam_format = "format_3746" - MetadataElement(name="table_rows", default=[], desc="table_rows", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value=[]) - MetadataElement(name="table_matrix_element_type", default="", desc="table_matrix_element_type", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value="") - MetadataElement(name="table_format", default="", desc="table_format", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value="") - MetadataElement(name="table_generated_by", default="", desc="table_generated_by", param=MetadataParameter, readonly=True, visible=True, optional=True, no_value="") - MetadataElement(name="table_matrix_type", default="", desc="table_matrix_type", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value="") - MetadataElement(name="table_shape", default=[], desc="table_shape", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value=[]) - MetadataElement(name="table_format_url", default="", desc="table_format_url", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value="") - MetadataElement(name="table_date", default="", desc="table_date", param=MetadataParameter, readonly=True, visible=True, optional=True, no_value="") - MetadataElement(name="table_type", default="", desc="table_type", param=MetadataParameter, readonly=True, visible=True, optional=True, no_value="") - MetadataElement(name="table_id", default=None, desc="table_id", param=MetadataParameter, readonly=True, visible=True, optional=True) - MetadataElement(name="table_columns", default=[], desc="table_columns", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value=[]) - MetadataElement(name="table_column_metadata_headers", default=[], desc="table_column_metadata_headers", param=MetadataParameter, readonly=True, visible=True, optional=True, no_value=[]) + MetadataElement( + name="table_rows", + default=[], + desc="table_rows", + param=MetadataParameter, + readonly=True, + visible=False, + optional=True, + no_value=[], + ) + MetadataElement( + name="table_matrix_element_type", + default="", + desc="table_matrix_element_type", + param=MetadataParameter, + readonly=True, + visible=False, + optional=True, + no_value="", + ) + MetadataElement( + name="table_format", + default="", + desc="table_format", + param=MetadataParameter, + readonly=True, + visible=False, + optional=True, + no_value="", + ) + MetadataElement( + name="table_generated_by", + default="", + desc="table_generated_by", + param=MetadataParameter, + readonly=True, + visible=True, + optional=True, + no_value="", + ) + MetadataElement( + name="table_matrix_type", + default="", + desc="table_matrix_type", + param=MetadataParameter, + readonly=True, + visible=False, + optional=True, + no_value="", + ) + MetadataElement( + name="table_shape", + default=[], + desc="table_shape", + param=MetadataParameter, + readonly=True, + visible=False, + optional=True, + no_value=[], + ) + MetadataElement( + name="table_format_url", + default="", + desc="table_format_url", + param=MetadataParameter, + readonly=True, + visible=False, + optional=True, + no_value="", + ) + MetadataElement( + name="table_date", + default="", + desc="table_date", + param=MetadataParameter, + readonly=True, + visible=True, + optional=True, + no_value="", + ) + MetadataElement( + name="table_type", + default="", + desc="table_type", + param=MetadataParameter, + readonly=True, + visible=True, + optional=True, + no_value="", + ) + MetadataElement( + name="table_id", + default=None, + desc="table_id", + param=MetadataParameter, + readonly=True, + visible=True, + optional=True, + ) + MetadataElement( + name="table_columns", + default=[], + desc="table_columns", + param=MetadataParameter, + readonly=True, + visible=False, + optional=True, + no_value=[], + ) + MetadataElement( + name="table_column_metadata_headers", + default=[], + desc="table_column_metadata_headers", + param=MetadataParameter, + readonly=True, + visible=True, + optional=True, + no_value=[], + ) def set_peek(self, dataset): super().set_peek(dataset) @@ -248,11 +382,11 @@ class Biom1(Json): with open(file_prefix.filename) as fh: prev_str = "" segment_str = fh.read(segment_size) - if segment_str.strip().startswith('{'): + if segment_str.strip().startswith("{"): while segment_str: current_str = prev_str + segment_str if '"format"' in current_str: - current_str = re.sub(r'\s', '', current_str) + current_str = re.sub(r"\s", "", current_str) if '"format":"BiologicalObservationMatrix' in current_str: is_biom = True break @@ -264,7 +398,7 @@ class Biom1(Json): def set_meta(self, dataset, **kwd): """ - Store metadata information from the BIOM file. + Store metadata information from the BIOM file. """ if dataset.has_data(): with open(dataset.file_name) as fh: @@ -275,28 +409,30 @@ class Biom1(Json): def _transform_dict_list_ids(dict_list): if dict_list: - return [x.get('id', None) for x in dict_list] + return [x.get("id", None) for x in dict_list] return [] - b_transform = {'rows': _transform_dict_list_ids, 'columns': _transform_dict_list_ids} - for (m_name, b_name) in [('table_rows', 'rows'), - ('table_matrix_element_type', 'matrix_element_type'), - ('table_format', 'format'), - ('table_generated_by', 'generated_by'), - ('table_matrix_type', 'matrix_type'), - ('table_shape', 'shape'), - ('table_format_url', 'format_url'), - ('table_date', 'date'), - ('table_type', 'type'), - ('table_id', 'id'), - ('table_columns', 'columns')]: + b_transform = {"rows": _transform_dict_list_ids, "columns": _transform_dict_list_ids} + for (m_name, b_name) in [ + ("table_rows", "rows"), + ("table_matrix_element_type", "matrix_element_type"), + ("table_format", "format"), + ("table_generated_by", "generated_by"), + ("table_matrix_type", "matrix_type"), + ("table_shape", "shape"), + ("table_format_url", "format_url"), + ("table_date", "date"), + ("table_type", "type"), + ("table_id", "id"), + ("table_columns", "columns"), + ]: try: metadata_value = json_dict.get(b_name, None) if b_name == "columns" and metadata_value: keep_columns = set() for column in metadata_value: - if column['metadata'] is not None: - for k, v in column['metadata'].items(): + if column["metadata"] is not None: + for k, v in column["metadata"].items(): if v is not None: keep_columns.add(k) final_list = sorted(list(keep_columns)) @@ -317,6 +453,7 @@ class ImgtJson(Json): "IMGT®, the international ImMunoGeneTics information system® http://www.imgt.org (founder and director: Marie-Paule Lefranc, Montpellier, France)." """ + file_ext = "imgt.json" MetadataElement(name="taxon_names", default=[], desc="taxonID: names", readonly=True, visible=True, no_value=[]) @@ -353,7 +490,7 @@ class ImgtJson(Json): try: with open(file_prefix.filename) as fh: segment_str = fh.read(load_size) - if segment_str.strip().startswith('['): + if segment_str.strip().startswith("["): if '"taxonId"' in segment_str and '"anchorPoints"' in segment_str: is_imgt = True except Exception: @@ -362,7 +499,7 @@ class ImgtJson(Json): def set_meta(self, dataset, **kwd): """ - Store metadata information from the imgt file. + Store metadata information from the imgt file. """ if dataset.has_data(): with open(dataset.file_name) as fh: @@ -370,8 +507,8 @@ class ImgtJson(Json): json_dict = json.load(fh) tax_names = [] for entry in json_dict: - if 'taxonId' in entry: - names = "%d: %s" % (entry['taxonId'], ','.join(entry['speciesNames'])) + if "taxonId" in entry: + names = "%d: %s" % (entry["taxonId"], ",".join(entry["speciesNames"])) tax_names.append(names) dataset.metadata.taxon_names = tax_names except Exception: @@ -381,9 +518,10 @@ class ImgtJson(Json): @build_sniff_from_prefix class GeoJson(Json): """ - GeoJSON is a geospatial data interchange format based on JavaScript Object Notation (JSON). - https://tools.ietf.org/html/rfc7946 + GeoJSON is a geospatial data interchange format based on JavaScript Object Notation (JSON). + https://tools.ietf.org/html/rfc7946 """ + file_ext = "geojson" def set_peek(self, dataset): @@ -417,7 +555,18 @@ class GeoJson(Json): try: with open(file_prefix.filename) as fh: segment_str = fh.read(load_size) - if any(x in segment_str for x in ["Point", "MultiPoint", "LineString", "MultiLineString", "Polygon", "MultiPolygon", "GeometryCollection"]): + if any( + x in segment_str + for x in [ + "Point", + "MultiPoint", + "LineString", + "MultiLineString", + "Polygon", + "MultiPolygon", + "GeometryCollection", + ] + ): if all(x in segment_str for x in ["type", "geometry", "coordinates"]): return True except Exception: @@ -428,9 +577,10 @@ class GeoJson(Json): @build_sniff_from_prefix class Obo(Text): """ - OBO file format description - https://owlcollab.github.io/oboformat/doc/GO.format.obo-1_2.html + OBO file format description + https://owlcollab.github.io/oboformat/doc/GO.format.obo-1_2.html """ + edam_data = "data_0582" edam_format = "format_2549" file_ext = "obo" @@ -440,24 +590,24 @@ class Obo(Text): dataset.peek = get_file_peek(dataset.file_name) dataset.blurb = "Open Biomedical Ontology (OBO)" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disc' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disc" def sniff_prefix(self, file_prefix: FilePrefix): """ - Try to guess the Obo filetype. - It usually starts with a "format-version:" string and has several stanzas which starts with "id:". + Try to guess the Obo filetype. + It usually starts with a "format-version:" string and has several stanzas which starts with "id:". """ - stanza = re.compile(r'^\[.*\]$') + stanza = re.compile(r"^\[.*\]$") handle = file_prefix.string_io() first_line = handle.readline() - if not first_line.startswith('format-version:'): + if not first_line.startswith("format-version:"): return False for line in handle: if stanza.match(line.strip()): # a stanza needs to begin with an ID tag - if next(handle).startswith('id:'): + if next(handle).startswith("id:"): return True return False @@ -468,10 +618,13 @@ class Arff(Text): An ARFF (Attribute-Relation File Format) file is an ASCII text file that describes a list of instances sharing a set of attributes. http://weka.wikispaces.com/ARFF """ + edam_format = "format_3581" file_ext = "arff" - MetadataElement(name="comment_lines", default=0, desc="Number of comment lines", readonly=True, optional=True, no_value=0) + MetadataElement( + name="comment_lines", default=0, desc="Number of comment lines", readonly=True, optional=True, no_value=0 + ) MetadataElement(name="columns", default=0, desc="Number of columns", readonly=True, visible=True, no_value=0) def set_peek(self, dataset): @@ -480,13 +633,13 @@ class Arff(Text): dataset.blurb = "Attribute-Relation File Format (ARFF)" dataset.blurb += f", {dataset.metadata.comment_lines} comments, {dataset.metadata.columns} attributes" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disc' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disc" def sniff_prefix(self, file_prefix: FilePrefix): """ - Try to guess the Arff filetype. - It usually starts with a "format-version:" string and has several stanzas which starts with "id:". + Try to guess the Arff filetype. + It usually starts with a "format-version:" string and has several stanzas which starts with "id:". """ handle = file_prefix.string_io() relation_found = False @@ -512,11 +665,11 @@ class Arff(Text): def set_meta(self, dataset, **kwd): """ - Trying to count the comment lines and the number of columns included. - A typical ARFF data block looks like this: - @DATA - 5.1,3.5,1.4,0.2,Iris-setosa - 4.9,3.0,1.4,0.2,Iris-setosa + Trying to count the comment lines and the number of columns included. + A typical ARFF data block looks like this: + @DATA + 5.1,3.5,1.4,0.2,Iris-setosa + 4.9,3.0,1.4,0.2,Iris-setosa """ comment_lines = column_count = 0 if dataset.has_data(): @@ -527,12 +680,12 @@ class Arff(Text): line = line.strip() if not line: continue - if line.startswith('%') and not first_real_line: + if line.startswith("%") and not first_real_line: comment_lines += 1 else: first_real_line = True if data_block: - if line.startswith('{'): + if line.startswith("{"): # Sparse representation """ @data @@ -541,18 +694,18 @@ class Arff(Text): @data {1 X, 3 Y, 4 "class A"}, {5} """ - token = line.split('}', 1) + token = line.split("}", 1) first_part = token[0] - last_column = first_part.split(',')[-1].strip() + last_column = first_part.split(",")[-1].strip() numeric_value = last_column.split()[0] column_count = int(numeric_value) if len(token) > 1: # we have an additional weight column_count -= 1 else: - columns = line.strip().split(',') + columns = line.strip().split(",") column_count = len(columns) - if columns[-1].strip().startswith('{'): + if columns[-1].strip().startswith("{"): # we have an additional weight at the end column_count -= 1 @@ -566,12 +719,17 @@ class Arff(Text): class SnpEffDb(Text): """Class describing a SnpEff genome build""" + edam_format = "format_3624" file_ext = "snpeffdb" MetadataElement(name="genome_version", default=None, desc="Genome Version", readonly=True, visible=True) MetadataElement(name="snpeff_version", default="SnpEff4.0", desc="SnpEff Version", readonly=True, visible=True) - MetadataElement(name="regulation", default=[], desc="Regulation Names", readonly=True, visible=True, no_value=[], optional=True) - MetadataElement(name="annotation", default=[], desc="Annotation Names", readonly=True, visible=True, no_value=[], optional=True) + MetadataElement( + name="regulation", default=[], desc="Regulation Names", readonly=True, visible=True, no_value=[], optional=True + ) + MetadataElement( + name="annotation", default=[], desc="Annotation Names", readonly=True, visible=True, no_value=[], optional=True + ) def __init__(self, **kwd): super().__init__(**kwd) @@ -580,10 +738,10 @@ class SnpEffDb(Text): def getSnpeffVersionFromFile(self, path): snpeff_version = None try: - with gzip.open(path, 'rt') as fh: + with gzip.open(path, "rt") as fh: buf = fh.read(100) lines = buf.splitlines() - m = re.match(r'^(SnpEff)\s+(\d+\.\d+).*$', lines[0].strip()) + m = re.match(r"^(SnpEff)\s+(\d+\.\d+).*$", lines[0].strip()) if m: snpeff_version = m.groups()[0] + m.groups()[1] except Exception: @@ -594,9 +752,9 @@ class SnpEffDb(Text): super().set_meta(dataset, **kwd) data_dir = dataset.extra_files_path # search data_dir/genome_version for files - regulation_pattern = 'regulation_(.+).bin' + regulation_pattern = "regulation_(.+).bin" # annotation files that are included in snpEff by a flag - annotations_dict = {'nextProt.bin': '-nextprot', 'motif.bin': '-motif', 'interactions.bin': '-interaction'} + annotations_dict = {"nextProt.bin": "-nextprot", "motif.bin": "-motif", "interactions.bin": "-interaction"} regulations = [] annotations = [] genome_version = None @@ -604,7 +762,7 @@ class SnpEffDb(Text): if data_dir and os.path.isdir(data_dir): for root, _, files in os.walk(data_dir): for fname in files: - if fname.startswith('snpEffectPredictor'): + if fname.startswith("snpEffectPredictor"): # if snpEffectPredictor.bin download succeeded genome_version = os.path.basename(root) dataset.metadata.genome_version = genome_version @@ -619,14 +777,14 @@ class SnpEffDb(Text): regulations.append(name) elif fname in annotations_dict: value = annotations_dict[fname] - name = value.lstrip('-') + name = value.lstrip("-") annotations.append(name) dataset.metadata.regulation = regulations dataset.metadata.annotation = annotations try: - with open(dataset.file_name, 'w') as fh: - fh.write(f"{genome_version}\n" if genome_version else 'Genome unknown') - fh.write(f"{snpeff_version}\n" if snpeff_version else 'SnpEff version unknown') + with open(dataset.file_name, "w") as fh: + fh.write(f"{genome_version}\n" if genome_version else "Genome unknown") + fh.write(f"{snpeff_version}\n" if snpeff_version else "SnpEff version unknown") if annotations: fh.write(f"annotations: {','.join(annotations)}\n") if regulations: @@ -650,36 +808,49 @@ class SnpSiftDbNSFP(Text): - Create tabix index $ tabix -s 1 -b 2 -e 2 dbNSFP2.3.txt.gz """ - file_ext = "snpsiftdbnsfp" - composite_type = 'auto_primary_file' - MetadataElement(name='reference_name', default='dbSNFP', desc='Reference Name', readonly=True, visible=True, set_in_upload=True, no_value='dbSNFP') + file_ext = "snpsiftdbnsfp" + composite_type = "auto_primary_file" + + MetadataElement( + name="reference_name", + default="dbSNFP", + desc="Reference Name", + readonly=True, + visible=True, + set_in_upload=True, + no_value="dbSNFP", + ) MetadataElement(name="bgzip", default=None, desc="dbNSFP bgzip", readonly=True, visible=True) MetadataElement(name="index", default=None, desc="Tabix Index File", readonly=True, visible=True) MetadataElement(name="annotation", default=[], desc="Annotation Names", readonly=True, visible=True, no_value=[]) def __init__(self, **kwd): super().__init__(**kwd) - self.add_composite_file('%s.gz', description='dbNSFP bgzip', substitute_name_with_metadata='reference_name', is_binary=True) - self.add_composite_file('%s.gz.tbi', description='Tabix Index File', substitute_name_with_metadata='reference_name', is_binary=True) + self.add_composite_file( + "%s.gz", description="dbNSFP bgzip", substitute_name_with_metadata="reference_name", is_binary=True + ) + self.add_composite_file( + "%s.gz.tbi", description="Tabix Index File", substitute_name_with_metadata="reference_name", is_binary=True + ) def generate_primary_file(self, dataset=None): """ This is called only at upload to write the html file cannot rename the datasets here - they come with the default unfortunately """ - return 'SnpSiftDbNSFP Composite Dataset' + return "SnpSiftDbNSFP Composite Dataset" def regenerate_primary_file(self, dataset): """ cannot do this until we are setting metadata """ annotations = f"dbNSFP Annotations: {','.join(dataset.metadata.annotation)}\n" - with open(dataset.file_name, 'a') as f: + with open(dataset.file_name, "a") as f: if dataset.metadata.bgzip: bn = dataset.metadata.bgzip f.write(bn) - f.write('\n') + f.write("\n") f.write(annotations) def set_meta(self, dataset, overwrite=True, **kwd): @@ -688,35 +859,40 @@ class SnpSiftDbNSFP(Text): if os.path.exists(efp): flist = os.listdir(efp) for fname in flist: - if fname.endswith('.gz'): + if fname.endswith(".gz"): dataset.metadata.bgzip = fname try: - with gzip.open(os.path.join(efp, fname), 'rt') as fh: + with gzip.open(os.path.join(efp, fname), "rt") as fh: buf = fh.read(5000) lines = buf.splitlines() - headers = lines[0].split('\t') + headers = lines[0].split("\t") dataset.metadata.annotation = headers[4:] except Exception as e: log.warning("set_meta fname: %s %s", fname, unicodify(e)) - if fname.endswith('.tbi'): + if fname.endswith(".tbi"): dataset.metadata.index = fname self.regenerate_primary_file(dataset) except Exception as e: - log.warning("set_meta fname: %s %s", dataset.file_name if dataset and dataset.file_name else 'Unkwown', unicodify(e)) + log.warning( + "set_meta fname: %s %s", + dataset.file_name if dataset and dataset.file_name else "Unkwown", + unicodify(e), + ) def set_peek(self, dataset): if not dataset.dataset.purged: dataset.peek = f"{dataset.metadata.reference_name} : {','.join(dataset.metadata.annotation)}" - dataset.blurb = f'{dataset.metadata.reference_name}' + dataset.blurb = f"{dataset.metadata.reference_name}" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disc' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disc" @build_sniff_from_prefix class IQTree(Text): """IQ-TREE format""" - file_ext = 'iqtree' + + file_ext = "iqtree" def sniff_prefix(self, file_prefix: FilePrefix): """ @@ -748,6 +924,7 @@ class Paf(Text): https://github.com/lh3/miniasm/blob/master/PAF.md """ + file_ext = "paf" def sniff_prefix(self, file_prefix: FilePrefix): @@ -763,13 +940,13 @@ class Paf(Text): return False for i in (1, 2, 3, 6, 7, 8, 9, 10, 11): int(line[i]) - if line[4] not in ('+', '-'): + if line[4] not in ("+", "-"): return False if not (0 <= int(line[11]) <= 255): return False # Check that the optional columns after the 12th contain SAM-like typed key-value pairs for i in range(12, len(line)): - if len(line[i].split(':')) != 3: + if len(line[i].split(":")) != 3: return False found_valid_lines = True return found_valid_lines @@ -782,6 +959,7 @@ class Gfa1(Text): http://gfa-spec.github.io/GFA-spec/GFA1.html """ + file_ext = "gfa1" def sniff_prefix(self, file_prefix: FilePrefix): @@ -795,27 +973,27 @@ class Gfa1(Text): """ found_valid_lines = False for line in iter_headers(file_prefix, "\t"): - if line[0].startswith('#'): + if line[0].startswith("#"): continue - if line[0] == 'H': - return len(line) == 2 and line[1] == 'VN:Z:1.0' - elif line[0] == 'S': + if line[0] == "H": + return len(line) == 2 and line[1] == "VN:Z:1.0" + elif line[0] == "S": if len(line) < 3: return False - elif line[0] == 'L': + elif line[0] == "L": if len(line) < 6: return False for i in (2, 4): - if line[i] not in ('+', '-'): + if line[i] not in ("+", "-"): return False - elif line[0] == 'C': + elif line[0] == "C": if len(line) < 7: return False for i in (2, 4): - if line[i] not in ('+', '-'): + if line[i] not in ("+", "-"): return False int(line[5]) - elif line[0] == 'P': + elif line[0] == "P": if len(line) < 4: return False else: @@ -831,6 +1009,7 @@ class Gfa2(Text): https://github.com/GFA-spec/GFA-spec/blob/master/GFA2.md """ + file_ext = "gfa2" def sniff_prefix(self, file_prefix: FilePrefix): @@ -844,23 +1023,23 @@ class Gfa2(Text): """ found_valid_lines = False for line in iter_headers(file_prefix, "\t"): - if line[0].startswith('#'): + if line[0].startswith("#"): continue - if line[0] == 'H': - return len(line) >= 2 and line[1] == 'VN:Z:2.0' - elif line[0] == 'S': + if line[0] == "H": + return len(line) >= 2 and line[1] == "VN:Z:2.0" + elif line[0] == "S": if len(line) < 3: return False - elif line[0] == 'F': + elif line[0] == "F": if len(line) < 8: return False - elif line[0] == 'E': + elif line[0] == "E": if len(line) < 9: return False - elif line[0] == 'G': + elif line[0] == "G": if len(line) < 6: return False - elif line[0] == 'O' or line[0] == 'U': + elif line[0] == "O" or line[0] == "U": if len(line) < 3: return False else: @@ -872,23 +1051,24 @@ class Gfa2(Text): @build_sniff_from_prefix class Yaml(Text): """Yaml files""" + file_ext = "yaml" def sniff_prefix(self, file_prefix: FilePrefix): """ - Try to load the string with the yaml module. If successful it's a yaml file. + Try to load the string with the yaml module. If successful it's a yaml file. """ return self._looks_like_yaml(file_prefix) def get_mime(self): """Returns the mime type of the datatype""" - return 'application/yaml' + return "application/yaml" def _yield_user_file_content(self, trans, from_dataset, filename, headers: Headers): # Override non-standard application/yaml mediatype with # non-standard text/x-yaml, so preview is shown in preview iframe, # instead of downloading the file. - headers['content-type'] = 'text/x-yaml' + headers["content-type"] = "text/x-yaml" return super()._yield_user_file_content(trans, from_dataset, filename, headers) def _looks_like_yaml(self, file_prefix: FilePrefix): diff --git a/lib/galaxy/datatypes/tracks.py b/lib/galaxy/datatypes/tracks.py index 6d28363c8da..02fd0f50882 100644 --- a/lib/galaxy/datatypes/tracks.py +++ b/lib/galaxy/datatypes/tracks.py @@ -22,8 +22,8 @@ class UCSCTrackHub(Html): Datatype for UCSC TrackHub """ - file_ext = 'trackhub' - composite_type = 'auto_primary_file' + file_ext = "trackhub" + composite_type = "auto_primary_file" def generate_primary_file(self, dataset=None): """ @@ -31,23 +31,24 @@ class UCSCTrackHub(Html): cannot rename the datasets here - they come with the default unfortunately """ rval = [ - 'Files for Composite Dataset (%s)

                \ - This composite dataset is composed of the following files:

                  ' % ( - self.file_ext)] + "Files for Composite Dataset (%s)

                  \ + This composite dataset is composed of the following files:

                    " + % (self.file_ext) + ] for composite_name, composite_file in self.get_composite_files(dataset=dataset).items(): - opt_text = '' + opt_text = "" if composite_file.optional: - opt_text = ' (optional)' + opt_text = " (optional)" rval.append(f'
                  • {composite_name}{opt_text}') - rval.append('
                  ') + rval.append("
                ") return "\n".join(rval) def set_peek(self, dataset): if not dataset.dataset.purged: dataset.peek = "Track Hub structure: Visualization in UCSC Track Hub" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def display_peek(self, dataset): try: diff --git a/lib/galaxy/datatypes/triples.py b/lib/galaxy/datatypes/triples.py index 29e9611761c..711c71d92a3 100644 --- a/lib/galaxy/datatypes/triples.py +++ b/lib/galaxy/datatypes/triples.py @@ -12,19 +12,20 @@ from . import ( binary, data, text, - xml + xml, ) log = logging.getLogger(__name__) -TURTLE_PREFIX_PATTERN = re.compile(r'@prefix\s+[^:]*:\s+<[^>]*>\s\.') -TURTLE_BASE_PATTERN = re.compile(r'@base\s+<[^>]*>\s\.') +TURTLE_PREFIX_PATTERN = re.compile(r"@prefix\s+[^:]*:\s+<[^>]*>\s\.") +TURTLE_BASE_PATTERN = re.compile(r"@base\s+<[^>]*>\s\.") class Triples(data.Data): """ The abstract base class for the file format that can contain triples """ + edam_data = "data_0582" edam_format = "format_2376" file_ext = "triples" @@ -39,10 +40,10 @@ class Triples(data.Data): """Set the peek and blurb text""" if not dataset.dataset.purged: dataset.peek = data.get_file_peek(dataset.file_name) - dataset.blurb = 'Triple data' + dataset.blurb = "Triple data" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" @build_sniff_from_prefix @@ -50,12 +51,13 @@ class NTriples(data.Text, Triples): """ The N-Triples triple data format """ + edam_format = "format_3256" file_ext = "nt" def sniff_prefix(self, file_prefix: FilePrefix): # . - if re.compile(r'<[^>]*>\s<[^>]*>\s<[^>]*>\s\.').search(file_prefix.contents_header): + if re.compile(r"<[^>]*>\s<[^>]*>\s<[^>]*>\s\.").search(file_prefix.contents_header): return True return False @@ -63,16 +65,17 @@ class NTriples(data.Text, Triples): """Set the peek and blurb text""" if not dataset.dataset.purged: dataset.peek = data.get_file_peek(dataset.file_name) - dataset.blurb = 'N-Triples triple data' + dataset.blurb = "N-Triples triple data" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" class N3(data.Text, Triples): """ The N3 triple data format """ + edam_format = "format_3257" file_ext = "n3" @@ -86,10 +89,10 @@ class N3(data.Text, Triples): """Set the peek and blurb text""" if not dataset.dataset.purged: dataset.peek = data.get_file_peek(dataset.file_name) - dataset.blurb = 'Notation-3 Triple data' + dataset.blurb = "Notation-3 Triple data" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" @build_sniff_from_prefix @@ -97,6 +100,7 @@ class Turtle(data.Text, Triples): """ The Turtle triple data format """ + edam_format = "format_3255" file_ext = "ttl" @@ -113,10 +117,10 @@ class Turtle(data.Text, Triples): """Set the peek and blurb text""" if not dataset.dataset.purged: dataset.peek = data.get_file_peek(dataset.file_name) - dataset.blurb = 'Turtle triple data' + dataset.blurb = "Turtle triple data" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" # TODO: we might want to look at rdflib or a similar, larger lib/egg @@ -125,12 +129,15 @@ class Rdf(xml.GenericXml, Triples): """ Resource Description Framework format (http://www.w3.org/RDF/). """ + edam_format = "format_3261" file_ext = "rdf" def sniff_prefix(self, file_prefix: FilePrefix): # %i)." % (size, self.MAX_SEQUENCE_SIZE) + assert ( + size <= self.MAX_SEQUENCE_SIZE + ), "Maximum length allowed for an individual sequence has been exceeded (%i > %i)." % ( + size, + self.MAX_SEQUENCE_SIZE, + ) species = species or [] self.size = size if not temp_file_handler: @@ -188,6 +193,7 @@ class RegionAlignment: if len(base) != 1: raise Exception("A genomic position can only have a length of 1.") return self.set_range(index, species, base) + # sets a range for a species def set_range(self, index, species, bases): @@ -212,7 +218,6 @@ class RegionAlignment: class GenomicRegionAlignment(RegionAlignment): - def __init__(self, start, end, species=None, temp_file_handler=None): species = species or [] RegionAlignment.__init__(self, end - start, species, temp_file_handler=temp_file_handler) @@ -236,7 +241,9 @@ class SplicedAlignment: temp_file_handler = TempFileHandler() self.temp_file_handler = temp_file_handler for i in range(len(exon_starts)): - self.exons.append(GenomicRegionAlignment(exon_starts[i], exon_ends[i], species, temp_file_handler=temp_file_handler)) + self.exons.append( + GenomicRegionAlignment(exon_starts[i], exon_ends[i], species, temp_file_handler=temp_file_handler) + ) # returns the names for species found in alignment, skipping names as requested def get_species_names(self, skip=None): @@ -290,13 +297,13 @@ def maf_index_by_uid(maf_uid, index_location_file): # read each line, if not enough fields, go to next line if line[0:1] == "#": continue - fields = line.split('\t') + fields = line.split("\t") if maf_uid == fields[1]: try: maf_files = fields[4].replace("\n", "").replace("\r", "").split(",") return bx.align.maf.MultiIndexed(maf_files, keep_open=True, parse_e_rows=False) except Exception as e: - raise Exception(f'MAF UID ({maf_uid}) found, but configuration appears to be malformed: {e}') + raise Exception(f"MAF UID ({maf_uid}) found, but configuration appears to be malformed: {e}") except Exception: pass return None @@ -348,7 +355,7 @@ def build_maf_index_species_chromosomes(filename, index_species=None): indexes.add(c.src, forward_strand_start, forward_strand_end, pos, max=c.src_size) except Exception as e: # most likely a bad MAF - log.debug(f'Building MAF index on {filename} failed: {e}') + log.debug(f"Building MAF index on {filename} failed: {e}") return (None, [], {}, 0) return (indexes, species, species_chromosomes, blocks) @@ -357,9 +364,12 @@ def build_maf_index_species_chromosomes(filename, index_species=None): def build_maf_index(maf_file, species=None): indexes, *_ = build_maf_index_species_chromosomes(maf_file, species) if indexes is not None: - with tempfile.NamedTemporaryFile(mode='w', delete=False) as index: + with tempfile.NamedTemporaryFile(mode="w", delete=False) as index: indexes.write(index) - return (bx.align.maf.Indexed(maf_file, index_filename=index.name, keep_open=True, parse_e_rows=False), index.name) + return ( + bx.align.maf.Indexed(maf_file, index_filename=index.name, keep_open=True, parse_e_rows=False), + index.name, + ) return (None, None) @@ -420,17 +430,25 @@ def orient_block_by_region(block, src, region, force_strand=None): # if force_strand / region.strand not in strand cache, reverse complement # we could have 2 sequences with same src, overlapping region, on different strands, this would cause no reverse_complementing strands = [c.strand for c in iter_components_by_src(block, src) if component_overlaps_region(c, region)] - if strands and (force_strand is None and region.strand not in strands) or (force_strand is not None and force_strand not in strands): + if ( + strands + and (force_strand is None and region.strand not in strands) + or (force_strand is not None and force_strand not in strands) + ): block = block.reverse_complement() return block def get_oriented_chopped_blocks_for_region(index, src, region, species=None, mincols=0, force_strand=None): - for block, _, _ in get_oriented_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols, force_strand): + for block, _, _ in get_oriented_chopped_blocks_with_index_offset_for_region( + index, src, region, species, mincols, force_strand + ): yield block -def get_oriented_chopped_blocks_with_index_offset_for_region(index, src, region, species=None, mincols=0, force_strand=None): +def get_oriented_chopped_blocks_with_index_offset_for_region( + index, src, region, species=None, mincols=0, force_strand=None +): for block, idx, offset in get_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols): yield orient_block_by_region(block, src, region, force_strand), idx, offset @@ -442,7 +460,9 @@ def iter_blocks_split_by_src(block, src): new_block.text_size = block.text_size for c in block.components: if c == src_c or c.src != src: - new_block.add_component(deepcopy(c)) # components have reference to alignment, don't want to lose reference to original alignment block in original components + new_block.add_component( + deepcopy(c) + ) # components have reference to alignment, don't want to lose reference to original alignment block in original components yield new_block @@ -477,7 +497,9 @@ def iter_blocks_split_by_species(block, species=None): for c in iter_components_by_src_start(block, spec): spec_dict[spec].append(c) - empty_block = bx.align.Alignment(score=block.score, attributes=deepcopy(block.attributes)) # should we copy attributes? + empty_block = bx.align.Alignment( + score=block.score, attributes=deepcopy(block.attributes) + ) # should we copy attributes? empty_block.text_size = block.text_size # call recursive function to split into each combo of spec/blocks for value in __split_components_by_species(list(spec_dict.values()), empty_block): @@ -499,12 +521,25 @@ def get_chopped_blocks_with_index_offset_for_region(index, src, region, species= # returns a filled region alignment for specified regions -def get_region_alignment(index, primary_species, chrom, start, end, strand='+', species=None, mincols=0, overwrite_with_gaps=True, temp_file_handler=None): +def get_region_alignment( + index, + primary_species, + chrom, + start, + end, + strand="+", + species=None, + mincols=0, + overwrite_with_gaps=True, + temp_file_handler=None, +): if species is not None: alignment = RegionAlignment(end - start, species, temp_file_handler=temp_file_handler) else: alignment = RegionAlignment(end - start, primary_species, temp_file_handler=temp_file_handler) - return fill_region_alignment(alignment, index, primary_species, chrom, start, end, strand, species, mincols, overwrite_with_gaps) + return fill_region_alignment( + alignment, index, primary_species, chrom, start, end, strand, species, mincols, overwrite_with_gaps + ) # reduces a block to only positions exisiting in the src provided @@ -516,19 +551,21 @@ def reduce_block_by_primary_genome(block, species, chromosome, region_start): start_offset = ref.start - region_start species_texts = {} for c in block.components: - species_texts[c.src.split('.')[0]] = list(c.text) + species_texts[c.src.split(".")[0]] = list(c.text) # remove locations which are gaps in the primary species, starting from the downstream end for i in range(len(species_texts[species]) - 1, -1, -1): - if species_texts[species][i] == '-': + if species_texts[species][i] == "-": for text in species_texts.values(): text.pop(i) for spec, text in species_texts.items(): - species_texts[spec] = ''.join(text) + species_texts[spec] = "".join(text) return (start_offset, species_texts) # fills a region alignment -def fill_region_alignment(alignment, index, primary_species, chrom, start, end, strand='+', species=None, mincols=0, overwrite_with_gaps=True): +def fill_region_alignment( + alignment, index, primary_species, chrom, start, end, strand="+", species=None, mincols=0, overwrite_with_gaps=True +): region = bx.intervals.Interval(start, end) region.chrom = chrom region.strand = strand @@ -546,10 +583,12 @@ def fill_region_alignment(alignment, index, primary_species, chrom, start, end, blocks.append((score, idx, offset)) # gap_chars_tuple = tuple( GAP_CHARS ) - gap_chars_str = ''.join(GAP_CHARS) + gap_chars_str = "".join(GAP_CHARS) # Loop through ordered blocks and layer by increasing score for block_dict in blocks: - for block in iter_blocks_split_by_species(block_dict[1].get_at_offset(block_dict[2])): # need to handle each occurance of sequence in block seperately + for block in iter_blocks_split_by_species( + block_dict[1].get_at_offset(block_dict[2]) + ): # need to handle each occurance of sequence in block seperately if component_overlaps_region(block.get_component_by_src(primary_src), region): block = chop_block_by_region(block, primary_src, region, species, mincols) # chop block block = orient_block_by_region(block, primary_src, region) # orient block @@ -559,7 +598,9 @@ def fill_region_alignment(alignment, index, primary_species, chrom, start, end, text = text.rstrip(gap_chars_str) gap_offset = 0 # while text.startswith( gap_chars_tuple ): - while True in [text.startswith(gap_char) for gap_char in GAP_CHARS]: # python2.4 doesn't accept a tuple for .startswith() + while True in [ + text.startswith(gap_char) for gap_char in GAP_CHARS + ]: # python2.4 doesn't accept a tuple for .startswith() gap_offset += 1 text = text[1:] if not text: @@ -575,19 +616,32 @@ def fill_region_alignment(alignment, index, primary_species, chrom, start, end, # returns a filled spliced region alignment for specified region with start and end lists -def get_spliced_region_alignment(index, primary_species, chrom, starts, ends, strand='+', species=None, mincols=0, overwrite_with_gaps=True, temp_file_handler=None): +def get_spliced_region_alignment( + index, + primary_species, + chrom, + starts, + ends, + strand="+", + species=None, + mincols=0, + overwrite_with_gaps=True, + temp_file_handler=None, +): # create spliced alignment object if species is not None: alignment = SplicedAlignment(starts, ends, species, temp_file_handler=temp_file_handler) else: alignment = SplicedAlignment(starts, ends, [primary_species], temp_file_handler=temp_file_handler) for exon in alignment.exons: - fill_region_alignment(exon, index, primary_species, chrom, exon.start, exon.end, strand, species, mincols, overwrite_with_gaps) + fill_region_alignment( + exon, index, primary_species, chrom, exon.start, exon.end, strand, species, mincols, overwrite_with_gaps + ) return alignment # loop through string array, only return non-commented lines -def line_enumerator(lines, comment_start='#'): +def line_enumerator(lines, comment_start="#"): i = 0 for line in lines: if not line.startswith(comment_start): @@ -607,16 +661,16 @@ def get_starts_ends_fields_from_gene_bed(line): raise Exception(f"Not a proper 12 column BED line ({line}).") tx_start = int(fields[1]) strand = fields[5] - if strand != '-': - strand = '+' # Default strand is + + if strand != "-": + strand = "+" # Default strand is + cds_start = int(fields[6]) cds_end = int(fields[7]) # Calculate and store starts and ends of coding exons region_start, region_end = cds_start, cds_end - exon_starts = list(map(int, fields[11].rstrip(',\n').split(','))) + exon_starts = list(map(int, fields[11].rstrip(",\n").split(","))) exon_starts = [x + tx_start for x in exon_starts] - exon_ends = list(map(int, fields[10].rstrip(',').split(','))) + exon_ends = list(map(int, fields[10].rstrip(",").split(","))) exon_ends = [x + y for x, y in zip(exon_starts, exon_ends)] for start, end in zip(exon_starts, exon_ends): start = max(start, region_start) @@ -651,7 +705,9 @@ def sort_block_components_by_block(block1, block2): # orders the components in block1 by the index of the component in block2 # block1 must be a subset of block2 # occurs in-place - return block1.components.sort(key=functools.cmp_to_key(lambda x, y: block2.components.index(x) - block2.components.index(y))) + return block1.components.sort( + key=functools.cmp_to_key(lambda x, y: block2.components.index(x) - block2.components.index(y)) + ) def get_species_in_maf(maf_filename): @@ -665,8 +721,8 @@ def get_species_in_maf(maf_filename): def parse_species_option(species): if species: - species = species.split(',') - if 'None' not in species: + species = species.split(",") + if "None" not in species: return species return None # provided species was '', None, or had 'None' in it @@ -677,12 +733,18 @@ def remove_temp_index_file(index_filename): except Exception: pass + # Below are methods to deal with FASTA files def get_fasta_header(component, attributes=None, suffix=None): attributes = attributes or {} - header = ">%s(%s):%i-%i|" % (component.src, component.strand, component.get_forward_strand_start(), component.get_forward_strand_end()) + header = ">%s(%s):%i-%i|" % ( + component.src, + component.strand, + component.get_forward_strand_start(), + component.get_forward_strand_end(), + ) for key, value in attributes.items(): header = f"{header}{key}={value}|" if suffix: @@ -696,33 +758,33 @@ def get_attributes_from_fasta_header(header): if not header: return {} attributes = {} - header = header.lstrip('>') + header = header.lstrip(">") header = header.strip() - fields = header.split('|') + fields = header.split("|") try: region = fields[0] - region = region.split('(', 1) - temp = region[0].split('.', 1) - attributes['species'] = temp[0] + region = region.split("(", 1) + temp = region[0].split(".", 1) + attributes["species"] = temp[0] if len(temp) == 2: - attributes['chrom'] = temp[1] + attributes["chrom"] = temp[1] else: - attributes['chrom'] = temp[0] - region = region[1].split(')', 1) - attributes['strand'] = region[0] - region = region[1].lstrip(':').split('-') - attributes['start'] = int(region[0]) - attributes['end'] = int(region[1]) + attributes["chrom"] = temp[0] + region = region[1].split(")", 1) + attributes["strand"] = region[0] + region = region[1].lstrip(":").split("-") + attributes["start"] = int(region[0]) + attributes["end"] = int(region[1]) except Exception: # fields 0 is not a region coordinate pass if len(fields) > 2: for i in range(1, len(fields) - 1): - prop = fields[i].split('=', 1) + prop = fields[i].split("=", 1) if len(prop) == 2: attributes[prop[0]] = prop[1] if len(fields) > 1: - attributes['__suffix__'] = fields[-1] + attributes["__suffix__"] = fields[-1] return attributes @@ -733,9 +795,10 @@ def iter_fasta_alignment(filename): self.text = text def extend(self, text): - self.text = self.text + text.replace('\n', '').replace('\r', '').strip() + self.text = self.text + text.replace("\n", "").replace("\r", "").strip() + # yields a list of fastaComponents for a FASTA file - f = open(filename, 'rb') + f = open(filename, "rb") components = [] # cur_component = None while True: @@ -749,8 +812,8 @@ def iter_fasta_alignment(filename): if components: yield components components = [] - elif line.startswith('>'): + elif line.startswith(">"): attributes = get_attributes_from_fasta_header(line) - components.append(fastaComponent(attributes['species'])) + components.append(fastaComponent(attributes["species"])) elif components: components[-1].extend(line) diff --git a/lib/galaxy/datatypes/xml.py b/lib/galaxy/datatypes/xml.py index e3a1dfd5e15..a975d8ba761 100644 --- a/lib/galaxy/datatypes/xml.py +++ b/lib/galaxy/datatypes/xml.py @@ -18,14 +18,15 @@ from . import ( log = logging.getLogger(__name__) -OWL_MARKER = re.compile(r'\>> GenericXml().sniff( fname ) False """ - return file_prefix.startswith(' 1: - raise NotImplementedError("Merging multiple XML files is non-trivial and must be implemented for each XML type") + raise NotImplementedError( + "Merging multiple XML files is non-trivial and must be implemented for each XML type" + ) # For one file only, use base class method (move/copy) data.Text.merge(split_files, output_file) - @dataproviders.decorators.dataprovider_factory('xml', dataproviders.hierarchy.XMLDataProvider.settings) + @dataproviders.decorators.dataprovider_factory("xml", dataproviders.hierarchy.XMLDataProvider.settings) def xml_dataprovider(self, dataset, **settings): dataset_source = dataproviders.dataset.DatasetDataProvider(dataset) return dataproviders.hierarchy.XMLDataProvider(dataset_source, **settings) @@ -77,31 +80,33 @@ class GenericXml(data.Text): @disable_parent_class_sniffing class MEMEXml(GenericXml): """MEME XML Output data""" + file_ext = "memexml" def set_peek(self, dataset): """Set the peek and blurb text""" if not dataset.dataset.purged: dataset.peek = data.get_file_peek(dataset.file_name) - dataset.blurb = 'MEME XML data' + dataset.blurb = "MEME XML data" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" @disable_parent_class_sniffing class CisML(GenericXml): """CisML XML data""" # see: http://www.ncbi.nlm.nih.gov/pubmed/15001475 + file_ext = "cisml" def set_peek(self, dataset): """Set the peek and blurb text""" if not dataset.dataset.purged: dataset.peek = data.get_file_peek(dataset.file_name) - dataset.blurb = 'CisML data' + dataset.blurb = "CisML data" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" class Dzi(GenericXml): @@ -111,18 +116,29 @@ class Dzi(GenericXml): """ # General elements. - MetadataElement(name="base_name", desc="Base name for this dataset", default='DeepZoomImage', readonly=True, set_in_upload=True) + MetadataElement( + name="base_name", desc="Base name for this dataset", default="DeepZoomImage", readonly=True, set_in_upload=True + ) MetadataElement(name="format", desc="File format of the tiles", default=None, readonly=True, visible=True) MetadataElement(name="tile_size", desc="Size of tiles", default=None, readonly=True, visible=True) # Collection elements. - MetadataElement(name="max_level", desc="Max pyramid level", default=None, readonly=True, optional=True, visible=True) + MetadataElement( + name="max_level", desc="Max pyramid level", default=None, readonly=True, optional=True, visible=True + ) MetadataElement(name="quality", desc="Quality", default=None, readonly=True, optional=True, visible=True) # Image elements. MetadataElement(name="height", desc="Size height", default=None, readonly=True, optional=True, visible=True) - MetadataElement(name="overlap", desc="Overlap of all four sides of tiles", default=None, readonly=True, optional=True, visible=True) + MetadataElement( + name="overlap", + desc="Overlap of all four sides of tiles", + default=None, + readonly=True, + optional=True, + visible=True, + ) MetadataElement(name="width", desc="Size width", default=None, readonly=True, optional=True, visible=True) - file_ext = 'dzi' + file_ext = "dzi" def __init__(self, **kwd): super().__init__(**kwd) @@ -130,31 +146,31 @@ class Dzi(GenericXml): def set_meta(self, dataset, **kwd): tree = util.parse_xml(dataset.file_name) root = tree.getroot() - dataset.metadata.format = root.get('Format') - dataset.metadata.tile_size = root.get('TileSize') + dataset.metadata.format = root.get("Format") + dataset.metadata.tile_size = root.get("TileSize") # DeepZoom image files can include # xml namespace attributes. - if root.tag.find('Collection') >= 0: - dataset.metadata.max_level = root.get('MaxLevel') - dataset.metadata.quality = root.get('Quality') - elif root.tag.find('Image') >= 0: - dataset.metadata.overlap = root.get('Overlap') + if root.tag.find("Collection") >= 0: + dataset.metadata.max_level = root.get("MaxLevel") + dataset.metadata.quality = root.get("Quality") + elif root.tag.find("Image") >= 0: + dataset.metadata.overlap = root.get("Overlap") for elem in root: - if elem.tag.find('Size') >= 0: - dataset.metadata.width = elem.get('Width') - dataset.metadata.height = elem.get('Height') + if elem.tag.find("Size") >= 0: + dataset.metadata.width = elem.get("Width") + dataset.metadata.height = elem.get("Height") def get_visualizations(self, dataset): - """ Returns a list of visualizations for datatype""" - return ['openseadragon'] + """Returns a list of visualizations for datatype""" + return ["openseadragon"] def set_peek(self, dataset): if not dataset.dataset.purged: dataset.peek = data.get_file_peek(dataset.file_name) dataset.blurb = "Deep Zoom Image" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disc' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disc" def sniff_prefix(self, file_prefix: FilePrefix): """ @@ -169,13 +185,14 @@ class Dzi(GenericXml): """ for line in file_prefix.line_iterator(): line = line.lower() - if line.find('= 0 or line.find('= 0: + if line.find("= 0 or line.find("= 0: return True return False class Phyloxml(GenericXml): """Format for defining phyloxml data http://www.phyloxml.org/""" + edam_data = "data_0872" edam_format = "format_3159" file_ext = "phyloxml" @@ -184,13 +201,13 @@ class Phyloxml(GenericXml): """Set the peek and blurb text""" if not dataset.dataset.purged: dataset.peek = data.get_file_peek(dataset.file_name) - dataset.blurb = 'Phyloxml data' + dataset.blurb = "Phyloxml data" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disk" def sniff_prefix(self, file_prefix: FilePrefix): - """"Checking for keyword - 'phyloxml' always in lowercase in the first few lines. + """ "Checking for keyword - 'phyloxml' always in lowercase in the first few lines. >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( '1.phyloxml' ) @@ -210,14 +227,15 @@ class Phyloxml(GenericXml): Returns a list of visualizations for datatype. """ - return ['phyloviz'] + return ["phyloviz"] class Owl(GenericXml): """ - Web Ontology Language OWL format description - http://www.w3.org/TR/owl-ref/ + Web Ontology Language OWL format description + http://www.w3.org/TR/owl-ref/ """ + edam_format = "format_3262" file_ext = "owl" @@ -226,21 +244,22 @@ class Owl(GenericXml): dataset.peek = data.get_file_peek(dataset.file_name) dataset.blurb = "Web Ontology Language OWL" else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disc' + dataset.peek = "file does not exist" + dataset.blurb = "file purged from disc" def sniff_prefix(self, file_prefix: FilePrefix): """ - Checking for keyword - ' str: @@ -49,8 +48,7 @@ class FilesSource(metaclass=abc.ABCMeta): """Realize source path (relative to uri root) to local file system path.""" def write_from(self, target_path, native_path, user_context=None): - """Write file at native path to target_path (relative to uri root). - """ + """Write file at native path to target_path (relative to uri root).""" @abc.abstractmethod def to_dict(self, for_serialization=False, user_context=None): @@ -79,10 +77,7 @@ class BaseFilesSource(FilesSource): return ( user_context is None or user_context.is_admin - or ( - self._user_has_required_roles(user_context) - and self._user_has_required_groups(user_context) - ) + or (self._user_has_required_roles(user_context) and self._user_has_required_groups(user_context)) ) @property diff --git a/lib/galaxy/files/sources/_pyfilesystem2.py b/lib/galaxy/files/sources/_pyfilesystem2.py index 8dba207caf4..13e0f9d2102 100644 --- a/lib/galaxy/files/sources/_pyfilesystem2.py +++ b/lib/galaxy/files/sources/_pyfilesystem2.py @@ -2,7 +2,13 @@ import abc import functools import logging import os -from typing import Any, Dict, List, Optional, Type +from typing import ( + Any, + Dict, + List, + Optional, + Type, +) import fs from fs.base import FS @@ -41,16 +47,16 @@ class PyFilesystem2FilesSource(BaseFilesSource): res.extend(map(to_dict, files)) return res else: - res = h.scandir(path, namespaces=['details']) + res = h.scandir(path, namespaces=["details"]) to_dict = functools.partial(self._resource_info_to_dict, path) return list(map(to_dict, res)) def _realize_to(self, source_path, native_path, user_context=None): - with open(native_path, 'wb') as write_file: + with open(native_path, "wb") as write_file: self._open_fs(user_context=user_context).download(source_path, write_file) def _write_from(self, target_path, native_path, user_context=None): - with open(native_path, 'rb') as read_file: + with open(native_path, "rb") as read_file: openfs = self._open_fs(user_context=user_context) dirname = fs.path.dirname(target_path) if not openfs.isdir(dirname): diff --git a/lib/galaxy/files/sources/anvil.py b/lib/galaxy/files/sources/anvil.py index 5ed894aed2b..aaaf31f134b 100644 --- a/lib/galaxy/files/sources/anvil.py +++ b/lib/galaxy/files/sources/anvil.py @@ -6,7 +6,7 @@ from ._pyfilesystem2 import PyFilesystem2FilesSource class AnVILFilesSource(PyFilesystem2FilesSource): - plugin_type = 'anvil' + plugin_type = "anvil" required_module = AnVILFS required_package = "fs.anvilfs" @@ -16,4 +16,4 @@ class AnVILFilesSource(PyFilesystem2FilesSource): return handle -__all__ = ('AnVILFilesSource', ) +__all__ = ("AnVILFilesSource",) diff --git a/lib/galaxy/files/sources/basespace.py b/lib/galaxy/files/sources/basespace.py index 95c09b250b7..3ed27bdb227 100644 --- a/lib/galaxy/files/sources/basespace.py +++ b/lib/galaxy/files/sources/basespace.py @@ -7,7 +7,7 @@ from ._pyfilesystem2 import PyFilesystem2FilesSource class BaseSpaceFilesSource(PyFilesystem2FilesSource): - plugin_type = 'basespace' + plugin_type = "basespace" required_module = BASESPACEFS required_package = "fs-basespace" @@ -17,4 +17,4 @@ class BaseSpaceFilesSource(PyFilesystem2FilesSource): return handle -__all__ = ('BaseSpaceFilesSource',) +__all__ = ("BaseSpaceFilesSource",) diff --git a/lib/galaxy/files/sources/dropbox.py b/lib/galaxy/files/sources/dropbox.py index 49a9b2f958d..6fa2a3b9d3c 100644 --- a/lib/galaxy/files/sources/dropbox.py +++ b/lib/galaxy/files/sources/dropbox.py @@ -7,7 +7,7 @@ from ._pyfilesystem2 import PyFilesystem2FilesSource class DropboxFilesSource(PyFilesystem2FilesSource): - plugin_type = 'dropbox' + plugin_type = "dropbox" required_module = DropboxFS required_package = "fs.dropboxfs" @@ -17,4 +17,4 @@ class DropboxFilesSource(PyFilesystem2FilesSource): return handle -__all__ = ('DropboxFilesSource',) +__all__ = ("DropboxFilesSource",) diff --git a/lib/galaxy/files/sources/galaxy.py b/lib/galaxy/files/sources/galaxy.py index 2ce2e8a4ef2..0f344906156 100644 --- a/lib/galaxy/files/sources/galaxy.py +++ b/lib/galaxy/files/sources/galaxy.py @@ -4,7 +4,7 @@ from .posix import PosixFilesSource class UserFtpFilesSource(PosixFilesSource): - plugin_type = 'gxftp' + plugin_type = "gxftp" def __init__(self, label="FTP Directory", doc="Galaxy User's FTP Directory", root="${user.ftp_dir}", **kwd): posix_kwds = dict( @@ -28,9 +28,15 @@ class UserFtpFilesSource(PosixFilesSource): class LibraryImportFilesSource(PosixFilesSource): - plugin_type = 'gximport' + plugin_type = "gximport" - def __init__(self, label="Library Import Directory", doc="Galaxy's library import directory", root="${config.library_import_dir}", **kwd): + def __init__( + self, + label="Library Import Directory", + doc="Galaxy's library import directory", + root="${config.library_import_dir}", + **kwd, + ): posix_kwds = dict( id="_import", root=root, @@ -48,9 +54,15 @@ class LibraryImportFilesSource(PosixFilesSource): class UserLibraryImportFilesSource(PosixFilesSource): - plugin_type = 'gxuserimport' + plugin_type = "gxuserimport" - def __init__(self, label="Library User Import Directory", doc="Galaxy's user library import directory", root="${config.user_library_import_dir}/${user.email}", **kwd): + def __init__( + self, + label="Library User Import Directory", + doc="Galaxy's user library import directory", + root="${config.user_library_import_dir}/${user.email}", + **kwd, + ): posix_kwds = dict( id="_userimport", root=root, @@ -67,4 +79,4 @@ class UserLibraryImportFilesSource(PosixFilesSource): return "gxuserimport" -__all__ = ('UserFtpFilesSource', 'LibraryImportFilesSource', 'UserLibraryImportFilesSource') +__all__ = ("UserFtpFilesSource", "LibraryImportFilesSource", "UserLibraryImportFilesSource") diff --git a/lib/galaxy/files/sources/googlecloudstorage.py b/lib/galaxy/files/sources/googlecloudstorage.py index e2d7ad08a87..bbe64e6a983 100644 --- a/lib/galaxy/files/sources/googlecloudstorage.py +++ b/lib/galaxy/files/sources/googlecloudstorage.py @@ -9,22 +9,22 @@ from ._pyfilesystem2 import PyFilesystem2FilesSource class GoogleCloudStorageFilesSource(PyFilesystem2FilesSource): - plugin_type = 'googlecloudstorage' + plugin_type = "googlecloudstorage" required_module = GCSFS required_package = "fs-gcsfs" def _open_fs(self, user_context): props = self._serialization_props(user_context) - bucket_name = props.pop('bucket_name', None) - root_path = props.pop('root_path', None) - project = props.pop('project', None) + bucket_name = props.pop("bucket_name", None) + root_path = props.pop("root_path", None) + project = props.pop("project", None) args = {} - if props.get('anonymous'): - args['client'] = Client.create_anonymous_client() - elif props.get('token'): - args['client'] = Client(project=project, credentials=Credentials(**props)) + if props.get("anonymous"): + args["client"] = Client.create_anonymous_client() + elif props.get("token"): + args["client"] = Client(project=project, credentials=Credentials(**props)) handle = GCSFS(bucket_name, root_path=root_path, retry=0, **args) return handle -__all__ = ('GoogleCloudStorageFilesSource',) +__all__ = ("GoogleCloudStorageFilesSource",) diff --git a/lib/galaxy/files/sources/googledrive.py b/lib/galaxy/files/sources/googledrive.py index 1c8da18849c..f0a4bb7249a 100644 --- a/lib/galaxy/files/sources/googledrive.py +++ b/lib/galaxy/files/sources/googledrive.py @@ -8,7 +8,7 @@ from ._pyfilesystem2 import PyFilesystem2FilesSource class GoogleDriveFilesSource(PyFilesystem2FilesSource): - plugin_type = 'googledrive' + plugin_type = "googledrive" required_module = GoogleDriveFS required_package = "fs.googledrivefs" @@ -19,4 +19,4 @@ class GoogleDriveFilesSource(PyFilesystem2FilesSource): return handle -__all__ = ('GoogleDriveFilesSource',) +__all__ = ("GoogleDriveFilesSource",) diff --git a/lib/galaxy/files/sources/onedata.py b/lib/galaxy/files/sources/onedata.py index d9e29d33c9b..59b98765f3d 100644 --- a/lib/galaxy/files/sources/onedata.py +++ b/lib/galaxy/files/sources/onedata.py @@ -7,7 +7,7 @@ from ._pyfilesystem2 import PyFilesystem2FilesSource class OneDataFilesSource(PyFilesystem2FilesSource): - plugin_type = 'onedata' + plugin_type = "onedata" required_module = OnedataFS required_package = "fs-onedatafs" @@ -17,4 +17,4 @@ class OneDataFilesSource(PyFilesystem2FilesSource): return handle -__all__ = ('OneDataFilesSource',) +__all__ = ("OneDataFilesSource",) diff --git a/lib/galaxy/files/sources/posix.py b/lib/galaxy/files/sources/posix.py index 11aaa7378e8..937c6be397a 100644 --- a/lib/galaxy/files/sources/posix.py +++ b/lib/galaxy/files/sources/posix.py @@ -1,7 +1,11 @@ import functools import os import shutil -from typing import Any, Dict, List +from typing import ( + Any, + Dict, + List, +) from galaxy import exceptions from galaxy.util.path import ( @@ -17,7 +21,7 @@ DEFAULT_ALLOW_SUBDIR_CREATION = True class PosixFilesSource(BaseFilesSource): - plugin_type = 'posix' + plugin_type = "posix" # If this were a PyFilesystem2FilesSource all that would be needed would be, # but we couldn't enforce security our way I suspect. @@ -36,7 +40,7 @@ class PosixFilesSource(BaseFilesSource): def _list(self, path="/", recursive=True, user_context=None): dir_path = self._to_native_path(path, user_context=user_context) if not self._safe_directory(dir_path): - raise exceptions.ObjectNotFound(f'The specified directory does not exist [{dir_path}].') + raise exceptions.ObjectNotFound(f"The specified directory does not exist [{dir_path}].") if recursive: res: List[Dict[str, Any]] = [] effective_root = self._effective_root(user_context) @@ -114,7 +118,9 @@ class PosixFilesSource(BaseFilesSource): def _safe_directory(self, directory): if self.enforce_symlink_security: if not safe_path(directory, allowlist=self._allowlist): - raise exceptions.ConfigDoesNotAllowException(f'directory ({directory}) is a symlink to a location not on the allowlist') + raise exceptions.ConfigDoesNotAllowException( + f"directory ({directory}) is a symlink to a location not on the allowlist" + ) if not os.path.exists(directory): return False @@ -135,4 +141,4 @@ class PosixFilesSource(BaseFilesSource): return self._file_sources_config.symlink_allowlist -__all__ = ('PosixFilesSource',) +__all__ = ("PosixFilesSource",) diff --git a/lib/galaxy/files/sources/s3.py b/lib/galaxy/files/sources/s3.py index 6ef398d7402..4e5652f0712 100644 --- a/lib/galaxy/files/sources/s3.py +++ b/lib/galaxy/files/sources/s3.py @@ -7,7 +7,7 @@ from ._pyfilesystem2 import PyFilesystem2FilesSource class S3FilesSource(PyFilesystem2FilesSource): - plugin_type = 's3' + plugin_type = "s3" required_module = S3FS required_package = "fs-s3fs" @@ -17,4 +17,4 @@ class S3FilesSource(PyFilesystem2FilesSource): return handle -__all__ = ('S3FilesSource',) +__all__ = ("S3FilesSource",) diff --git a/lib/galaxy/files/sources/s3fs.py b/lib/galaxy/files/sources/s3fs.py index ffa6b15d7da..2c85609dd33 100644 --- a/lib/galaxy/files/sources/s3fs.py +++ b/lib/galaxy/files/sources/s3fs.py @@ -1,7 +1,11 @@ import functools import logging import os -from typing import Any, Dict, List +from typing import ( + Any, + Dict, + List, +) try: import s3fs @@ -17,14 +21,14 @@ log = logging.getLogger(__name__) class S3FsFilesSource(BaseFilesSource): - plugin_type = 's3fs' + plugin_type = "s3fs" def __init__(self, **kwd): if s3fs is None: raise Exception("Package s3fs unavailable but required for this file source plugin.") props = self._parse_common_config_opts(kwd) - self._bucket = props.pop("bucket", '') - self._endpoint_url = props.pop('endpoint_url', None) + self._bucket = props.pop("bucket", "") + self._endpoint_url = props.pop("endpoint_url", None) assert self._endpoint_url or self._bucket self._props = props @@ -58,7 +62,7 @@ class S3FsFilesSource(BaseFilesSource): def _open_fs(self, user_context=None): if self._endpoint_url: - self._props.update({'client_kwargs': {'endpoint_url': self._endpoint_url}}) + self._props.update({"client_kwargs": {"endpoint_url": self._endpoint_url}}) fs = s3fs.S3FileSystem(**self._props) return fs @@ -87,4 +91,4 @@ class S3FsFilesSource(BaseFilesSource): return effective_props -__all__ = ('S3FsFilesSource',) +__all__ = ("S3FsFilesSource",) diff --git a/lib/galaxy/files/sources/ssh.py b/lib/galaxy/files/sources/ssh.py index ed2ec98f30c..15114ce687f 100644 --- a/lib/galaxy/files/sources/ssh.py +++ b/lib/galaxy/files/sources/ssh.py @@ -13,7 +13,7 @@ class SshFilesSource(PyFilesystem2FilesSource): def _open_fs(self, user_context): props = self._serialization_props(user_context) - path = props.pop('path') + path = props.pop("path") handle = SSHFS(**props) if path: handle = handle.opendir(path) diff --git a/lib/galaxy/files/sources/webdav.py b/lib/galaxy/files/sources/webdav.py index ba43e106b2d..4a9a3f0e7ce 100644 --- a/lib/galaxy/files/sources/webdav.py +++ b/lib/galaxy/files/sources/webdav.py @@ -7,7 +7,7 @@ from ._pyfilesystem2 import PyFilesystem2FilesSource class WebDavFilesSource(PyFilesystem2FilesSource): - plugin_type = 'webdav' + plugin_type = "webdav" required_module = WebDAVFS required_package = "fs.webdavfs" @@ -17,4 +17,4 @@ class WebDavFilesSource(PyFilesystem2FilesSource): return handle -__all__ = ('WebDavFilesSource',) +__all__ = ("WebDavFilesSource",) diff --git a/lib/galaxy/files/unittest_utils/__init__.py b/lib/galaxy/files/unittest_utils/__init__.py index d500de23e4f..b823894a5ad 100644 --- a/lib/galaxy/files/unittest_utils/__init__.py +++ b/lib/galaxy/files/unittest_utils/__init__.py @@ -2,11 +2,13 @@ import os import tempfile from typing import Tuple -from galaxy.files import ConfiguredFileSources, ConfiguredFileSourcesConfig +from galaxy.files import ( + ConfiguredFileSources, + ConfiguredFileSourcesConfig, +) class TestConfiguredFileSources(ConfiguredFileSources): - def __init__(self, file_sources_config: ConfiguredFileSourcesConfig, conf_dict: dict, test_root: str): super().__init__(file_sources_config, conf_dict=conf_dict) self.test_root = test_root diff --git a/lib/galaxy/forms/forms.py b/lib/galaxy/forms/forms.py index a2a8effc116..80e6c9067f1 100644 --- a/lib/galaxy/forms/forms.py +++ b/lib/galaxy/forms/forms.py @@ -5,7 +5,7 @@ FormDefinition and field factories # Can this functionality be further abstracted and merged with form_builder? from galaxy.model import ( FormDefinition, - FormDefinitionCurrent + FormDefinitionCurrent, ) from galaxy.util import string_as_bool @@ -21,10 +21,12 @@ class FormDefinitionFactory: """ Return new FormDefinition. """ - assert form_type in self.form_types, f'Invalid FormDefinition type ( {form_type} not in {self.form_types.keys()} )' - assert name, 'FormDefinition requires a name' + assert ( + form_type in self.form_types + ), f"Invalid FormDefinition type ( {form_type} not in {self.form_types.keys()} )" + assert name, "FormDefinition requires a name" if description is None: - description = '' + description = "" if layout is None: layout = [] if fields is None: @@ -32,12 +34,14 @@ class FormDefinitionFactory: # Create new FormDefinitionCurrent if form_definition_current is None: form_definition_current = FormDefinitionCurrent() - rval = FormDefinition(name=name, - desc=description, - form_type=self.form_types[form_type], - form_definition_current=form_definition_current, - layout=layout, - fields=fields) + rval = FormDefinition( + name=name, + desc=description, + form_type=self.form_types[form_type], + form_definition_current=form_definition_current, + layout=layout, + fields=fields, + ) form_definition_current.latest_form = rval return rval @@ -45,97 +49,121 @@ class FormDefinitionFactory: """ Return FormDefinition created from an xml element. """ - name = elem.get('name', None) - description = elem.get('description', None) - form_type = elem.get('type', None) + name = elem.get("name", None) + description = elem.get("description", None) + form_type = elem.get("type", None) # load layout layout = [] - layouts_elem = elem.find('layout') + layouts_elem = elem.find("layout") if layouts_elem: - for layout_elem in layouts_elem.findall('grid'): - layout_name = layout_elem.get('name', None) - assert layout_name and layout_name not in layout, 'Layout grid element requires a unique name.' + for layout_elem in layouts_elem.findall("grid"): + layout_name = layout_elem.get("name", None) + assert layout_name and layout_name not in layout, "Layout grid element requires a unique name." layout.append(layout_name) # load fields fields = [] - fields_elem = elem.find('fields') + fields_elem = elem.find("fields") if fields_elem is not None: - for field_elem in fields_elem.findall('field'): - field_type = field_elem.get('type') - assert field_type in self.field_type_factories, f'Invalid form field type ( {field_type} ).' + for field_elem in fields_elem.findall("field"): + field_type = field_elem.get("type") + assert field_type in self.field_type_factories, f"Invalid form field type ( {field_type} )." fields.append(self.field_type_factories[field_type].from_elem(field_elem, layout)) # create and return new form - return self.new(form_type, name, description=description, fields=fields, layout=layout, form_definition_current=form_definition_current) + return self.new( + form_type, + name, + description=description, + fields=fields, + layout=layout, + form_definition_current=form_definition_current, + ) class FormDefinitionFieldFactory: type: str def __get_stored_field_type(self, **kwds): - raise Exception('not implemented') + raise Exception("not implemented") def new(self, name=None, label=None, required=False, helptext=None, default=None, visible=True, layout=None): """ Return new FormDefinition field. """ rval = {} - assert name, 'Must provide a name' - rval['name'] = name + assert name, "Must provide a name" + rval["name"] = name if not label: - rval['label'] = name + rval["label"] = name else: - rval['label'] = label + rval["label"] = label if required: - rval['required'] = 'required' + rval["required"] = "required" else: - rval['required'] = 'optional' + rval["required"] = "optional" if helptext is None: - helptext = '' - rval['helptext'] = helptext + helptext = "" + rval["helptext"] = helptext if default is None: - default = '' - rval['default'] = default - rval['visible'] = visible + default = "" + rval["default"] = default + rval["visible"] = visible # if layout is None: #is this needed? # layout = '' - rval['layout'] = layout + rval["layout"] = layout return rval def from_elem(self, elem, layout=None): """ Return FormDefinition created from an xml element. """ - name = elem.get('name') - label = elem.get('label') - required = string_as_bool(elem.get('required', 'false')) - default = elem.get('value') - helptext = elem.get('helptext') - visible = string_as_bool(elem.get('visible', 'true')) - field_layout = elem.get('layout', None) + name = elem.get("name") + label = elem.get("label") + required = string_as_bool(elem.get("required", "false")) + default = elem.get("value") + helptext = elem.get("helptext") + visible = string_as_bool(elem.get("visible", "true")) + field_layout = elem.get("layout", None) if field_layout: - assert layout and field_layout in layout, f'Invalid layout specified: {field_layout} not in {layout}' - field_layout = str(layout.index(field_layout)) # existing behavior: integer indexes are stored as strings. why? - return self.new(name=name, label=label, required=required, helptext=helptext, default=default, visible=visible, layout=field_layout) + assert layout and field_layout in layout, f"Invalid layout specified: {field_layout} not in {layout}" + field_layout = str( + layout.index(field_layout) + ) # existing behavior: integer indexes are stored as strings. why? + return self.new( + name=name, + label=label, + required=required, + helptext=helptext, + default=default, + visible=visible, + layout=field_layout, + ) class FormDefinitionTextFieldFactory(FormDefinitionFieldFactory): - type = 'text' + type = "text" def __get_stored_field_type(self, area): if area: - return 'TextArea' + return "TextArea" else: - return 'TextField' + return "TextField" - def new(self, name=None, label=None, required=False, helptext=None, default=None, visible=True, layout=None, area=False): + def new( + self, name=None, label=None, required=False, helptext=None, default=None, visible=True, layout=None, area=False + ): """ Return new FormDefinition field. """ - rval = super().new(name=name, label=label, - required=required, helptext=helptext, - default=default, visible=visible, - layout=layout) - rval['type'] = self.__get_stored_field_type(area) + rval = super().new( + name=name, + label=label, + required=required, + helptext=helptext, + default=default, + visible=visible, + layout=layout, + ) + rval["type"] = self.__get_stored_field_type(area) return rval def from_elem(self, elem, layout=None): @@ -143,25 +171,32 @@ class FormDefinitionTextFieldFactory(FormDefinitionFieldFactory): Return FormDefinition field created from an xml element. """ rval = super().from_elem(elem, layout=layout) - rval['type'] = self.__get_stored_field_type(string_as_bool(elem.get('area', 'false'))) + rval["type"] = self.__get_stored_field_type(string_as_bool(elem.get("area", "false"))) return rval class FormDefinitionPasswordFieldFactory(FormDefinitionFieldFactory): - type = 'password' + type = "password" def __get_stored_field_type(self): - return 'PasswordField' + return "PasswordField" - def new(self, name=None, label=None, required=False, helptext=None, default=None, visible=True, layout=None, area=False): + def new( + self, name=None, label=None, required=False, helptext=None, default=None, visible=True, layout=None, area=False + ): """ Return new FormDefinition field. """ - rval = super().new(name=name, label=label, - required=required, helptext=helptext, - default=default, visible=visible, - layout=layout) - rval['type'] = self.__get_stored_field_type() + rval = super().new( + name=name, + label=label, + required=required, + helptext=helptext, + default=default, + visible=visible, + layout=layout, + ) + rval["type"] = self.__get_stored_field_type() return rval def from_elem(self, elem, layout=None): @@ -169,25 +204,30 @@ class FormDefinitionPasswordFieldFactory(FormDefinitionFieldFactory): Return FormDefinition field created from an xml element. """ rval = super().from_elem(elem, layout=layout) - rval['type'] = self.__get_stored_field_type() + rval["type"] = self.__get_stored_field_type() return rval class FormDefinitionAddressFieldFactory(FormDefinitionFieldFactory): - type = 'address' + type = "address" def __get_stored_field_type(self): - return 'AddressField' + return "AddressField" def new(self, name=None, label=None, required=False, helptext=None, default=None, visible=True, layout=None): """ Return new FormDefinition field. """ - rval = super().new(name=name, label=label, - required=required, helptext=helptext, - default=default, visible=visible, - layout=layout) - rval['type'] = self.__get_stored_field_type() + rval = super().new( + name=name, + label=label, + required=required, + helptext=helptext, + default=default, + visible=visible, + layout=layout, + ) + rval["type"] = self.__get_stored_field_type() return rval def from_elem(self, elem, layout=None): @@ -195,25 +235,30 @@ class FormDefinitionAddressFieldFactory(FormDefinitionFieldFactory): Return FormDefinition field created from an xml element. """ rval = super().from_elem(elem, layout=layout) - rval['type'] = self.__get_stored_field_type() + rval["type"] = self.__get_stored_field_type() return rval class FormDefinitionWorkflowFieldFactory(FormDefinitionFieldFactory): - type = 'workflow' + type = "workflow" def __get_stored_field_type(self): - return 'WorkflowField' + return "WorkflowField" def new(self, name=None, label=None, required=False, helptext=None, default=None, visible=True, layout=None): """ Return new FormDefinition field. """ - rval = super().new(name=name, label=label, - required=required, helptext=helptext, - default=default, visible=visible, - layout=layout) - rval['type'] = self.__get_stored_field_type() + rval = super().new( + name=name, + label=label, + required=required, + helptext=helptext, + default=default, + visible=visible, + layout=layout, + ) + rval["type"] = self.__get_stored_field_type() return rval def from_elem(self, elem, layout=None): @@ -221,25 +266,30 @@ class FormDefinitionWorkflowFieldFactory(FormDefinitionFieldFactory): Return FormDefinition field created from an xml element. """ rval = super().from_elem(elem, layout=layout) - rval['type'] = self.__get_stored_field_type() + rval["type"] = self.__get_stored_field_type() return rval class FormDefinitionWorkflowMappingFieldFactory(FormDefinitionFieldFactory): - type = 'workflowmapping' + type = "workflowmapping" def __get_stored_field_type(self): - return 'WorkflowMappingField' + return "WorkflowMappingField" def new(self, name=None, label=None, required=False, helptext=None, default=None, visible=True, layout=None): """ Return new FormDefinition field. """ - rval = super().new(name=name, label=label, - required=required, helptext=helptext, - default=default, visible=visible, - layout=layout) - rval['type'] = self.__get_stored_field_type() + rval = super().new( + name=name, + label=label, + required=required, + helptext=helptext, + default=default, + visible=visible, + layout=layout, + ) + rval["type"] = self.__get_stored_field_type() return rval def from_elem(self, elem, layout=None): @@ -247,25 +297,30 @@ class FormDefinitionWorkflowMappingFieldFactory(FormDefinitionFieldFactory): Return FormDefinition field created from an xml element. """ rval = super().from_elem(elem, layout=layout) - rval['type'] = self.__get_stored_field_type() + rval["type"] = self.__get_stored_field_type() return rval class FormDefinitionHistoryFieldFactory(FormDefinitionFieldFactory): - type = 'history' + type = "history" def __get_stored_field_type(self): - return 'HistoryField' + return "HistoryField" def new(self, name=None, label=None, required=False, helptext=None, default=None, visible=True, layout=None): """ Return new FormDefinition field. """ - rval = super().new(name=name, label=label, - required=required, helptext=helptext, - default=default, visible=visible, - layout=layout) - rval['type'] = self.__get_stored_field_type() + rval = super().new( + name=name, + label=label, + required=required, + helptext=helptext, + default=default, + visible=visible, + layout=layout, + ) + rval["type"] = self.__get_stored_field_type() return rval def from_elem(self, elem, layout=None): @@ -273,32 +328,48 @@ class FormDefinitionHistoryFieldFactory(FormDefinitionFieldFactory): Return FormDefinition field created from an xml element. """ rval = super().from_elem(elem, layout=layout) - rval['type'] = self.__get_stored_field_type() + rval["type"] = self.__get_stored_field_type() return rval class FormDefinitionSelectFieldFactory(FormDefinitionFieldFactory): - type = 'select' + type = "select" def __get_stored_field_type(self, checkboxes): if checkboxes: - return 'CheckboxField' + return "CheckboxField" else: - return 'SelectField' + return "SelectField" - def new(self, name=None, label=None, required=False, helptext=None, default=None, visible=True, layout=None, options=None, checkboxes=False): + def new( + self, + name=None, + label=None, + required=False, + helptext=None, + default=None, + visible=True, + layout=None, + options=None, + checkboxes=False, + ): """ Return new FormDefinition field. """ options = options or [] - rval = super().new(name=name, label=label, - required=required, helptext=helptext, - default=default, visible=visible, - layout=layout) - rval['type'] = self.__get_stored_field_type(checkboxes) + rval = super().new( + name=name, + label=label, + required=required, + helptext=helptext, + default=default, + visible=visible, + layout=layout, + ) + rval["type"] = self.__get_stored_field_type(checkboxes) if options is None: options = [] - rval['selectlist'] = options + rval["selectlist"] = options return rval def from_elem(self, elem, layout=None): @@ -306,22 +377,27 @@ class FormDefinitionSelectFieldFactory(FormDefinitionFieldFactory): Return FormDefinition field created from an xml element. """ rval = super().from_elem(elem, layout=layout) - rval['type'] = self.__get_stored_field_type(string_as_bool(elem.get('checkboxes', 'false'))) + rval["type"] = self.__get_stored_field_type(string_as_bool(elem.get("checkboxes", "false"))) # load select options - rval['selectlist'] = [] - for select_option in elem.findall('option'): - value = select_option.get('value', None) + rval["selectlist"] = [] + for select_option in elem.findall("option"): + value = select_option.get("value", None) assert value is not None, 'Must provide a "value" for a select option' - rval['selectlist'].append(value) + rval["selectlist"].append(value) return rval -field_type_factories = {field.type: field() for field in (FormDefinitionTextFieldFactory, - FormDefinitionPasswordFieldFactory, - FormDefinitionAddressFieldFactory, - FormDefinitionSelectFieldFactory, - FormDefinitionWorkflowFieldFactory, - FormDefinitionWorkflowMappingFieldFactory, - FormDefinitionHistoryFieldFactory)} +field_type_factories = { + field.type: field() + for field in ( + FormDefinitionTextFieldFactory, + FormDefinitionPasswordFieldFactory, + FormDefinitionAddressFieldFactory, + FormDefinitionSelectFieldFactory, + FormDefinitionWorkflowFieldFactory, + FormDefinitionWorkflowMappingFieldFactory, + FormDefinitionHistoryFieldFactory, + ) +} form_factory = FormDefinitionFactory(FORM_TYPES, field_type_factories) diff --git a/lib/galaxy/job_execution/actions/post.py b/lib/galaxy/job_execution/actions/post.py index ede02a4a0ca..21e0215a9cf 100644 --- a/lib/galaxy/job_execution/actions/post.py +++ b/lib/galaxy/job_execution/actions/post.py @@ -20,6 +20,7 @@ class DefaultJobAction: """ Base job action. """ + name = "DefaultJobAction" verbose_name = "Default Job" @@ -28,7 +29,9 @@ class DefaultJobAction: pass @classmethod - def execute_on_mapped_over(cls, trans, sa_session, action, step_inputs, step_outputs, replacement_dict, final_job_state=None): + def execute_on_mapped_over( + cls, trans, sa_session, action, step_inputs, step_outputs, replacement_dict, final_job_state=None + ): pass @classmethod @@ -43,6 +46,7 @@ class EmailAction(DefaultJobAction): """ This action sends an email to the galaxy user responsible for a job. """ + name = "EmailAction" verbose_name = "Email Notification" @@ -53,14 +57,14 @@ class EmailAction(DefaultJobAction): history_id_encoded = app.security.encode_id(job.history_id) link = f"{app.config.galaxy_infrastructure_url}/histories/view?id={history_id_encoded}" if frm is None: - if action.action_arguments and 'host' in action.action_arguments: - host = action.action_arguments['host'] + if action.action_arguments and "host" in action.action_arguments: + host = action.action_arguments["host"] else: host = socket.getfqdn() - frm = f'galaxy-no-reply@{host}' + frm = f"galaxy-no-reply@{host}" to = job.get_user_email() subject = f"Galaxy job completion notification from history '{job.history.name}'" - outdata = ',\n'.join(ds.dataset.display_name() for ds in job.output_datasets) + outdata = ",\n".join(ds.dataset.display_name() for ds in job.output_datasets) body = f"Your Galaxy job generating dataset(s):\n\n{outdata}\n\nis complete as of {datetime.datetime.now().strftime('%I:%M')}. Click the link below to access your data: \n{link}" send_mail(frm, to, subject, body, app.config) except Exception as e: @@ -68,8 +72,10 @@ class EmailAction(DefaultJobAction): @classmethod def get_short_str(cls, pja): - if pja.action_arguments and 'host' in pja.action_arguments: - return f"Email the current user from server {escape(pja.action_arguments['host'])} when this job is complete." + if pja.action_arguments and "host" in pja.action_arguments: + return ( + f"Email the current user from server {escape(pja.action_arguments['host'])} when this job is complete." + ) else: return "Email the current user when this job is complete." @@ -78,6 +84,7 @@ class ValidateOutputsAction(DefaultJobAction): """ This action validates the produced outputs against the expected datatype. """ + name = "ValidateOutputsAction" verbose_name = "Validate Tool Outputs" @@ -98,18 +105,19 @@ class ChangeDatatypeAction(DefaultJobAction): @classmethod def execute(cls, app, sa_session, action, job, replacement_dict, final_job_state=None): for dataset_assoc in job.output_datasets: - if action.output_name == '' or dataset_assoc.name == action.output_name: - app.datatypes_registry.change_datatype(dataset_assoc.dataset, action.action_arguments['newtype']) + if action.output_name == "" or dataset_assoc.name == action.output_name: + app.datatypes_registry.change_datatype(dataset_assoc.dataset, action.action_arguments["newtype"]) for dataset_collection_assoc in job.output_dataset_collection_instances: - if action.output_name == '' or dataset_collection_assoc.name == action.output_name: + if action.output_name == "" or dataset_collection_assoc.name == action.output_name: for dataset_instance in dataset_collection_assoc.dataset_collection_instance.dataset_instances: if dataset_instance: - app.datatypes_registry.change_datatype(dataset_instance, action.action_arguments['newtype']) + app.datatypes_registry.change_datatype(dataset_instance, action.action_arguments["newtype"]) @classmethod def get_short_str(cls, pja): - return "Set the datatype of output '{}' to '{}'".format(escape(pja.output_name), - escape(pja.action_arguments['newtype'])) + return "Set the datatype of output '{}' to '{}'".format( + escape(pja.output_name), escape(pja.action_arguments["newtype"]) + ) class RenameDatasetAction(DefaultJobAction): @@ -117,7 +125,9 @@ class RenameDatasetAction(DefaultJobAction): verbose_name = "Rename Dataset" @classmethod - def execute_on_mapped_over(cls, trans, sa_session, action, step_inputs, step_outputs, replacement_dict, final_job_state=None): + def execute_on_mapped_over( + cls, trans, sa_session, action, step_inputs, step_outputs, replacement_dict, final_job_state=None + ): # Prevent renaming a dataset to the empty string. input_names = {} # Lookp through inputs find one with "to_be_replaced" input @@ -129,15 +139,15 @@ class RenameDatasetAction(DefaultJobAction): new_name = cls._gen_new_name(action, input_names, replacement_dict) if new_name: for name, step_output in step_outputs.items(): - if action.output_name == '' or name == action.output_name: + if action.output_name == "" or name == action.output_name: step_output.name = new_name @classmethod def _gen_new_name(self, action, input_names, replacement_dict): new_name = None - if action.action_arguments and action.action_arguments.get('newname', ''): - new_name = action.action_arguments['newname'] + if action.action_arguments and action.action_arguments.get("newname", ""): + new_name = action.action_arguments["newname"] # TODO: Unify and simplify replacement options. # Add interface through workflow editor UI @@ -183,7 +193,7 @@ class RenameDatasetAction(DefaultJobAction): replacement = input_names.get(input_file_var, "") # In case name was None. - replacement = replacement or '' + replacement = replacement or "" # Do operations on replacement # Any control that is not defined will be ignored. # This should be moved out to a class or module function @@ -229,19 +239,20 @@ class RenameDatasetAction(DefaultJobAction): new_name = cls._gen_new_name(action, input_names, replacement_dict) if new_name: for dataset_assoc in job.output_datasets: - if action.output_name == '' or dataset_assoc.name == action.output_name: + if action.output_name == "" or dataset_assoc.name == action.output_name: dataset_assoc.dataset.name = new_name for dataset_collection_assoc in job.output_dataset_collection_instances: - if action.output_name == '' or dataset_collection_assoc.name == action.output_name: + if action.output_name == "" or dataset_collection_assoc.name == action.output_name: dataset_collection_assoc.dataset_collection_instance.name = new_name @classmethod def get_short_str(cls, pja): # Prevent renaming a dataset to the empty string. - if pja.action_arguments and pja.action_arguments.get('newname', ''): - return "Rename output '{}' to '{}'.".format(escape(pja.output_name), - escape(pja.action_arguments['newname'])) + if pja.action_arguments and pja.action_arguments.get("newname", ""): + return "Rename output '{}' to '{}'.".format( + escape(pja.output_name), escape(pja.action_arguments["newname"]) + ) else: return "Rename action used without a new name specified. Output name will be unchanged." @@ -254,13 +265,15 @@ class HideDatasetAction(DefaultJobAction): def execute(cls, app, sa_session, action, job, replacement_dict, final_job_state=None): if final_job_state != job.states.ERROR: for output_association in job.output_datasets + job.output_dataset_collection_instances: - if action.output_name == '' or output_association.name == action.output_name: + if action.output_name == "" or output_association.name == action.output_name: output_association.item.visible = False @classmethod - def execute_on_mapped_over(cls, trans, sa_session, action, step_inputs, step_outputs, replacement_dict, final_job_state=None): + def execute_on_mapped_over( + cls, trans, sa_session, action, step_inputs, step_outputs, replacement_dict, final_job_state=None + ): for name, step_output in step_outputs.items(): - if action.output_name == '' or name == action.output_name: + if action.output_name == "" or name == action.output_name: step_output.visible = False @classmethod @@ -276,13 +289,15 @@ class DeleteDatasetAction(DefaultJobAction): @classmethod def execute(cls, app, sa_session, action, job, replacement_dict, final_job_state=None): for output_association in job.output_datasets + job.output_dataset_collection_instances: - if action.output_name == '' or output_association.name == action.output_name: + if action.output_name == "" or output_association.name == action.output_name: output_association.item.deleted = True @classmethod - def execute_on_mapped_over(cls, trans, sa_session, action, step_inputs, step_outputs, replacement_dict, final_job_state=None): + def execute_on_mapped_over( + cls, trans, sa_session, action, step_inputs, step_outputs, replacement_dict, final_job_state=None + ): for name, step_output in step_outputs.items(): - if action.output_name == '' or name == action.output_name: + if action.output_name == "" or name == action.output_name: step_output.deleted = True @classmethod @@ -297,12 +312,12 @@ class ColumnSetAction(DefaultJobAction): @classmethod def execute(cls, app, sa_session, action, job, replacement_dict, final_job_state=None): for dataset_assoc in job.output_datasets: - if action.output_name == '' or dataset_assoc.name == action.output_name: + if action.output_name == "" or dataset_assoc.name == action.output_name: for k, v in action.action_arguments.items(): if v: # Try to use both pure integer and 'cX' format. if not isinstance(v, int): - if v[0] == 'c': + if v[0] == "c": v = v[1:] v = int(v) if v != 0: @@ -320,7 +335,7 @@ class SetMetadataAction(DefaultJobAction): @classmethod def execute(cls, app, sa_session, action, job, replacement_dict, final_job_state=None): for data in job.output_datasets: - data.set_metadata(action.action_arguments['newtype']) + data.set_metadata(action.action_arguments["newtype"]) class DeleteIntermediatesAction(DefaultJobAction): @@ -353,7 +368,11 @@ class DeleteIntermediatesAction(DefaultJobAction): return outputs_defined = wfi.workflow.has_outputs_defined() if outputs_defined: - wfi_steps = [wfistep for wfistep in wfi.steps if not wfistep.workflow_step.workflow_outputs and wfistep.workflow_step.type == "tool"] + wfi_steps = [ + wfistep + for wfistep in wfi.steps + if not wfistep.workflow_step.workflow_outputs and wfistep.workflow_step.type == "tool" + ] jobs_to_check = [] for wfi_step in wfi_steps: sa_session.refresh(wfi_step) @@ -366,21 +385,32 @@ class DeleteIntermediatesAction(DefaultJobAction): creating_jobs = [] for input_dataset in j2c.input_datasets: if not input_dataset.dataset: - log.debug(f"PJA Async Issue: No dataset attached to input_dataset {input_dataset.id} during handling of workflow invocation {wfi}") + log.debug( + f"PJA Async Issue: No dataset attached to input_dataset {input_dataset.id} during handling of workflow invocation {wfi}" + ) elif not input_dataset.dataset.creating_job: - log.debug(f"PJA Async Issue: No creating job attached to dataset {input_dataset.dataset.id} during handling of workflow invocation {wfi}") + log.debug( + f"PJA Async Issue: No creating job attached to dataset {input_dataset.dataset.id} during handling of workflow invocation {wfi}" + ) else: creating_jobs.append((input_dataset, input_dataset.dataset.creating_job)) for (input_dataset, creating_job) in creating_jobs: sa_session.refresh(creating_job) sa_session.refresh(input_dataset) - for input_dataset in [x.dataset for (x, creating_job) in creating_jobs if creating_job.workflow_invocation_step and creating_job.workflow_invocation_step.workflow_invocation == wfi]: + for input_dataset in [ + x.dataset + for (x, creating_job) in creating_jobs + if creating_job.workflow_invocation_step + and creating_job.workflow_invocation_step.workflow_invocation == wfi + ]: # note that the above input_dataset is a reference to a # job.input_dataset.dataset at this point safe_to_delete = True for job_to_check in [d_j.job for d_j in input_dataset.dependent_jobs]: if job_to_check != job and job_to_check.state not in [job.states.OK, job.states.DELETED]: - log.trace(f"Workflow Intermediates cleanup attempted, but non-terminal state '{job_to_check.state}' detected for job {job_to_check.id}") + log.trace( + f"Workflow Intermediates cleanup attempted, but non-terminal state '{job_to_check.state}' detected for job {job_to_check.id}" + ) safe_to_delete = False if safe_to_delete: # Support purging here too. @@ -402,27 +432,35 @@ class TagDatasetAction(DefaultJobAction): direction = "to" @classmethod - def execute_on_mapped_over(cls, trans, sa_session, action, step_inputs, step_outputs, replacement_dict, final_job_state=None): + def execute_on_mapped_over( + cls, trans, sa_session, action, step_inputs, step_outputs, replacement_dict, final_job_state=None + ): tag_handler = trans.app.tag_handler.create_tag_handler_session() if action.action_arguments: - tags = [t.replace('#', 'name:') if t.startswith('#') else t for t in [t.strip() for t in action.action_arguments.get('tags', '').split(',') if t.strip()]] + tags = [ + t.replace("#", "name:") if t.startswith("#") else t + for t in [t.strip() for t in action.action_arguments.get("tags", "").split(",") if t.strip()] + ] if tags: for name, step_output in step_outputs.items(): - if action.output_name == '' or name == action.output_name: + if action.output_name == "" or name == action.output_name: cls._execute(tag_handler, trans.user, step_output, tags) @classmethod def execute(cls, app, sa_session, action, job, replacement_dict, final_job_state=None): if action.action_arguments: tag_handler = app.tag_handler.create_tag_handler_session() - tags = [t.replace('#', 'name:') if t.startswith('#') else t for t in [t.strip() for t in action.action_arguments.get('tags', '').split(',') if t.strip()]] + tags = [ + t.replace("#", "name:") if t.startswith("#") else t + for t in [t.strip() for t in action.action_arguments.get("tags", "").split(",") if t.strip()] + ] if tags: for dataset_assoc in job.output_datasets: - if action.output_name == '' or dataset_assoc.name == action.output_name: + if action.output_name == "" or dataset_assoc.name == action.output_name: cls._execute(tag_handler, job.user, dataset_assoc.dataset, tags) for dataset_collection_assoc in job.output_dataset_collection_instances: - if action.output_name == '' or dataset_collection_assoc.name == action.output_name: + if action.output_name == "" or dataset_collection_assoc.name == action.output_name: cls._execute(tag_handler, job.user, dataset_collection_assoc.dataset_collection_instance, tags) @classmethod @@ -431,11 +469,10 @@ class TagDatasetAction(DefaultJobAction): @classmethod def get_short_str(cls, pja): - if pja.action_arguments and pja.action_arguments.get('tags', ''): - return "{} tag(s) '{}' {} '{}'.".format(cls.action, - escape(pja.action_arguments['tags']), - cls.direction, - escape(pja.output_name)) + if pja.action_arguments and pja.action_arguments.get("tags", ""): + return "{} tag(s) '{}' {} '{}'.".format( + cls.action, escape(pja.action_arguments["tags"]), cls.direction, escape(pja.output_name) + ) else: return f"{cls.action} Tag action used without a tag specified. No tag will be added." @@ -453,26 +490,36 @@ class RemoveTagDatasetAction(TagDatasetAction): class ActionBox: - actions = {"RenameDatasetAction": RenameDatasetAction, - "HideDatasetAction": HideDatasetAction, - "ChangeDatatypeAction": ChangeDatatypeAction, - "ColumnSetAction": ColumnSetAction, - "EmailAction": EmailAction, - "DeleteIntermediatesAction": DeleteIntermediatesAction, - "TagDatasetAction": TagDatasetAction, - "RemoveTagDatasetAction": RemoveTagDatasetAction} - public_actions = ['RenameDatasetAction', 'ChangeDatatypeAction', - 'ColumnSetAction', 'EmailAction', - 'DeleteIntermediatesAction', 'TagDatasetAction', - 'RemoveTagDatasetAction'] + actions = { + "RenameDatasetAction": RenameDatasetAction, + "HideDatasetAction": HideDatasetAction, + "ChangeDatatypeAction": ChangeDatatypeAction, + "ColumnSetAction": ColumnSetAction, + "EmailAction": EmailAction, + "DeleteIntermediatesAction": DeleteIntermediatesAction, + "TagDatasetAction": TagDatasetAction, + "RemoveTagDatasetAction": RemoveTagDatasetAction, + } + public_actions = [ + "RenameDatasetAction", + "ChangeDatatypeAction", + "ColumnSetAction", + "EmailAction", + "DeleteIntermediatesAction", + "TagDatasetAction", + "RemoveTagDatasetAction", + ] # Actions that can be applied ahead of the job execution while workflow is still # being scheduled and jobs created. - immediate_actions = ['ChangeDatatypeAction', 'RenameDatasetAction', - 'TagDatasetAction', 'RemoveTagDatasetAction'] + immediate_actions = ["ChangeDatatypeAction", "RenameDatasetAction", "TagDatasetAction", "RemoveTagDatasetAction"] # Actions that will be applied to implicit mapped over collection outputs and not # just individual outputs when steps include mapped over tools and implicit collection outputs. - mapped_over_output_actions = ['RenameDatasetAction', 'HideDatasetAction', - 'TagDatasetAction', 'RemoveTagDatasetAction'] + mapped_over_output_actions = [ + "RenameDatasetAction", + "HideDatasetAction", + "TagDatasetAction", + "RemoveTagDatasetAction", + ] @classmethod def get_short_str(cls, action): @@ -485,30 +532,34 @@ class ActionBox: def handle_incoming(cls, incoming): npd = {} for key, val in incoming.items(): - if key.startswith('pja'): - sp = key.split('__') + if key.startswith("pja"): + sp = key.split("__") ao_key = sp[2] + sp[1] # flag / output_name / pjatype / desc if ao_key not in npd: - npd[ao_key] = {'action_type': sp[2], - 'output_name': sp[1], - 'action_arguments': {}} + npd[ao_key] = {"action_type": sp[2], "output_name": sp[1], "action_arguments": {}} if len(sp) > 3: - if sp[3] == 'output_name': - npd[ao_key]['output_name'] = val + if sp[3] == "output_name": + npd[ao_key]["output_name"] = val else: - npd[ao_key]['action_arguments'][sp[3]] = val + npd[ao_key]["action_arguments"][sp[3]] = val else: # Not pja stuff. pass return npd @classmethod - def execute_on_mapped_over(cls, trans, sa_session, pja, step_inputs, step_outputs, replacement_dict=None, final_job_state=None): + def execute_on_mapped_over( + cls, trans, sa_session, pja, step_inputs, step_outputs, replacement_dict=None, final_job_state=None + ): if pja.action_type in ActionBox.actions: - ActionBox.actions[pja.action_type].execute_on_mapped_over(trans, sa_session, pja, step_inputs, step_outputs, replacement_dict, final_job_state=final_job_state) + ActionBox.actions[pja.action_type].execute_on_mapped_over( + trans, sa_session, pja, step_inputs, step_outputs, replacement_dict, final_job_state=final_job_state + ) @classmethod def execute(cls, app, sa_session, pja, job, replacement_dict=None, final_job_state=None): if pja.action_type in ActionBox.actions: - ActionBox.actions[pja.action_type].execute(app, sa_session, pja, job, replacement_dict, final_job_state=final_job_state) + ActionBox.actions[pja.action_type].execute( + app, sa_session, pja, job, replacement_dict, final_job_state=final_job_state + ) diff --git a/lib/galaxy/job_execution/compute_environment.py b/lib/galaxy/job_execution/compute_environment.py index 3de4e16aeee..66966e06797 100644 --- a/lib/galaxy/job_execution/compute_environment.py +++ b/lib/galaxy/job_execution/compute_environment.py @@ -9,13 +9,13 @@ from galaxy.model import Job class ComputeEnvironment(metaclass=ABCMeta): - """ Definition of the job as it will be run on the (potentially) remote + """Definition of the job as it will be run on the (potentially) remote compute server. """ @abstractmethod def output_names(self): - """ Output unqualified filenames defined by job. """ + """Output unqualified filenames defined by job.""" @abstractmethod def input_path_rewrite(self, dataset): @@ -43,11 +43,11 @@ class ComputeEnvironment(metaclass=ABCMeta): @abstractmethod def working_directory(self): - """ Job working directory (potentially remote) """ + """Job working directory (potentially remote)""" @abstractmethod def config_directory(self): - """ Directory containing config files (potentially remote) """ + """Directory containing config files (potentially remote)""" @abstractmethod def env_config_directory(self): @@ -55,20 +55,19 @@ class ComputeEnvironment(metaclass=ABCMeta): @abstractmethod def sep(self): - """ os.path.sep for the platform this job will execute in. - """ + """os.path.sep for the platform this job will execute in.""" @abstractmethod def new_file_path(self): - """ Absolute path to dump new files for this job on compute server. """ + """Absolute path to dump new files for this job on compute server.""" @abstractmethod def tool_directory(self): - """ Absolute path to tool files for this job on compute server. """ + """Absolute path to tool files for this job on compute server.""" @abstractmethod def version_path(self): - """ Location of the version file for the underlying tool. """ + """Location of the version file for the underlying tool.""" @abstractmethod def home_directory(self): @@ -88,7 +87,6 @@ class ComputeEnvironment(metaclass=ABCMeta): class SimpleComputeEnvironment: - def config_directory(self): return os.path.join(self.working_directory(), "configs") # type: ignore[attr-defined] @@ -97,7 +95,7 @@ class SimpleComputeEnvironment: class SharedComputeEnvironment(SimpleComputeEnvironment, ComputeEnvironment): - """ Default ComputeEnvironment for job and task wrapper to pass + """Default ComputeEnvironment for job and task wrapper to pass to ToolEvaluator - valid when Galaxy and compute share all the relevant file systems. """ diff --git a/lib/galaxy/job_execution/datasets.py b/lib/galaxy/job_execution/datasets.py index 9a872b59eba..d06e68fee71 100644 --- a/lib/galaxy/job_execution/datasets.py +++ b/lib/galaxy/job_execution/datasets.py @@ -4,7 +4,7 @@ Utility classes allowing Job interface to reason about datasets. import os.path from abc import ( ABCMeta, - abstractmethod + abstractmethod, ) @@ -14,7 +14,6 @@ def dataset_path_rewrites(dataset_paths): class DatasetPath: - def __init__( self, dataset_id, @@ -59,7 +58,7 @@ class DatasetPath: class DatasetPathRewriter(metaclass=ABCMeta): - """ Used by runner to rewrite paths. """ + """Used by runner to rewrite paths.""" @abstractmethod def rewrite_dataset_path(self, dataset, dataset_type): @@ -70,17 +69,15 @@ class DatasetPathRewriter(metaclass=ABCMeta): class NullDatasetPathRewriter: - """ Used by default for jobwrapper, do not rewrite anything. - """ + """Used by default for jobwrapper, do not rewrite anything.""" def rewrite_dataset_path(self, dataset, dataset_type): - """ Keep path the same. - """ + """Keep path the same.""" return None class OutputsToWorkingDirectoryPathRewriter: - """ Rewrites all paths to place them in the specified working + """Rewrites all paths to place them in the specified working directory for normal jobs when Galaxy is configured with app.config.outputs_to_working_directory. Job runner base class is responsible for copying these out after job is complete. @@ -91,9 +88,8 @@ class OutputsToWorkingDirectoryPathRewriter: self.outputs_directory_name = outputs_directory_name def rewrite_dataset_path(self, dataset, dataset_type): - """ Keep path the same. - """ - if dataset_type == 'output': + """Keep path the same.""" + if dataset_type == "output": base_output_directory = os.path.abspath(self.working_directory) if self.outputs_directory_name is not None: base_output_directory = os.path.join(base_output_directory, self.outputs_directory_name) @@ -105,7 +101,7 @@ class OutputsToWorkingDirectoryPathRewriter: class TaskPathRewriter: - """ Rewrites all paths to place them in the specified working + """Rewrites all paths to place them in the specified working directory for TaskWrapper. TaskWrapper is responsible for putting them there and pulling them out. """ @@ -115,15 +111,18 @@ class TaskPathRewriter: self.job_dataset_path_rewriter = job_dataset_path_rewriter def rewrite_dataset_path(self, dataset, dataset_type): - """ - """ + """ """ dataset_file_name = dataset.file_name job_file_name = self.job_dataset_path_rewriter.rewrite_dataset_path(dataset, dataset_type) or dataset_file_name return os.path.join(self.working_directory, os.path.basename(job_file_name)) def get_path_rewriter(outputs_to_working_directory, working_directory, outputs_directory, is_task): - job_dataset_path_rewriter = OutputsToWorkingDirectoryPathRewriter(working_directory, outputs_directory) if outputs_to_working_directory else NullDatasetPathRewriter() + job_dataset_path_rewriter = ( + OutputsToWorkingDirectoryPathRewriter(working_directory, outputs_directory) + if outputs_to_working_directory + else NullDatasetPathRewriter() + ) if is_task: return TaskPathRewriter(working_directory, job_dataset_path_rewriter=job_dataset_path_rewriter) return job_dataset_path_rewriter diff --git a/lib/galaxy/job_execution/output_collect.py b/lib/galaxy/job_execution/output_collect.py index 05dbc41b29a..d5fe7ac089a 100644 --- a/lib/galaxy/job_execution/output_collect.py +++ b/lib/galaxy/job_execution/output_collect.py @@ -5,7 +5,13 @@ import operator import os import re from tempfile import NamedTemporaryFile -from typing import Callable, Dict, List, Optional, Union +from typing import ( + Callable, + Dict, + List, + Optional, + Union, +) from sqlalchemy.orm.scoping import ScopedSession @@ -23,9 +29,11 @@ from galaxy.model.store.discover import ( discover_target_directory, DiscoveredFile, JsonCollectedDatasetMatch, - MetadataSourceProvider as AbstractMetadataSourceProvider, - ModelPersistenceContext, - PermissionProvider as AbstractPermissionProvider, +) +from galaxy.model.store.discover import MetadataSourceProvider as AbstractMetadataSourceProvider +from galaxy.model.store.discover import ModelPersistenceContext +from galaxy.model.store.discover import PermissionProvider as AbstractPermissionProvider +from galaxy.model.store.discover import ( persist_elements_to_folder, persist_elements_to_hdca, persist_hdas, @@ -56,7 +64,6 @@ log = logging.getLogger(__name__) # PermissionProvider and MetadataSourceProvider are abstractions over input data used to # collect and produce dynamic outputs. class PermissionProvider(AbstractPermissionProvider): - def __init__(self, inp_data, security_agent, job): self._job = job self._security_agent = security_agent @@ -88,7 +95,6 @@ class PermissionProvider(AbstractPermissionProvider): class MetadataSourceProvider(AbstractMetadataSourceProvider): - def __init__(self, inp_data): self._inp_data = inp_data @@ -162,7 +168,9 @@ def collect_dynamic_outputs( try: collection_builder = builder.BoundCollectionBuilder(collection) - dataset_collectors = [dataset_collector(description) for description in output_collection_def.dataset_collector_descriptions] + dataset_collectors = [ + dataset_collector(description) for description in output_collection_def.dataset_collector_descriptions + ] output_name = output_collection_def.name filenames = job_context.find_files(output_name, collection, dataset_collectors) job_context.populate_collection_elements( @@ -190,7 +198,9 @@ class BaseJobContext: def find_files(self, output_name, collection, dataset_collectors): filenames = {} - for discovered_file in discover_files(output_name, self.tool_provided_metadata, dataset_collectors, self.job_working_directory, collection): + for discovered_file in discover_files( + output_name, self.tool_provided_metadata, dataset_collectors, self.job_working_directory, collection + ): self.increment_discovered_file_count() filenames[discovered_file.path] = discovered_file return filenames @@ -200,20 +210,19 @@ class BaseJobContext: class JobContext(ModelPersistenceContext, BaseJobContext): - def __init__( - self, - tool, - tool_provided_metadata, - job, - job_working_directory, - permission_provider, - metadata_source_provider, - input_dbkey, - object_store, - final_job_state, - max_discovered_files: Optional[int], - flush_per_n_datasets=None, + self, + tool, + tool_provided_metadata, + job, + job_working_directory, + permission_provider, + metadata_source_provider, + input_dbkey, + object_store, + final_job_state, + max_discovered_files: Optional[int], + flush_per_n_datasets=None, ): self.tool = tool self._metadata_source_provider = metadata_source_provider @@ -227,7 +236,7 @@ class JobContext(ModelPersistenceContext, BaseJobContext): self._object_store = object_store self.final_job_state = final_job_state self._flush_per_n_datasets = flush_per_n_datasets - self.max_discovered_files = float('inf') if max_discovered_files is None else max_discovered_files + self.max_discovered_files = float("inf") if max_discovered_files is None else max_discovered_files self.discovered_file_count = 0 self._tag_handler = None @@ -240,6 +249,7 @@ class JobContext(ModelPersistenceContext, BaseJobContext): @property def work_context(self): from galaxy.work.context import WorkRequestContext + return WorkRequestContext(self.app, user=self.user) @property @@ -287,7 +297,9 @@ class JobContext(ModelPersistenceContext, BaseJobContext): def get_library_folder(self, destination): app = self.app library_folder_manager = app.library_folder_manager - library_folder = library_folder_manager.get(self.work_context, app.security.decode_id(destination.get("library_folder_id"))) + library_folder = library_folder_manager.get( + self.work_context, app.security.decode_id(destination.get("library_folder_id")) + ) return library_folder def get_hdca(self, object_id): @@ -304,9 +316,7 @@ class JobContext(ModelPersistenceContext, BaseJobContext): history = self.job.history trans = self.work_context collection_manager = self.app.dataset_collection_manager - hdca = collection_manager.precreate_dataset_collection_instance( - trans, history, name, structure=structure - ) + hdca = collection_manager.precreate_dataset_collection_instance(trans, history, name, structure=structure) return hdca def add_output_dataset_association(self, name, dataset): @@ -327,7 +337,9 @@ class JobContext(ModelPersistenceContext, BaseJobContext): # Permissions must be the same on the LibraryDatasetDatasetAssociation and the associated LibraryDataset trans.app.security_agent.copy_library_permissions(trans, ld, ldda) # Copy the current user's DefaultUserPermissions to the new LibraryDatasetDatasetAssociation.dataset - trans.app.security_agent.set_all_dataset_permissions(ldda.dataset, trans.app.security_agent.user_get_default_permissions(trans.user)) + trans.app.security_agent.set_all_dataset_permissions( + ldda.dataset, trans.app.security_agent.user_get_default_permissions(trans.user) + ) library_folder.add_library_dataset(ld, genome_build=ldda.dbkey) trans.sa_session.add(library_folder) trans.sa_session.flush() @@ -378,15 +390,24 @@ class JobContext(ModelPersistenceContext, BaseJobContext): class SessionlessJobContext(SessionlessModelPersistenceContext, BaseJobContext): - - def __init__(self, metadata_params, tool_provided_metadata, object_store, export_store, import_store, working_directory, final_job_state, max_discovered_files: Optional[int]): + def __init__( + self, + metadata_params, + tool_provided_metadata, + object_store, + export_store, + import_store, + working_directory, + final_job_state, + max_discovered_files: Optional[int], + ): # TODO: use a metadata source provider... (pop from inputs and add parameter) super().__init__(object_store, export_store, working_directory) self.metadata_params = metadata_params self.tool_provided_metadata = tool_provided_metadata self.import_store = import_store self.final_job_state = final_job_state - self.max_discovered_files = float('inf') if max_discovered_files is None else max_discovered_files + self.max_discovered_files = float("inf") if max_discovered_files is None else max_discovered_files self.discovered_file_count = 0 def output_collection_def(self, name): @@ -453,9 +474,13 @@ def collect_primary_datasets(job_context: Union[JobContext, SessionlessJobContex dataset_collectors = [DEFAULT_DATASET_COLLECTOR] output_def = job_context.output_def(name) if output_def is not None: - dataset_collectors = [dataset_collector(description) for description in output_def.dataset_collector_descriptions] + dataset_collectors = [ + dataset_collector(description) for description in output_def.dataset_collector_descriptions + ] filenames = {} - for discovered_file in discover_files(name, job_context.tool_provided_metadata, dataset_collectors, job_working_directory, outdata): + for discovered_file in discover_files( + name, job_context.tool_provided_metadata, dataset_collectors, job_working_directory, outdata + ): job_context.increment_discovered_file_count() filenames[discovered_file.path] = discovered_file for filename_index, (filename, discovered_file) in enumerate(filenames.items()): @@ -490,10 +515,12 @@ def collect_primary_datasets(job_context: Union[JobContext, SessionlessJobContex # TODO: should be able to disambiguate files in different directories... new_primary_filename = os.path.split(filename)[-1] - new_primary_datasets_attributes = job_context.tool_provided_metadata.get_new_dataset_meta_by_basename(name, new_primary_filename) + new_primary_datasets_attributes = job_context.tool_provided_metadata.get_new_dataset_meta_by_basename( + name, new_primary_filename + ) extra_files = None if new_primary_datasets_attributes: - extra_files_path = new_primary_datasets_attributes.get('extra_files', None) + extra_files_path = new_primary_datasets_attributes.get("extra_files", None) if extra_files_path: extra_files = os.path.join(job_working_directory, extra_files_path) primary_data = job_context.create_dataset( @@ -508,10 +535,10 @@ def collect_primary_datasets(job_context: Union[JobContext, SessionlessJobContex init_from=outdata, dataset_attributes=new_primary_datasets_attributes, creating_job_id=job_context.get_job_id() if job_context else None, - storage_callbacks=storage_callbacks + storage_callbacks=storage_callbacks, ) # Associate new dataset with job - job_context.add_output_dataset_association(f'__new_primary_file_{name}|{designation}__', primary_data) + job_context.add_output_dataset_association(f"__new_primary_file_{name}|{designation}__", primary_data) job_context.add_datasets_to_history([primary_data], for_output_dataset=outdata) # Add dataset to return dict primary_datasets[name][designation] = primary_data @@ -540,7 +567,11 @@ def discover_files(output_name, tool_provided_metadata, extra_file_collectors, j for dataset in tool_provided_metadata.get_new_datasets(output_name): filename = dataset["filename"] path = os.path.join(target_directory, filename) - yield DiscoveredFile(path, extra_file_collector, JsonCollectedDatasetMatch(dataset, extra_file_collector, filename, path=path)) + yield DiscoveredFile( + path, + extra_file_collector, + JsonCollectedDatasetMatch(dataset, extra_file_collector, filename, path=path), + ) else: for (match, collector) in walk_over_file_collectors(extra_file_collectors, job_working_directory, matchable): yield DiscoveredFile(match.path, collector, match) @@ -549,7 +580,9 @@ def discover_files(output_name, tool_provided_metadata, extra_file_collectors, j def walk_over_file_collectors(extra_file_collectors, job_working_directory, matchable): for extra_file_collector in extra_file_collectors: assert extra_file_collector.discover_via == "pattern" - for match in walk_over_extra_files(extra_file_collector.directory, extra_file_collector, job_working_directory, matchable): + for match in walk_over_extra_files( + extra_file_collector.directory, extra_file_collector, job_working_directory, matchable + ): yield match, extra_file_collector @@ -570,7 +603,9 @@ def walk_over_extra_files(target_dir, extra_file_collector, job_working_director new_parent_paths = parent_paths[:] new_parent_paths.append(filename) # The current directory is already validated, so use that as the next job_working_directory when recursing - for match in walk_over_extra_files(filename, extra_file_collector, directory, matchable, parent_paths=new_parent_paths): + for match in walk_over_extra_files( + filename, extra_file_collector, directory, matchable, parent_paths=new_parent_paths + ): yield match else: match = extra_file_collector.match(matchable, filename, path=path, parent_paths=parent_paths) @@ -594,7 +629,6 @@ def dataset_collector(dataset_collection_description): class ToolMetadataDatasetCollector: - def __init__(self, dataset_collection_description): self.discover_via = dataset_collection_description.discover_via self.default_dbkey = dataset_collection_description.default_dbkey @@ -605,7 +639,6 @@ class ToolMetadataDatasetCollector: class DatasetCollector: - def __init__(self, dataset_collection_description): self.discover_via = dataset_collection_description.discover_via # dataset_collection_description is an abstract description @@ -623,7 +656,7 @@ class DatasetCollector: self.match_relative_path = dataset_collection_description.match_relative_path def _pattern_for_dataset(self, dataset_instance=None): - token_replacement = r'\d+' + token_replacement = r"\d+" if dataset_instance: token_replacement = str(dataset_instance.id) return self.pattern.replace(DATASET_ID_TOKEN, token_replacement) @@ -680,7 +713,7 @@ def read_exit_code_from(exit_code_file, id_tag): def default_exit_code_file(files_dir, id_tag): - return os.path.join(files_dir, f'galaxy_{id_tag}.ec') + return os.path.join(files_dir, f"galaxy_{id_tag}.ec") def collect_extra_files(object_store, dataset, job_working_directory): @@ -693,7 +726,7 @@ def collect_extra_files(object_store, dataset, job_working_directory): # not be created in the object store at all, which might be a # problem. for root, _dirs, files in os.walk(temp_file_path): - extra_dir = root.replace(os.path.join(job_working_directory, "working"), '', 1).lstrip(os.path.sep) + extra_dir = root.replace(os.path.join(job_working_directory, "working"), "", 1).lstrip(os.path.sep) for f in files: object_store.update_from_file( dataset.dataset, @@ -701,26 +734,28 @@ def collect_extra_files(object_store, dataset, job_working_directory): alt_name=f, file_name=os.path.join(root, f), create=True, - preserve_symlinks=True + preserve_symlinks=True, ) except Exception as e: log.debug("Error in collect_associated_files: %s", unicodify(e)) # Handle composite datatypes of auto_primary_file type - if dataset.datatype.composite_type == 'auto_primary_file' and not dataset.has_data(): + if dataset.datatype.composite_type == "auto_primary_file" and not dataset.has_data(): try: - with NamedTemporaryFile(mode='w') as temp_fh: + with NamedTemporaryFile(mode="w") as temp_fh: temp_fh.write(dataset.datatype.generate_primary_file(dataset)) temp_fh.flush() object_store.update_from_file(dataset.dataset, file_name=temp_fh.name, create=True) dataset.set_size() except Exception as e: - log.warning('Unable to generate primary composite file automatically for %s: %s', dataset.dataset.id, unicodify(e)) + log.warning( + "Unable to generate primary composite file automatically for %s: %s", dataset.dataset.id, unicodify(e) + ) def collect_shrinked_content_from_path(path): try: - with open(path, 'rb') as fh: + with open(path, "rb") as fh: return shrink_and_unicodify(fh.read().strip()) except FileNotFoundError: return None diff --git a/lib/galaxy/job_execution/ports/__init__.py b/lib/galaxy/job_execution/ports/__init__.py index a8838df5aa0..db717ad245e 100644 --- a/lib/galaxy/job_execution/ports/__init__.py +++ b/lib/galaxy/job_execution/ports/__init__.py @@ -1,3 +1,3 @@ from .view import JobPortsView -__all__ = ('JobPortsView', ) +__all__ = ("JobPortsView",) diff --git a/lib/galaxy/job_execution/ports/view.py b/lib/galaxy/job_execution/ports/view.py index 0b4c94c5c7e..807c2d5d4b7 100644 --- a/lib/galaxy/job_execution/ports/view.py +++ b/lib/galaxy/job_execution/ports/view.py @@ -2,15 +2,17 @@ import logging from galaxy import ( model, - util + util, +) +from galaxy.exceptions import ( + ItemAccessibilityException, + ObjectAttributeMissingException, ) -from galaxy.exceptions import ItemAccessibilityException, ObjectAttributeMissingException log = logging.getLogger(__name__) class JobPortsView: - def __init__(self, app): self._app = app diff --git a/lib/galaxy/job_execution/setup.py b/lib/galaxy/job_execution/setup.py index e4f827bbc43..709580bc578 100644 --- a/lib/galaxy/job_execution/setup.py +++ b/lib/galaxy/job_execution/setup.py @@ -1,7 +1,15 @@ """Utilities to help job and tool code setup jobs.""" import json import os -from typing import Any, cast, Dict, List, Optional, Tuple, Union +from typing import ( + Any, + cast, + Dict, + List, + Optional, + Tuple, + Union, +) from galaxy.files import ( ConfiguredFileSources, @@ -21,8 +29,8 @@ from galaxy.model import ( from galaxy.util import safe_makedirs from galaxy.util.dictifiable import Dictifiable -TOOL_PROVIDED_JOB_METADATA_FILE = 'galaxy.json' -TOOL_PROVIDED_JOB_METADATA_KEYS = ['name', 'info', 'dbkey', 'created_from_basename'] +TOOL_PROVIDED_JOB_METADATA_FILE = "galaxy.json" +TOOL_PROVIDED_JOB_METADATA_KEYS = ["name", "info", "dbkey", "created_from_basename"] OutputHdasAndType = Dict[str, Tuple[DatasetInstance, DatasetPath]] @@ -31,56 +39,57 @@ OutputPaths = List[DatasetPath] class JobIO(Dictifiable): dict_collection_visible_keys = ( - 'job_id', - 'working_directory', - 'outputs_directory', - 'outputs_to_working_directory', - 'galaxy_url', - 'version_path', - 'tool_directory', - 'home_directory', - 'tmp_directory', - 'tool_data_path', - 'galaxy_data_manager_data_path', - 'new_file_path', - 'len_file_path', - 'builds_file_path', - 'file_sources_dict', - 'check_job_script_integrity', - 'check_job_script_integrity_count', - 'check_job_script_integrity_sleep', - 'tool_source', - 'tool_source_class', - 'tool_dir', - 'is_task', + "job_id", + "working_directory", + "outputs_directory", + "outputs_to_working_directory", + "galaxy_url", + "version_path", + "tool_directory", + "home_directory", + "tmp_directory", + "tool_data_path", + "galaxy_data_manager_data_path", + "new_file_path", + "len_file_path", + "builds_file_path", + "file_sources_dict", + "check_job_script_integrity", + "check_job_script_integrity_count", + "check_job_script_integrity_sleep", + "tool_source", + "tool_source_class", + "tool_dir", + "is_task", ) def __init__( - self, - sa_session, - job: Job, - working_directory: str, - outputs_directory: str, - outputs_to_working_directory: bool, - galaxy_url: str, - version_path: str, - tool_directory: str, - home_directory: str, - tmp_directory: str, - tool_data_path: str, - galaxy_data_manager_data_path: str, - new_file_path: str, - len_file_path: str, - builds_file_path: str, - check_job_script_integrity: bool, - check_job_script_integrity_count: int, - check_job_script_integrity_sleep: float, - file_sources_dict: Dict[str, Any], - user_context: Union[ProvidesUserFileSourcesUserContext, Dict['str', Any]], - tool_source: Optional[str] = None, - tool_source_class: Optional['str'] = 'XmlToolSource', - tool_dir: Optional[str] = None, - is_task: bool = False): + self, + sa_session, + job: Job, + working_directory: str, + outputs_directory: str, + outputs_to_working_directory: bool, + galaxy_url: str, + version_path: str, + tool_directory: str, + home_directory: str, + tmp_directory: str, + tool_data_path: str, + galaxy_data_manager_data_path: str, + new_file_path: str, + len_file_path: str, + builds_file_path: str, + check_job_script_integrity: bool, + check_job_script_integrity_count: int, + check_job_script_integrity_sleep: float, + file_sources_dict: Dict[str, Any], + user_context: Union[ProvidesUserFileSourcesUserContext, Dict["str", Any]], + tool_source: Optional[str] = None, + tool_source_class: Optional["str"] = "XmlToolSource", + tool_dir: Optional[str] = None, + is_task: bool = False, + ): user_context_instance: Union[ProvidesUserFileSourcesUserContext, DictFileSourcesUserContext] if isinstance(user_context, dict): user_context_instance = DictFileSourcesUserContext(**user_context) @@ -123,18 +132,18 @@ class JobIO(Dictifiable): @classmethod def from_dict(cls, io_dict, sa_session): - io_dict.pop('model_class') - job_id = io_dict.pop('job_id') + io_dict.pop("model_class") + job_id = io_dict.pop("job_id") job = sa_session.query(Job).get(job_id) return cls(sa_session=sa_session, job=job, **io_dict) def to_dict(self): io_dict = super().to_dict() - io_dict['user_context'] = self.user_context.to_dict() + io_dict["user_context"] = self.user_context.to_dict() return io_dict def to_json(self, path): - with open(path, 'w') as out: + with open(path, "w") as out: out.write(json.dumps(self.to_dict())) @property @@ -191,7 +200,7 @@ class JobIO(Dictifiable): def get_input_path(self, dataset: DatasetInstance): real_path = dataset.file_name - false_path = self.dataset_path_rewriter.rewrite_dataset_path(dataset, 'input') + false_path = self.dataset_path_rewriter.rewrite_dataset_path(dataset, "input") return DatasetPath( dataset.dataset.id, real_path=real_path, @@ -232,15 +241,17 @@ class JobIO(Dictifiable): results = [] for da in job.output_datasets + job.output_library_datasets: - da_false_path = dataset_path_rewriter.rewrite_dataset_path(da.dataset, 'output') + da_false_path = dataset_path_rewriter.rewrite_dataset_path(da.dataset, "output") mutable = da.dataset.dataset.external_filename is None - dataset_path = DatasetPath(da.dataset.dataset.id, da.dataset.file_name, false_path=da_false_path, mutable=mutable) + dataset_path = DatasetPath( + da.dataset.dataset.id, da.dataset.file_name, false_path=da_false_path, mutable=mutable + ) results.append((da.name, da.dataset, dataset_path)) self._output_paths = [t[2] for t in results] self._output_hdas_and_paths = {t[0]: t[1:] for t in results} if special: - false_path = dataset_path_rewriter.rewrite_dataset_path(special, 'output') + false_path = dataset_path_rewriter.rewrite_dataset_path(special, "output") dsp = DatasetPath(special.dataset.id, special.dataset.file_name, false_path) self._output_paths.append(dsp) self._output_hdas_and_paths["output_file"] = (special.fda, dsp) @@ -262,8 +273,6 @@ def ensure_configs_directory(work_dir): def create_working_directory_for_job(object_store, job): - object_store.create( - job, base_dir='job_work', dir_only=True, obj_dir=True) - working_directory = object_store.get_filename( - job, base_dir='job_work', dir_only=True, obj_dir=True) + object_store.create(job, base_dir="job_work", dir_only=True, obj_dir=True) + working_directory = object_store.get_filename(job, base_dir="job_work", dir_only=True, obj_dir=True) return working_directory diff --git a/lib/galaxy/job_metrics/__init__.py b/lib/galaxy/job_metrics/__init__.py index bbc79ac4840..50a79af9745 100644 --- a/lib/galaxy/job_metrics/__init__.py +++ b/lib/galaxy/job_metrics/__init__.py @@ -47,12 +47,12 @@ class JobMetrics: self.set_destination_instrumenter(destination_id, instrumenter) def set_destination_conf_element(self, destination_id, element): - plugin_source = plugin_config.PluginConfigSource('xml', element) + plugin_source = plugin_config.PluginConfigSource("xml", element) instrumenter = JobInstrumenter(self.plugin_classes, plugin_source) self.set_destination_instrumenter(destination_id, instrumenter) def set_destination_conf_dicts(self, destination_id, conf_dicts): - plugin_source = plugin_config.PluginConfigSource('dict', conf_dicts) + plugin_source = plugin_config.PluginConfigSource("dict", conf_dicts) instrumenter = JobInstrumenter(self.plugin_classes, plugin_source) self.set_destination_instrumenter(destination_id, instrumenter) @@ -66,11 +66,11 @@ class JobMetrics: def __plugins_dict(self): import galaxy.job_metrics.instrumenters - return plugin_config.plugins_dict(galaxy.job_metrics.instrumenters, 'plugin_type') + + return plugin_config.plugins_dict(galaxy.job_metrics.instrumenters, "plugin_type") class NullJobInstrumenter: - def pre_execute_commands(self, job_directory): return None @@ -85,7 +85,6 @@ NULL_JOB_INSTRUMENTER = NullJobInstrumenter() class JobInstrumenter: - def __init__(self, plugin_classes, plugins_source, **kwargs): self.extra_kwargs = kwargs self.plugin_classes = plugin_classes diff --git a/lib/galaxy/job_metrics/collectl/cli.py b/lib/galaxy/job_metrics/collectl/cli.py index bea4e4a7ba6..1c370fe1858 100644 --- a/lib/galaxy/job_metrics/collectl/cli.py +++ b/lib/galaxy/job_metrics/collectl/cli.py @@ -138,4 +138,4 @@ class CollectlCli: raise Exception("Problem running collectl command.") -__all__ = ('CollectlCli', ) +__all__ = ("CollectlCli",) diff --git a/lib/galaxy/job_metrics/collectl/processes.py b/lib/galaxy/job_metrics/collectl/processes.py index c06037c0368..a665e76b82a 100644 --- a/lib/galaxy/job_metrics/collectl/processes.py +++ b/lib/galaxy/job_metrics/collectl/processes.py @@ -84,7 +84,7 @@ DEFAULT_STATISTICS = [ def parse_process_statistics(statistics): - """ Turn string or list of strings into list of tuples in format ( stat, + """Turn string or list of strings into list of tuples in format ( stat, resource ) where stat is a value from STATISTIC_TYPES and resource is a value from PROCESS_COLUMNS. """ @@ -103,8 +103,7 @@ def parse_process_statistics(statistics): def generate_process_statistics(collectl_playback_cli, pid, statistics=DEFAULT_STATISTICS): - """ Playback collectl file and generate summary statistics. - """ + """Playback collectl file and generate summary statistics.""" with tempfile.NamedTemporaryFile() as tmp_tsv: collectl_playback_cli.run(stdout=tmp_tsv) with open(tmp_tsv.name) as tsv_file: @@ -139,7 +138,6 @@ def _read_process_statistics(tsv_file, pid, statistics): class CollectlProcessSummarizer: - def __init__(self, pid, statistics): self.pid = pid self.statistics = statistics @@ -223,7 +221,7 @@ class CollectlProcessSummarizer: class CollectlProcessInterval: - """ Represent all rows in collectl playback file for given time slice with + """Represent all rows in collectl playback file for given time slice with ability to filter out just rows corresponding to the process tree corresponding to a given pid. """ @@ -248,4 +246,4 @@ def _tuplize_statistic(statistic): return statistic -__all__ = ('generate_process_statistics', ) +__all__ = ("generate_process_statistics",) diff --git a/lib/galaxy/job_metrics/collectl/stats.py b/lib/galaxy/job_metrics/collectl/stats.py index ff974bacc76..a5685dd4288 100644 --- a/lib/galaxy/job_metrics/collectl/stats.py +++ b/lib/galaxy/job_metrics/collectl/stats.py @@ -4,7 +4,6 @@ memory. class StatisticsTracker: - def __init__(self): self.min = None self.max = None diff --git a/lib/galaxy/job_metrics/collectl/subsystems.py b/lib/galaxy/job_metrics/collectl/subsystems.py index bac7e50eb19..d4816593c20 100644 --- a/lib/galaxy/job_metrics/collectl/subsystems.py +++ b/lib/galaxy/job_metrics/collectl/subsystems.py @@ -4,25 +4,22 @@ Subsystems are essentially monitoring plugins available within collectl. """ from abc import ( ABCMeta, - abstractmethod + abstractmethod, ) class CollectlSubsystem(metaclass=ABCMeta): - """ Class providing an abstraction of collectl subsytems. - """ + """Class providing an abstraction of collectl subsytems.""" @property @abstractmethod def command_line_arg(self): - """ Return single letter command-line argument used by collectl CLI. - """ + """Return single letter command-line argument used by collectl CLI.""" @property @abstractmethod def name(self): - """ High-level name for subsystem as consumed by this module. - """ + """High-level name for subsystem as consumed by this module.""" class ProcessesSubsystem(CollectlSubsystem): @@ -75,4 +72,4 @@ def get_subsystem(name): return SUBSYSTEM_DICT[name] -__all__ = ('get_subsystem', ) +__all__ = ("get_subsystem",) diff --git a/lib/galaxy/job_metrics/instrumenters/__init__.py b/lib/galaxy/job_metrics/instrumenters/__init__.py index 27a21a7114c..cc15408da54 100644 --- a/lib/galaxy/job_metrics/instrumenters/__init__.py +++ b/lib/galaxy/job_metrics/instrumenters/__init__.py @@ -5,10 +5,9 @@ These are responsible for collecting and formatting a coherent set of metrics. import os.path from abc import ( ABCMeta, - abstractmethod + abstractmethod, ) - from .. import formatting INSTRUMENT_FILE_PREFIX = "__instrument" @@ -16,22 +15,23 @@ INSTRUMENT_FILE_PREFIX = "__instrument" class InstrumentPlugin(metaclass=ABCMeta): """Describes how to instrument job scripts and retrieve collected metrics.""" + formatter = formatting.JobMetricFormatter() @property @abstractmethod def plugin_type(self): - """ Short string providing labelling this plugin """ + """Short string providing labelling this plugin""" def pre_execute_instrument(self, job_directory): - """ Optionally return one or more commands to instrument job. These + """Optionally return one or more commands to instrument job. These commands will be executed on the compute server prior to the job running. """ return None def post_execute_instrument(self, job_directory): - """ Optionally return one or more commands to instrument job. These + """Optionally return one or more commands to instrument job. These commands will be executed on the compute server after the tool defined command is ran. """ @@ -39,14 +39,14 @@ class InstrumentPlugin(metaclass=ABCMeta): @abstractmethod def job_properties(self, job_id, job_directory): - """ Collect properties for this plugin from specified job directory. + """Collect properties for this plugin from specified job directory. This method will run on the Galaxy server and can assume files created in job_directory with pre_execute_instrument and post_execute_instrument are available. """ def _instrument_file_name(self, name): - """ Provide a common pattern for naming files used by instrumentation + """Provide a common pattern for naming files used by instrumentation plugins - to ease their staging out of remote job directories. """ return f"{INSTRUMENT_FILE_PREFIX}_{self.plugin_type}_{name}" diff --git a/lib/galaxy/job_metrics/instrumenters/cgroup.py b/lib/galaxy/job_metrics/instrumenters/cgroup.py index 5273465999e..48f07c1499a 100644 --- a/lib/galaxy/job_metrics/instrumenters/cgroup.py +++ b/lib/galaxy/job_metrics/instrumenters/cgroup.py @@ -3,7 +3,10 @@ import logging import numbers from collections import namedtuple -from galaxy.util import asbool, nice_size +from galaxy.util import ( + asbool, + nice_size, +) from . import InstrumentPlugin from .. import formatting @@ -18,12 +21,12 @@ TITLES = { "memory.failcnt": "Failed to allocate memory count", "memory.oom_control.oom_kill_disable": "OOM Control enabled", "memory.oom_control.under_oom": "Was OOM Killer active?", - "cpuacct.usage": "CPU Time" + "cpuacct.usage": "CPU Time", } CONVERSION = { "memory.oom_control.oom_kill_disable": lambda x: "No" if x == 1 else "Yes", "memory.oom_control.under_oom": lambda x: "Yes" if x == 1 else "No", - "cpuacct.usage": lambda x: formatting.seconds_to_str(x / 10**9) # convert nanoseconds + "cpuacct.usage": lambda x: formatting.seconds_to_str(x / 10**9), # convert nanoseconds } CPU_USAGE_TEMPLATE = r""" if [ -e "/proc/$$/cgroup" -a -d "{cgroup_mount}" ]; then @@ -37,7 +40,9 @@ if [ -e "/proc/$$/cgroup" -a -d "{cgroup_mount}" ]; then fi; done; fi -""".replace("\n", " ").strip() +""".replace( + "\n", " " +).strip() MEMORY_USAGE_TEMPLATE = """ if [ -e "/proc/$$/cgroup" -a -d "{cgroup_mount}" ]; then cgroup_path=$(cat "/proc/$$/cgroup" | awk -F':' '$2=="memory"{{print $3}}'); @@ -48,14 +53,15 @@ if [ -e "/proc/$$/cgroup" -a -d "{cgroup_mount}" ]; then echo "__$(basename $f)__" >> {metrics}; cat "$f" >> {metrics} 2>/dev/null; done; fi -""".replace("\n", " ").strip() +""".replace( + "\n", " " +).strip() Metric = namedtuple("Metric", ("key", "subkey", "value")) class CgroupPluginFormatter(formatting.JobMetricFormatter): - def format(self, key, value): title = TITLES.get(key, key) if key in CONVERSION: @@ -71,8 +77,8 @@ class CgroupPluginFormatter(formatting.JobMetricFormatter): class CgroupPlugin(InstrumentPlugin): - """ Plugin that collects memory and cpu utilization from within a cgroup. - """ + """Plugin that collects memory and cpu utilization from within a cgroup.""" + plugin_type = "cgroup" formatter = CgroupPluginFormatter() @@ -98,10 +104,14 @@ class CgroupPlugin(InstrumentPlugin): def __record_cgroup_cpu_usage(self, job_directory): # comounted cgroups (which cpu and cpuacct are on the supported Linux distros) can appear in any order (cpu,cpuacct or cpuacct,cpu) - return CPU_USAGE_TEMPLATE.format(metrics=self.__cgroup_metrics_file(job_directory), cgroup_mount=self.cgroup_mount) + return CPU_USAGE_TEMPLATE.format( + metrics=self.__cgroup_metrics_file(job_directory), cgroup_mount=self.cgroup_mount + ) def __record_cgroup_memory_usage(self, job_directory): - return MEMORY_USAGE_TEMPLATE.format(metrics=self.__cgroup_metrics_file(job_directory), cgroup_mount=self.cgroup_mount) + return MEMORY_USAGE_TEMPLATE.format( + metrics=self.__cgroup_metrics_file(job_directory), cgroup_mount=self.cgroup_mount + ) def __cgroup_metrics_file(self, job_directory): return self._instrument_file_path(job_directory, "_metrics") @@ -126,7 +136,7 @@ class CgroupPlugin(InstrumentPlugin): metrics[metric.subkey] = metric.value def __read_key_value(self, line, key): - if line.startswith('__') and line.endswith('__'): + if line.startswith("__") and line.endswith("__"): # line is the beginning of a new param key = line[2:][:-2] return (None, key) @@ -151,4 +161,4 @@ class CgroupPlugin(InstrumentPlugin): return value -__all__ = ('CgroupPlugin', ) +__all__ = ("CgroupPlugin",) diff --git a/lib/galaxy/job_metrics/instrumenters/collectl.py b/lib/galaxy/job_metrics/instrumenters/collectl.py index 3b696f0d89d..ee740c274d9 100644 --- a/lib/galaxy/job_metrics/instrumenters/collectl.py +++ b/lib/galaxy/job_metrics/instrumenters/collectl.py @@ -9,7 +9,7 @@ from .. import formatting from ..collectl import ( cli, processes, - subsystems + subsystems, ) log = logging.getLogger(__name__) @@ -27,11 +27,12 @@ FORMATTED_RESOURCE_TITLES = { "WSYS": "Disk Writes", } -EMPTY_COLLECTL_FILE_MESSAGE = "Skipping process summary due to empty file... job probably did not run long enough for collectl to gather data." +EMPTY_COLLECTL_FILE_MESSAGE = ( + "Skipping process summary due to empty file... job probably did not run long enough for collectl to gather data." +) class CollectlFormatter(formatting.JobMetricFormatter): - def format(self, key, value): if key == "pid": return ("Process ID", int(value)) @@ -52,9 +53,10 @@ class CollectlFormatter(formatting.JobMetricFormatter): class CollectlPlugin(InstrumentPlugin): - """ Run collectl along with job to capture system and/or process data + """Run collectl along with job to capture system and/or process data according to specified collectl subsystems. """ + plugin_type = "collectl" formatter = CollectlFormatter() @@ -71,7 +73,9 @@ class CollectlPlugin(InstrumentPlugin): self.log_collectl_program_output = util.asbool(kwargs.get("log_collectl_program_output", False)) if self.summarize_process_data: if subsystems.get_subsystem("process") not in self.subsystems: - raise Exception("Collectl plugin misconfigured - cannot summarize_process_data without process subsystem being enabled.") + raise Exception( + "Collectl plugin misconfigured - cannot summarize_process_data without process subsystem being enabled." + ) process_statistics = kwargs.get("process_statistics", None) # None will let processes module use default set of statistics @@ -82,7 +86,7 @@ class CollectlPlugin(InstrumentPlugin): commands = [] # Capture PID of process so we can walk its ancestors when building # statistics for the whole job. - commands.append(f'''echo "$$" > '{self.__pid_file(job_directory)}' ''') + commands.append(f"""echo "$$" > '{self.__pid_file(job_directory)}' """) # Run collectl in record mode to capture process and system level # statistics according to supplied subsystems. commands.append(self.__collectl_record_command(job_directory)) @@ -167,11 +171,7 @@ class CollectlPlugin(InstrumentPlugin): self.collectl_recorder_args = collectl_recorder_args def __summarize_process_data(self, pid, collectl_log_path): - playback_cli_args = dict( - collectl_path=self.local_collectl_path, - playback_path=collectl_log_path, - sep="9" - ) + playback_cli_args = dict(collectl_path=self.local_collectl_path, playback_path=collectl_log_path, sep="9") if not os.stat(collectl_log_path).st_size: log.debug(EMPTY_COLLECTL_FILE_MESSAGE) return [] @@ -212,4 +212,4 @@ def procfilt_argument(procfilt_on): return "" -__all__ = ('CollectlPlugin', ) +__all__ = ("CollectlPlugin",) diff --git a/lib/galaxy/job_metrics/instrumenters/core.py b/lib/galaxy/job_metrics/instrumenters/core.py index 9118d4b36ba..756524979cc 100644 --- a/lib/galaxy/job_metrics/instrumenters/core.py +++ b/lib/galaxy/job_metrics/instrumenters/core.py @@ -15,7 +15,6 @@ RUNTIME_SECONDS_KEY = "runtime_seconds" class CorePluginFormatter(formatting.JobMetricFormatter): - def format(self, key, value): value = int(value) if key == GALAXY_SLOTS_KEY: @@ -27,13 +26,14 @@ class CorePluginFormatter(formatting.JobMetricFormatter): else: # TODO: Use localized version of this from galaxy.ini title = "Job Start Time" if key == START_EPOCH_KEY else "Job End Time" - return (title, time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(value))) + return (title, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(value))) class CorePlugin(InstrumentPlugin): - """ Simple plugin that collects data without external dependencies. In + """Simple plugin that collects data without external dependencies. In particular it currently collects value set for Galaxy slots. """ + plugin_type = "core" formatter = CorePluginFormatter() @@ -69,15 +69,15 @@ class CorePlugin(InstrumentPlugin): def __record_galaxy_slots_command(self, job_directory): galaxy_slots_file = self.__galaxy_slots_file(job_directory) - return f'''echo "$GALAXY_SLOTS" > '{galaxy_slots_file}' ''' + return f"""echo "$GALAXY_SLOTS" > '{galaxy_slots_file}' """ def __record_galaxy_memory_mb_command(self, job_directory): galaxy_memory_mb_file = self.__galaxy_memory_mb_file(job_directory) - return f'''echo "$GALAXY_MEMORY_MB" > '{galaxy_memory_mb_file}' ''' + return f"""echo "$GALAXY_MEMORY_MB" > '{galaxy_memory_mb_file}' """ def __record_seconds_since_epoch_to_file(self, job_directory, name): path = self._instrument_file_path(job_directory, f"epoch_{name}") - return f"date +\"%s\" > {path}" + return f'date +"%s" > {path}' def __read_seconds_since_epoch(self, job_directory, name): path = self._instrument_file_path(job_directory, f"epoch_{name}") @@ -98,4 +98,4 @@ class CorePlugin(InstrumentPlugin): return value -__all__ = ('CorePlugin', ) +__all__ = ("CorePlugin",) diff --git a/lib/galaxy/job_metrics/instrumenters/cpuinfo.py b/lib/galaxy/job_metrics/instrumenters/cpuinfo.py index 9d6ed45e176..4c1672d3fbf 100644 --- a/lib/galaxy/job_metrics/instrumenters/cpuinfo.py +++ b/lib/galaxy/job_metrics/instrumenters/cpuinfo.py @@ -12,7 +12,6 @@ PROCESSOR_LINE = re.compile(r"processor\s*\:\s*(\d+)") class CpuInfoFormatter(formatting.JobMetricFormatter): - def format(self, key, value): if key == "processor_count": return "Processor Count", f"{int(value)}" @@ -21,9 +20,10 @@ class CpuInfoFormatter(formatting.JobMetricFormatter): class CpuInfoPlugin(InstrumentPlugin): - """ Gather information about processor configuration from /proc/cpuinfo. + """Gather information about processor configuration from /proc/cpuinfo. Linux only. """ + plugin_type = "cpuinfo" formatter = CpuInfoFormatter() @@ -60,4 +60,4 @@ class CpuInfoPlugin(InstrumentPlugin): return self._instrument_file_path(job_directory, "cpuinfo") -__all__ = ('CpuInfoPlugin', ) +__all__ = ("CpuInfoPlugin",) diff --git a/lib/galaxy/job_metrics/instrumenters/env.py b/lib/galaxy/job_metrics/instrumenters/env.py index 8e21dcd5709..b46d4a68796 100644 --- a/lib/galaxy/job_metrics/instrumenters/env.py +++ b/lib/galaxy/job_metrics/instrumenters/env.py @@ -13,9 +13,10 @@ class EnvFormatter(formatting.JobMetricFormatter): class EnvPlugin(InstrumentPlugin): - """ Instrumentation plugin capable of recording all or specific environment + """Instrumentation plugin capable of recording all or specific environment variables for a job at runtime. """ + plugin_type = "env" formatter = EnvFormatter() @@ -28,29 +29,28 @@ class EnvPlugin(InstrumentPlugin): self.variables = variables def pre_execute_instrument(self, job_directory): - """ Use env to dump all environment variables to a file. - """ + """Use env to dump all environment variables to a file.""" return f"env > '{self.__env_file(job_directory)}'" def post_execute_instrument(self, job_directory): return None def job_properties(self, job_id, job_directory): - """ Recover environment variables dumped out on compute server and filter + """Recover environment variables dumped out on compute server and filter out specific variables if needed. """ variables = self.variables properties = {} - env_string = ''.join(open(self.__env_file(job_directory)).readlines()) + env_string = "".join(open(self.__env_file(job_directory)).readlines()) while env_string: # Check if the next lines contain a shell function. # We use '\n\}\n' as regex termination because shell # functions can be nested. # We use the non-greedy '.+?' because of re.DOTALL . - m = re.match(r'([^=]+)=(\(\) \{.+?\n\})\n', env_string, re.DOTALL) + m = re.match(r"([^=]+)=(\(\) \{.+?\n\})\n", env_string, re.DOTALL) if m is None: - m = re.match('([^=]+)=(.*)\n', env_string) + m = re.match("([^=]+)=(.*)\n", env_string) if m is None: # Some problem recording or reading back env output. message_template = "Problem parsing env metric output for job %s - properties will be incomplete" @@ -60,7 +60,7 @@ class EnvPlugin(InstrumentPlugin): (var, value) = m.groups() if not variables or var in variables: properties[var] = value - env_string = env_string[m.end():] + env_string = env_string[m.end() :] return properties @@ -68,4 +68,4 @@ class EnvPlugin(InstrumentPlugin): return self._instrument_file_path(job_directory, "vars") -__all__ = ('EnvPlugin', ) +__all__ = ("EnvPlugin",) diff --git a/lib/galaxy/job_metrics/instrumenters/hostname.py b/lib/galaxy/job_metrics/instrumenters/hostname.py index f8cbe26b8d3..813c44297ac 100644 --- a/lib/galaxy/job_metrics/instrumenters/hostname.py +++ b/lib/galaxy/job_metrics/instrumenters/hostname.py @@ -8,14 +8,13 @@ log = logging.getLogger(__name__) class HostnameFormatter(formatting.JobMetricFormatter): - def format(self, key, value): return key, value class HostnamePlugin(InstrumentPlugin): - """ Gather hostname - """ + """Gather hostname""" + plugin_type = "hostname" formatter = HostnameFormatter() @@ -27,10 +26,10 @@ class HostnamePlugin(InstrumentPlugin): def job_properties(self, job_id, job_directory): with open(self.__instrument_hostname_path(job_directory)) as f: - return {'hostname': f.read().strip()} + return {"hostname": f.read().strip()} def __instrument_hostname_path(self, job_directory): return self._instrument_file_path(job_directory, "hostname") -__all__ = ('HostnamePlugin', ) +__all__ = ("HostnamePlugin",) diff --git a/lib/galaxy/job_metrics/instrumenters/meminfo.py b/lib/galaxy/job_metrics/instrumenters/meminfo.py index 7cdb212cc34..9c10a381f02 100644 --- a/lib/galaxy/job_metrics/instrumenters/meminfo.py +++ b/lib/galaxy/job_metrics/instrumenters/meminfo.py @@ -5,27 +5,23 @@ from galaxy import util from . import InstrumentPlugin from .. import formatting - MEMINFO_LINE = re.compile(r"(\w+)\s*\:\s*(\d+) kB") # Important (non-verbose) meminfo property titles. -MEMINFO_TITLES = { - "memtotal": "Total System Memory", - "swaptotal": "Total System Swap" -} +MEMINFO_TITLES = {"memtotal": "Total System Memory", "swaptotal": "Total System Swap"} class MemInfoFormatter(formatting.JobMetricFormatter): - def format(self, key, value): title = MEMINFO_TITLES.get(key, key) return title, util.nice_size(value * 1000) # kB = *1000, KB = *1024 - wikipedia class MemInfoPlugin(InstrumentPlugin): - """ Gather information about processor configuration from /proc/cpuinfo. + """Gather information about processor configuration from /proc/cpuinfo. Linux only. """ + plugin_type = "meminfo" formatter = MemInfoFormatter() @@ -58,4 +54,4 @@ class MemInfoPlugin(InstrumentPlugin): return self._instrument_file_path(job_directory, "meminfo") -__all__ = ('MemInfoPlugin', ) +__all__ = ("MemInfoPlugin",) diff --git a/lib/galaxy/job_metrics/instrumenters/uname.py b/lib/galaxy/job_metrics/instrumenters/uname.py index 288ba2f5c42..b4f96434d6e 100644 --- a/lib/galaxy/job_metrics/instrumenters/uname.py +++ b/lib/galaxy/job_metrics/instrumenters/uname.py @@ -4,15 +4,15 @@ from .. import formatting class UnameFormatter(formatting.JobMetricFormatter): - def format(self, key, value): return "Operating System", value class UnamePlugin(InstrumentPlugin): - """ Use uname to gather operating system information about remote system + """Use uname to gather operating system information about remote system job is running on. Linux only. """ + plugin_type = "uname" formatter = UnameFormatter() @@ -32,4 +32,4 @@ class UnamePlugin(InstrumentPlugin): return self._instrument_file_path(job_directory, "uname") -__all__ = ('UnamePlugin', ) +__all__ = ("UnamePlugin",) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 7524e216adb..499210b86cd 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -13,7 +13,12 @@ import sys import time import traceback from json import loads -from typing import Any, Dict, List, TYPE_CHECKING +from typing import ( + Any, + Dict, + List, + TYPE_CHECKING, +) import packaging.version import yaml @@ -36,11 +41,10 @@ from galaxy.job_execution.output_collect import ( collect_extra_files, collect_shrinked_content_from_path, ) -from galaxy.job_execution.setup import ( # noqa: F401 +from galaxy.job_execution.setup import ( # noqa: F401; This is read by certain misbehaving tool wrappers that import Galaxy internals create_working_directory_for_job, ensure_configs_directory, JobIO, - # This is read by certain misbehaving tool wrappers that import Galaxy internals TOOL_PROVIDED_JOB_METADATA_FILE, TOOL_PROVIDED_JOB_METADATA_KEYS, ) @@ -48,7 +52,10 @@ from galaxy.jobs.mapper import ( JobMappingException, JobRunnerMapper, ) -from galaxy.jobs.runners import BaseJobRunner, JobState +from galaxy.jobs.runners import ( + BaseJobRunner, + JobState, +) from galaxy.metadata import get_metadata_compute_strategy from galaxy.model import store from galaxy.model.store.discover import MaxDiscoveredFilesExceededError @@ -67,7 +74,7 @@ from galaxy.util import ( parse_xml_string, RWXRWXRWX, safe_makedirs, - unicodify + unicodify, ) from galaxy.util.bunch import Bunch from galaxy.util.expressions import ExpressionContext @@ -82,7 +89,7 @@ if TYPE_CHECKING: log = logging.getLogger(__name__) # Override with config.default_job_shell. -DEFAULT_JOB_SHELL = '/bin/bash' +DEFAULT_JOB_SHELL = "/bin/bash" DEFAULT_LOCAL_WORKERS = 4 DEFAULT_CLEANUP_JOB = "always" @@ -95,22 +102,22 @@ class JobDestination(Bunch): """ def __init__(self, **kwds): - self['id'] = None - self['url'] = None - self['tags'] = None - self['runner'] = None - self['legacy'] = False - self['converted'] = False - self['shell'] = None - self['env'] = [] - self['resubmit'] = [] + self["id"] = None + self["url"] = None + self["tags"] = None + self["runner"] = None + self["legacy"] = False + self["converted"] = False + self["shell"] = None + self["env"] = [] + self["resubmit"] = [] # dict is appropriate (rather than a bunch) since keys may not be valid as attributes - self['params'] = dict() + self["params"] = dict() # Use the values persisted in an existing job - if 'from_job' in kwds and kwds['from_job'].destination_id is not None: - self['id'] = kwds['from_job'].destination_id - self['params'] = kwds['from_job'].destination_params + if "from_job" in kwds and kwds["from_job"].destination_id is not None: + self["id"] = kwds["from_job"].destination_id + self["params"] = kwds["from_job"].destination_params super().__init__(**kwds) @@ -124,9 +131,9 @@ class JobToolConfiguration(Bunch): """ def __init__(self, **kwds): - self['handler'] = None - self['destination'] = None - self['params'] = dict() + self["handler"] = None + self["destination"] = None + self["params"] = dict() super().__init__(**kwds) def get_resource_group(self): @@ -135,8 +142,8 @@ class JobToolConfiguration(Bunch): def config_exception(e, file): abs_path = os.path.abspath(file) - message = f'Problem parsing the XML in file {abs_path}, ' - message += 'please correct the indicated portion of the file and restart Galaxy. ' + message = f"Problem parsing the XML in file {abs_path}, " + message += "please correct the indicated portion of the file and restart Galaxy. " message += unicodify(e) log.exception(message) return Exception(message) @@ -149,23 +156,20 @@ def job_config_xml_to_dict(config, root): config_dict["runners"] = runners # Parser plugins section populate 'runners' and 'dynamic' in config_dict. - plugins = root.find('plugins') + plugins = root.find("plugins") if plugins is not None: - for plugin in ConfiguresHandlers._findall_with_required(plugins, 'plugin', ('id', 'type', 'load')): - if plugin.get('type') == 'runner': - workers = plugin.get('workers', plugins.get('workers', JobConfiguration.DEFAULT_NWORKERS)) + for plugin in ConfiguresHandlers._findall_with_required(plugins, "plugin", ("id", "type", "load")): + if plugin.get("type") == "runner": + workers = plugin.get("workers", plugins.get("workers", JobConfiguration.DEFAULT_NWORKERS)) runner_kwds = JobConfiguration.get_params(config, plugin) - plugin_id = plugin.get('id') - runner_info = dict(id=plugin_id, - load=plugin.get('load'), - workers=int(workers), - kwds=runner_kwds) + plugin_id = plugin.get("id") + runner_info = dict(id=plugin_id, load=plugin.get("load"), workers=int(workers), kwds=runner_kwds) runners[plugin_id] = runner_info else: log.error(f"Unknown plugin type: {plugin.get('type')}") - for plugin in ConfiguresHandlers._findall_with_required(plugins, 'plugin', ('id', 'type')): - if plugin.get('id') == 'dynamic' and plugin.get('type') == 'runner': + for plugin in ConfiguresHandlers._findall_with_required(plugins, "plugin", ("id", "type")): + if plugin.get("id") == "dynamic" and plugin.get("type") == "runner": config_dict["dynamic"] = JobConfiguration.get_params(config, plugin) handling_config_dict = ConfiguresHandlers.xml_to_dict(config, root.find("handlers")) @@ -174,9 +178,9 @@ def job_config_xml_to_dict(config, root): # Parse destinations environments = [] - destinations = root.find('destinations') - for destination in ConfiguresHandlers._findall_with_required(destinations, 'destination', ('id', 'runner')): - destination_id = destination.get('id') + destinations = root.find("destinations") + for destination in ConfiguresHandlers._findall_with_required(destinations, "destination", ("id", "runner")): + destination_id = destination.get("id") destination_metrics = destination.get("metrics", None) environment = {"id": destination_id} @@ -188,9 +192,9 @@ def job_config_xml_to_dict(config, root): else: metrics_to_dict = {"src": "path", "path": destination_metrics} else: - metrics_elements = ConfiguresHandlers._findall_with_required(destination, 'job_metrics', ()) + metrics_elements = ConfiguresHandlers._findall_with_required(destination, "job_metrics", ()) if metrics_elements: - metrics_to_dict = {"src": "xml_element", 'xml_element': metrics_elements[0]} + metrics_to_dict = {"src": "xml_element", "xml_element": metrics_elements[0]} environment["metrics"] = metrics_to_dict @@ -200,45 +204,45 @@ def job_config_xml_to_dict(config, root): params["docker_sudo"] = "true" # TODO: handle enabled/disabled in configure_from - environment['params'] = params - environment['env'] = JobConfiguration.get_envs(destination) + environment["params"] = params + environment["env"] = JobConfiguration.get_envs(destination) destination_resubmits = JobConfiguration.get_resubmits(destination) if destination_resubmits: - environment['resubmit'] = destination_resubmits + environment["resubmit"] = destination_resubmits # TODO: handle empty resubmits defaults in configure_from - runner = destination.get('runner') + runner = destination.get("runner") if runner: - environment['runner'] = runner + environment["runner"] = runner - tags = destination.get('tags') + tags = destination.get("tags") # Store tags as a list if tags is not None: - tags = [x.strip() for x in tags.split(',')] - environment['tags'] = tags + tags = [x.strip() for x in tags.split(",")] + environment["tags"] = tags environments.append(environment) - config_dict['execution'] = { - 'environments': environments, + config_dict["execution"] = { + "environments": environments, } default_destination = ConfiguresHandlers.get_xml_default(config, destinations) if default_destination: - config_dict['execution']['default'] = default_destination + config_dict["execution"]["default"] = default_destination resources_config_dict = {} resource_groups = {} # Parse resources... - resources = root.find('resources') + resources = root.find("resources") if resources is not None: default_resource_group = resources.get("default", None) if default_resource_group: resources_config_dict["default"] = default_resource_group - for group in ConfiguresHandlers._findall_with_required(resources, 'group'): - group_id = group.get('id') - fields_str = group.get('fields', None) or group.text or '' + for group in ConfiguresHandlers._findall_with_required(resources, "group"): + group_id = group.get("id") + fields_str = group.get("fields", None) or group.text or "" fields = [f for f in fields_str.split(",") if f] resource_groups[group_id] = fields @@ -246,36 +250,36 @@ def job_config_xml_to_dict(config, root): config_dict["resources"] = resources_config_dict # Parse tool mappings - tools = root.find('tools') - config_dict['tools'] = [] + tools = root.find("tools") + config_dict["tools"] = [] if tools is not None: - for tool in tools.findall('tool'): + for tool in tools.findall("tool"): # There can be multiple definitions with identical ids, but different params tool_mapping_conf = {} - for key in ['handler', 'destination', 'id', 'resources', 'class']: + for key in ["handler", "destination", "id", "resources", "class"]: value = tool.get(key) if value: if key == "destination": key = "environment" tool_mapping_conf[key] = value tool_mapping_conf["params"] = JobConfiguration.get_params(config, tool) - config_dict['tools'].append(tool_mapping_conf) + config_dict["tools"].append(tool_mapping_conf) limits_config = [] - limits = root.find('limits') + limits = root.find("limits") if limits is not None: - for limit in JobConfiguration._findall_with_required(limits, 'limit', ('type',)): + for limit in JobConfiguration._findall_with_required(limits, "limit", ("type",)): limit_dict = {} - for key in ['type', 'tag', 'id', 'window']: - if key == 'type' and key.startswith('destination_'): + for key in ["type", "tag", "id", "window"]: + if key == "type" and key.startswith("destination_"): key = f"environment_{key[len('destination_'):]}" value = limit.get(key) if value: limit_dict[key] = value - limit_dict['value'] = limit.text + limit_dict["value"] = limit.text limits_config.append(limit_dict) - config_dict['limits'] = limits_config + config_dict["limits"] = limits_config return config_dict @@ -284,6 +288,7 @@ class JobConfiguration(ConfiguresHandlers): These features are configured in the job configuration, by default, ``job_conf.xml`` """ + runner_plugins: List[dict] handlers: dict handler_runner_plugins: Dict[str, str] @@ -292,7 +297,7 @@ class JobConfiguration(ConfiguresHandlers): resource_groups: Dict[str, list] destinations: Dict[str, tuple] resource_parameters: Dict[str, Any] - DEFAULT_BASE_HANDLER_POOLS = ('job-handlers',) + DEFAULT_BASE_HANDLER_POOLS = ("job-handlers",) DEFAULT_NWORKERS = 4 @@ -308,8 +313,7 @@ class JobConfiguration(ConfiguresHandlers): """ def __init__(self, app: MinimalManagerApp): - """Parse the job configuration XML. - """ + """Parse the job configuration XML.""" self.app = app self.runner_plugins = [] self.dynamic_params = None @@ -327,34 +331,38 @@ class JobConfiguration(ConfiguresHandlers): self.resource_groups = {} self.default_resource_group = None self.resource_parameters = {} - self.limits = Bunch(registered_user_concurrent_jobs=None, - anonymous_user_concurrent_jobs=None, - walltime=None, - walltime_delta=None, - total_walltime={}, - output_size=None, - destination_user_concurrent_jobs={}, - destination_total_concurrent_jobs={}) + self.limits = Bunch( + registered_user_concurrent_jobs=None, + anonymous_user_concurrent_jobs=None, + walltime=None, + walltime_delta=None, + total_walltime={}, + output_size=None, + destination_user_concurrent_jobs={}, + destination_total_concurrent_jobs={}, + ) default_resubmits = [] default_resubmit_condition = self.app.config.default_job_resubmission_condition if default_resubmit_condition: - default_resubmits.append(dict( - environment=None, - condition=default_resubmit_condition, - handler=None, - delay=None, - )) + default_resubmits.append( + dict( + environment=None, + condition=default_resubmit_condition, + handler=None, + delay=None, + ) + ) self.default_resubmits = default_resubmits self.__parse_resource_parameters() # Initialize the config try: - if 'job_config' in self.app.config.config_dict: + if "job_config" in self.app.config.config_dict: job_config_dict = self.app.config.config_dict["job_config"] else: job_config_file = self.app.config.job_config_file - if '.xml' in job_config_file: + if ".xml" in job_config_file: tree = load(job_config_file) job_config_dict = self.__parse_job_conf_xml(tree) else: @@ -363,15 +371,19 @@ class JobConfiguration(ConfiguresHandlers): # Load tasks if configured if self.app.config.use_tasked_jobs: - job_config_dict["runners"]["tasks"] = dict(id='tasks', load='tasks', workers=self.app.config.local_task_queue_workers, kwds={}) + job_config_dict["runners"]["tasks"] = dict( + id="tasks", load="tasks", workers=self.app.config.local_task_queue_workers, kwds={} + ) self._configure_from_dict(job_config_dict) - log.debug('Done loading job configuration') + log.debug("Done loading job configuration") except OSError: - log.warning('Job configuration "%s" does not exist, using default job configuration', - self.app.config.job_config_file) + log.warning( + 'Job configuration "%s" does not exist, using default job configuration', + self.app.config.job_config_file, + ) self.__set_default_job_conf() except Exception as e: raise config_exception(e, job_config_file) @@ -383,7 +395,7 @@ class JobConfiguration(ConfiguresHandlers): # with a flat dictionary. kwds = {} for key, value in runner_info.items(): - if key in ['id', 'load', 'workers']: + if key in ["id", "load", "workers"]: continue kwds[key] = value runner_info["kwds"] = kwds @@ -392,7 +404,7 @@ class JobConfiguration(ConfiguresHandlers): continue runner_info["id"] = runner_id if runner_id == "dynamic": - log.warning('Deprecated treatment of dynamic running configuration as an actual job runner.') + log.warning("Deprecated treatment of dynamic running configuration as an actual job runner.") self.dynamic_params = runner_info["kwds"] continue self.runner_plugins.append(runner_info) @@ -407,17 +419,20 @@ class JobConfiguration(ConfiguresHandlers): self._set_default_handler_assignment_methods() else: self.app.application_stack.init_job_handling(self) - log.info("Job handler assignment methods set to: %s", ', '.join(self.handler_assignment_methods)) + log.info("Job handler assignment methods set to: %s", ", ".join(self.handler_assignment_methods)) for tag, handlers in [(t, h) for t, h in self.handlers.items() if isinstance(h, list)]: - log.info("Tag [%s] handlers: %s", tag, ', '.join(handlers)) - self.handler_ready_window_size = int(handling_config_dict.get( - 'ready_window_size', JobConfiguration.DEFAULT_HANDLER_READY_WINDOW_SIZE)) + log.info("Tag [%s] handlers: %s", tag, ", ".join(handlers)) + self.handler_ready_window_size = int( + handling_config_dict.get("ready_window_size", JobConfiguration.DEFAULT_HANDLER_READY_WINDOW_SIZE) + ) # Parse environments job_metrics = self.app.job_metrics - execution_dict = job_config_dict.get('execution', {}) + execution_dict = job_config_dict.get("execution", {}) environments = execution_dict.get("environments", []) - enviroment_iter = map(lambda e: (e["id"], e), environments) if isinstance(environments, list) else environments.items() + enviroment_iter = ( + map(lambda e: (e["id"], e), environments) if isinstance(environments, list) else environments.items() + ) for environment_id, environment_dict in enviroment_iter: metrics = environment_dict.get("metrics") if metrics is None: @@ -445,12 +460,12 @@ class JobConfiguration(ConfiguresHandlers): # allowing a flat configuration of these things. params = {} for key, value in environment_dict.items(): - if key in ['id', 'tags', 'runner', 'shell', 'env', 'resubmit']: + if key in ["id", "tags", "runner", "shell", "env", "resubmit"]: continue params[key] = value environment_dict["params"] = params - for key in ['tags', 'runner', 'shell', 'env', 'resubmit', 'params']: + for key in ["tags", "runner", "shell", "env", "resubmit", "params"]: if key in environment_dict: destination_kwds[key] = environment_dict[key] destination_kwds["id"] = environment_id @@ -470,7 +485,9 @@ class JobConfiguration(ConfiguresHandlers): self.destinations[tag].append(job_destination) # Determine the default destination - self.default_destination_id = self._ensure_default_set(execution_dict.get("default"), list(self.destinations.keys()), auto=True) + self.default_destination_id = self._ensure_default_set( + execution_dict.get("default"), list(self.destinations.keys()), auto=True + ) # Read in resources resources = job_config_dict.get("resources", {}) @@ -478,13 +495,13 @@ class JobConfiguration(ConfiguresHandlers): for group_id, fields in resources.get("groups", {}).items(): self.resource_groups[group_id] = fields - tools = job_config_dict.get('tools', []) + tools = job_config_dict.get("tools", []) for tool in tools: - raw_tool_id = tool.get('id') - tool_class = tool.get('class') + raw_tool_id = tool.get("id") + tool_class = tool.get("class") if raw_tool_id is not None: assert tool_class is None - tool_id = raw_tool_id.lower().rstrip('/') + tool_id = raw_tool_id.lower().rstrip("/") if tool_id not in self.tools: self.tools[tool_id] = list() else: @@ -509,46 +526,45 @@ class JobConfiguration(ConfiguresHandlers): else: self.tool_classes[tool_class].append(jtc) - types = dict(registered_user_concurrent_jobs=int, - anonymous_user_concurrent_jobs=int, - walltime=str, - total_walltime=str, - output_size=util.size_to_bytes) + types = dict( + registered_user_concurrent_jobs=int, + anonymous_user_concurrent_jobs=int, + walltime=str, + total_walltime=str, + output_size=util.size_to_bytes, + ) # Parse job limits for limit_dict in job_config_dict.get("limits", []): - limit_type = limit_dict.get('type') + limit_type = limit_dict.get("type") if limit_type.startswith("environment_"): limit_type = f"destination_{limit_type[len('environment_'):]}" limit_value = limit_dict.get("value") # concurrent_jobs renamed to destination_user_concurrent_jobs in job_conf.xml - if limit_type in ('destination_user_concurrent_jobs', 'concurrent_jobs', 'destination_total_concurrent_jobs'): - id = limit_dict.get('tag', None) or limit_dict.get('id') - if limit_type == 'destination_total_concurrent_jobs': + if limit_type in ( + "destination_user_concurrent_jobs", + "concurrent_jobs", + "destination_total_concurrent_jobs", + ): + id = limit_dict.get("tag", None) or limit_dict.get("id") + if limit_type == "destination_total_concurrent_jobs": self.limits.destination_total_concurrent_jobs[id] = int(limit_value) else: self.limits.destination_user_concurrent_jobs[id] = int(limit_value) - elif limit_type == 'total_walltime': - self.limits.total_walltime["window"] = ( - int(limit_dict.get('window')) or 30 - ) - self.limits.total_walltime["raw"] = ( - types.get(limit_type, str)(limit_value) - ) + elif limit_type == "total_walltime": + self.limits.total_walltime["window"] = int(limit_dict.get("window")) or 30 + self.limits.total_walltime["raw"] = types.get(limit_type, str)(limit_value) elif limit_value: self.limits.__dict__[limit_type] = types.get(limit_type, str)(limit_value) if self.limits.walltime is not None: - h, m, s = (int(v) for v in self.limits.walltime.split(':')) + h, m, s = (int(v) for v in self.limits.walltime.split(":")) self.limits.walltime_delta = datetime.timedelta(0, s, 0, 0, m, h) if "raw" in self.limits.total_walltime: - h, m, s = (int(v) for v in - self.limits.total_walltime["raw"].split(':')) - self.limits.total_walltime["delta"] = datetime.timedelta( - 0, s, 0, 0, m, h - ) + h, m, s = (int(v) for v in self.limits.total_walltime["raw"].split(":")) + self.limits.total_walltime["delta"] = datetime.timedelta(0, s, 0, 0, m, h) def __parse_job_conf_xml(self, tree): """Loads the new-style job configuration from options in the job config file (by default, job_conf.xml). @@ -557,7 +573,7 @@ class JobConfiguration(ConfiguresHandlers): :type tree: ``lxml.etree._Element`` """ root = tree.getroot() - log.debug(f'Loading job configuration from {self.app.config.job_config_file}') + log.debug(f"Loading job configuration from {self.app.config.job_config_file}") job_config_dict = job_config_xml_to_dict(self.app.config, root) return job_config_dict @@ -570,10 +586,10 @@ class JobConfiguration(ConfiguresHandlers): def __set_default_job_conf(self): # Run jobs locally - self.runner_plugins = [dict(id='local', load='local', workers=DEFAULT_LOCAL_WORKERS)] + self.runner_plugins = [dict(id="local", load="local", workers=DEFAULT_LOCAL_WORKERS)] # Load tasks if configured if self.app.config.use_tasked_jobs: - self.runner_plugins.append(dict(id='tasks', load='tasks', workers=DEFAULT_LOCAL_WORKERS)) + self.runner_plugins.append(dict(id="tasks", load="tasks", workers=DEFAULT_LOCAL_WORKERS)) # Set the handlers self._init_handler_assignment_methods() if not self.handler_assignment_methods_configured: @@ -582,12 +598,12 @@ class JobConfiguration(ConfiguresHandlers): self.app.application_stack.init_job_handling(self) self.handler_ready_window_size = JobConfiguration.DEFAULT_HANDLER_READY_WINDOW_SIZE # Set the destination - self.default_destination_id = 'local' - self.destinations['local'] = [JobDestination(id='local', runner='local')] - log.debug('Done loading job configuration') + self.default_destination_id = "local" + self.destinations["local"] = [JobDestination(id="local", runner="local")] + log.debug("Done loading job configuration") def get_tool_resource_xml(self, tool_id, tool_type): - """ Given a tool id, return XML elements describing parameters to + """Given a tool id, return XML elements describing parameters to insert into job resources. :tool id: A tool ID (a string) @@ -595,7 +611,7 @@ class JobConfiguration(ConfiguresHandlers): :returns: List of parameter elements. """ - if tool_id and tool_type in ('default', 'manage_data'): + if tool_id and tool_type in ("default", "manage_data"): # TODO: Only works with exact matches, should handle different kinds of ids # the way destination lookup does. resource_group = None @@ -615,7 +631,7 @@ class JobConfiguration(ConfiguresHandlers): if fields: conditional_element = parse_xml_string(self.JOB_RESOURCE_CONDITIONAL_XML) - when_yes_elem = conditional_element.findall('when')[1] + when_yes_elem = conditional_element.findall("when")[1] for parameter in fields: when_yes_elem.append(parameter) return conditional_element @@ -626,19 +642,19 @@ class JobConfiguration(ConfiguresHandlers): @staticmethod def get_params(config, parent): rval = {} - for param in parent.findall('param'): - key = param.get('id') + for param in parent.findall("param"): + key = param.get("id") if key in ["container", "container_override"]: - containers = map(requirements.container_from_element, param.findall('container')) + containers = map(requirements.container_from_element, param.findall("container")) param_value = list(map(lambda c: c.to_dict(), containers)) else: param_value = param.text - if 'from_environ' in param.attrib: - environ_var = param.attrib['from_environ'] + if "from_environ" in param.attrib: + environ_var = param.attrib["from_environ"] param_value = os.environ.get(environ_var, param_value) - elif 'from_config' in param.attrib: - config_val = param.attrib['from_config'] + elif "from_config" in param.attrib: + config_val = param.attrib["from_config"] param_value = config.config_dict.get(config_val, param_value) rval[key] = param_value @@ -664,14 +680,16 @@ class JobConfiguration(ConfiguresHandlers): :returns: dict """ rval = [] - for param in parent.findall('env'): - rval.append(dict( - name=param.get('id'), - file=param.get('file'), - execute=param.get('exec'), - value=param.text, - raw=util.asbool(param.get('raw', 'false')) - )) + for param in parent.findall("env"): + rval.append( + dict( + name=param.get("id"), + file=param.get("file"), + execute=param.get("exec"), + value=param.text, + raw=util.asbool(param.get("raw", "false")), + ) + ) return rval @staticmethod @@ -684,13 +702,15 @@ class JobConfiguration(ConfiguresHandlers): :returns: dict """ rval = [] - for resubmit in parent.findall('resubmit'): - rval.append(dict( - condition=resubmit.get('condition'), - environment=resubmit.get('destination'), - handler=resubmit.get('handler'), - delay=resubmit.get('delay'), - )) + for resubmit in parent.findall("resubmit"): + rval.append( + dict( + condition=resubmit.get("condition"), + environment=resubmit.get("destination"), + handler=resubmit.get("handler"), + delay=resubmit.get("delay"), + ) + ) return rval def __is_enabled(self, params): @@ -711,7 +731,9 @@ class JobConfiguration(ConfiguresHandlers): :returns: JobToolConfiguration -- a representation of a element that uses the default handler and destination """ - return JobToolConfiguration(id='_default_', handler=self.default_handler_id, destination=self.default_destination_id) + return JobToolConfiguration( + id="_default_", handler=self.default_handler_id, destination=self.default_destination_id + ) # Called upon instantiation of a Tool object def get_job_tool_configurations(self, ids, tool_classes): @@ -793,24 +815,28 @@ class JobConfiguration(ConfiguresHandlers): """ rval = {} if handler_id in self.handler_runner_plugins: - plugins_to_load = [rp for rp in self.runner_plugins if rp['id'] in self.handler_runner_plugins[handler_id]] - log.info("Handler '%s' will load specified runner plugins: %s", handler_id, ', '.join(rp['id'] for rp in plugins_to_load)) + plugins_to_load = [rp for rp in self.runner_plugins if rp["id"] in self.handler_runner_plugins[handler_id]] + log.info( + "Handler '%s' will load specified runner plugins: %s", + handler_id, + ", ".join(rp["id"] for rp in plugins_to_load), + ) else: plugins_to_load = self.runner_plugins log.info("Handler '%s' will load all configured runner plugins", handler_id) for runner in plugins_to_load: class_names = [] module = None - id = runner['id'] - load = runner['load'] - if ':' in load: + id = runner["id"] + load = runner["load"] + if ":" in load: # Name to load was specified as ':' - module_name, class_name = load.rsplit(':', 1) + module_name, class_name = load.rsplit(":", 1) class_names = [class_name] module = __import__(module_name) else: # Name to load was specified as '' - if '.' not in load: + if "." not in load: # For legacy reasons, try from galaxy.jobs.runners first if there's no '.' in the name module_name = f"galaxy.jobs.runners.{load}" try: @@ -844,13 +870,20 @@ class JobConfiguration(ConfiguresHandlers): log.warning(f"A non-class name was found in __all__, ignoring: {id}") continue except AssertionError: - log.warning(f"Job runner classes must be subclassed from BaseJobRunner, {id} has bases: {runner_class.__bases__}") + log.warning( + f"Job runner classes must be subclassed from BaseJobRunner, {id} has bases: {runner_class.__bases__}" + ) continue try: - rval[id] = runner_class(self.app, runner.get('workers', JobConfiguration.DEFAULT_NWORKERS), **runner.get('kwds', {})) + rval[id] = runner_class( + self.app, runner.get("workers", JobConfiguration.DEFAULT_NWORKERS), **runner.get("kwds", {}) + ) except TypeError: - log.exception("Job runner '%s:%s' has not been converted to a new-style runner or encountered TypeError on load", - module_name, class_name) + log.exception( + "Job runner '%s:%s' has not been converted to a new-style runner or encountered TypeError on load", + module_name, + class_name, + ) rval[id] = runner_class(self.app) log.debug(f"Loaded job runner '{module_name}:{class_name}' as '{id}'") return rval @@ -881,7 +914,9 @@ class JobConfiguration(ConfiguresHandlers): :param job_runners: All loaded job runner plugins. :type job_runners: list of job runner plugins """ - for id, destination in [(id, destinations[0]) for id, destinations in self.destinations.items() if self.is_id(destinations)]: + for id, destination in [ + (id, destinations[0]) for id, destinations in self.destinations.items() if self.is_id(destinations) + ]: # Only need to deal with real destinations, not members of tags if destination.legacy and not destination.converted: if destination.runner in job_runners: @@ -894,11 +929,12 @@ class JobConfiguration(ConfiguresHandlers): else: log.debug(f"Legacy destination with id '{id}', url '{destination.url}' converted, got params:") else: - log.warning(f"Legacy destination with id '{id}' could not be converted: Unknown runner plugin: {destination.runner}") + log.warning( + f"Legacy destination with id '{id}' could not be converted: Unknown runner plugin: {destination.runner}" + ) class HasResourceParameters: - def get_resource_parameters(self, job=None): # Find the dymically inserted resource parameters and give them # to rule. @@ -925,9 +961,10 @@ class JobWrapper(HasResourceParameters): Wraps a 'model.Job' with convenience methods for running processes and state management. """ + is_task = False - def __init__(self, job, queue: 'JobHandlerQueue', use_persisted_destination=False): + def __init__(self, job, queue: "JobHandlerQueue", use_persisted_destination=False): self.job_id = job.id self.session_id = job.session_id self.user_id = job.user_id @@ -971,19 +1008,28 @@ class JobWrapper(HasResourceParameters): def external_output_metadata(self): if self.__external_output_metadata is None: try: - metadata_strategy_override = self.get_destination_configuration('metadata_strategy', None) + metadata_strategy_override = self.get_destination_configuration("metadata_strategy", None) except JobMappingException: metadata_strategy_override = None if self.__has_tasks: metadata_strategy_override = "directory" - self.__external_output_metadata = get_metadata_compute_strategy(self.app.config, self.job_id, metadata_strategy_override=metadata_strategy_override, tool_id=self.tool.id) + self.__external_output_metadata = get_metadata_compute_strategy( + self.app.config, + self.job_id, + metadata_strategy_override=metadata_strategy_override, + tool_id=self.tool.id, + ) return self.__external_output_metadata @property def remote_command_line(self): - use_remote = self.get_destination_configuration('tool_evaluation_strategy') == 'remote' + use_remote = self.get_destination_configuration("tool_evaluation_strategy") == "remote" # It wouldn't be hard to support history export, but we want to do this in task queue workers anyway ... - return use_remote and self.external_output_metadata.extended and not self.sa_session.query(model.JobExportHistoryArchive).filter_by(job=self.get_job()).first() + return ( + use_remote + and self.external_output_metadata.extended + and not self.sa_session.query(model.JobExportHistoryArchive).filter_by(job=self.get_job()).first() + ) def tool_directory(self): tool_dir = self.tool.tool_dir @@ -1027,8 +1073,7 @@ class JobWrapper(HasResourceParameters): @property def outputs_directory(self): - """Default location of ``outputs_to_working_directory``. - """ + """Default location of ``outputs_to_working_directory``.""" return None if self.created_with_galaxy_version < packaging.version.parse("20.01") else "outputs" @property @@ -1051,8 +1096,7 @@ class JobWrapper(HasResourceParameters): @property def cleanup_job(self): - """ Remove the job after it is complete, should return "always", "onsuccess", or "never". - """ + """Remove the job after it is complete, should return "always", "onsuccess", or "never".""" return self.get_destination_configuration("cleanup_job", DEFAULT_CLEANUP_JOB) @property @@ -1061,14 +1105,14 @@ class JobWrapper(HasResourceParameters): @property def use_metadata_binary(self): - return util.asbool(self.get_destination_configuration('use_metadata_binary', "False")) + return util.asbool(self.get_destination_configuration("use_metadata_binary", "False")) def can_split(self): # Should the job handler split this job up? return self.app.config.use_tasked_jobs and self.tool.parallelism def get_job_runner_url(self): - log.warning(f'({self.job_id}) Job runner URLs are deprecated, use destinations instead.') + log.warning(f"({self.job_id}) Job runner URLs are deprecated, use destinations instead.") return self.job_destination.url def get_parallelism(self): @@ -1076,7 +1120,7 @@ class JobWrapper(HasResourceParameters): @property def shell(self): - return self.job_destination.shell or getattr(self.app.config, 'default_job_shell', DEFAULT_JOB_SHELL) + return self.job_destination.shell or getattr(self.app.config, "default_job_shell", DEFAULT_JOB_SHELL) def disable_commands_in_new_shell(self): """Provide an extension point to disable this isolation, @@ -1100,7 +1144,7 @@ class JobWrapper(HasResourceParameters): @property def galaxy_virtual_env(self): - return os.environ.get('VIRTUAL_ENV', None) + return os.environ.get("VIRTUAL_ENV", None) # legacy naming get_job_runner = get_job_runner_url @@ -1156,10 +1200,9 @@ class JobWrapper(HasResourceParameters): return os.path.abspath(os.path.join(self.working_directory, COMMAND_VERSION_FILENAME)) def __prepare_upload_paramfile(self, job): - """Special case paramfile handling for the upload tool. Copies the paramfile to the working directory - """ - new = os.path.join(self.working_directory, 'upload_params.json') - param_file_path = json.loads(next(iter(param.value for param in job.parameters if param.name == 'paramfile'))) + """Special case paramfile handling for the upload tool. Copies the paramfile to the working directory""" + new = os.path.join(self.working_directory, "upload_params.json") + param_file_path = json.loads(next(iter(param.value for param in job.parameters if param.name == "paramfile"))) try: shutil.copy2(param_file_path, new) except OSError as exc: @@ -1186,7 +1229,7 @@ class JobWrapper(HasResourceParameters): return self.sa_session.query(model.GenomeIndexToolData).filter_by(job=job).first() # TODO: The upload tool actions that create the paramfile can probably be turned in to a configfile to remove this special casing - if job.tool_id == 'upload1': + if job.tool_id == "upload1": self.__prepare_upload_paramfile(job) tool_evaluator = self._get_tool_evaluator(job) @@ -1201,11 +1244,13 @@ class JobWrapper(HasResourceParameters): self.galaxy_lib_dir if self.tool.requires_galaxy_python_environment or self.remote_command_line: # These tools (upload, metadata, data_source) may need access to the datatypes registry. - self.app.datatypes_registry.to_xml_file(os.path.join(self.working_directory, 'registry.xml')) + self.app.datatypes_registry.to_xml_file(os.path.join(self.working_directory, "registry.xml")) if self.remote_command_line: - os.makedirs(os.path.join(self.working_directory, 'metadata', 'outputs_new'), exist_ok=True) - self.job_io.to_json(path=os.path.join(self.working_directory, 'metadata', 'outputs_new', 'job_io.json')) - self.app.tool_data_tables.to_json(path=os.path.join(self.working_directory, 'metadata', 'outputs_new', 'tool_data_tables.json')) + os.makedirs(os.path.join(self.working_directory, "metadata", "outputs_new"), exist_ok=True) + self.job_io.to_json(path=os.path.join(self.working_directory, "metadata", "outputs_new", "job_io.json")) + self.app.tool_data_tables.to_json( + path=os.path.join(self.working_directory, "metadata", "outputs_new", "tool_data_tables.json") + ) job.dependencies = self.tool.dependencies self.sa_session.add(job) self.sa_session.flush() @@ -1220,18 +1265,16 @@ class JobWrapper(HasResourceParameters): # The tool execution is given a working directory beneath the # "job" working directory. safe_makedirs(self.tool_working_directory) - safe_makedirs(os.path.join(working_directory, 'outputs')) - log.debug('(%s) Working directory for job is: %s', - self.job_id, self.working_directory) + safe_makedirs(os.path.join(working_directory, "outputs")) + log.debug("(%s) Working directory for job is: %s", self.job_id, self.working_directory) except ObjectInvalid: - raise Exception('(%s) Unable to create job working directory', - job.id) + raise Exception("(%s) Unable to create job working directory", job.id) @property def guest_ports(self): if hasattr(self, "interactivetools"): # This works when the job is being prepared - guest_ports = [ep.get('port') for ep in self.interactivetools] + guest_ports = [ep.get("port") for ep in self.interactivetools] return guest_ports else: # This works when handling a running job @@ -1251,12 +1294,13 @@ class JobWrapper(HasResourceParameters): self._set_object_store_ids(job) self.__working_directory = self.app.object_store.get_filename( - job, base_dir='job_work', dir_only=True, obj_dir=True) + job, base_dir="job_work", dir_only=True, obj_dir=True + ) return self.__working_directory def working_directory_exists(self): job = self.get_job() - return self.app.object_store.exists(job, base_dir='job_work', dir_only=True, obj_dir=True) + return self.app.object_store.exists(job, base_dir="job_work", dir_only=True, obj_dir=True) @property def tool_working_directory(self): @@ -1268,24 +1312,22 @@ class JobWrapper(HasResourceParameters): def clear_working_directory(self): job = self.get_job() if not os.path.exists(self.working_directory): - log.warning('(%s): Working directory clear requested but %s does ' - 'not exist', - self.job_id, - self.working_directory) + log.warning( + "(%s): Working directory clear requested but %s does " "not exist", self.job_id, self.working_directory + ) return self.object_store.create( - job, base_dir='job_work', dir_only=True, obj_dir=True, - extra_dir='_cleared_contents', extra_dir_at_root=True) + job, base_dir="job_work", dir_only=True, obj_dir=True, extra_dir="_cleared_contents", extra_dir_at_root=True + ) base = self.object_store.get_filename( - job, base_dir='job_work', dir_only=True, obj_dir=True, - extra_dir='_cleared_contents', extra_dir_at_root=True) - date_str = datetime.datetime.now().strftime('%Y%m%d-%H%M%S') + job, base_dir="job_work", dir_only=True, obj_dir=True, extra_dir="_cleared_contents", extra_dir_at_root=True + ) + date_str = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") arc_dir = os.path.join(base, date_str) shutil.move(self.working_directory, arc_dir) self._setup_working_directory(job=job) - log.debug('(%s) Previous working directory moved to %s', - self.job_id, arc_dir) + log.debug("(%s) Previous working directory moved to %s", self.job_id, arc_dir) def default_compute_environment(self, job=None): if not job: @@ -1297,7 +1339,7 @@ class JobWrapper(HasResourceParameters): # Restore parameters from the database job = self.get_job() if job.user is None and job.galaxy_session is None: - raise Exception(f'Job {job.id} has no user and no session.') + raise Exception(f"Job {job.id} has no user and no session.") return job def _get_tool_evaluator(self, job): @@ -1315,7 +1357,9 @@ class JobWrapper(HasResourceParameters): if os.path.exists(path): util.umask_fix_perms(path, self.app.config.umask, 0o666, self.app.config.gid) - def fail(self, message, exception=False, tool_stdout="", tool_stderr="", exit_code=None, job_stdout=None, job_stderr=None): + def fail( + self, message, exception=False, tool_stdout="", tool_stderr="", exit_code=None, job_stdout=None, job_stderr=None + ): """ Indicate job failure by setting state and message on all output datasets. @@ -1329,9 +1373,13 @@ class JobWrapper(HasResourceParameters): try: self.job_destination except JobMappingException as exc: - log.debug("(%s) fail(): Job destination raised JobMappingException('%s'), caching fake '__fail__' " - "destination for completion of fail method", self.get_id_tag(), unicodify(exc.failure_message)) - self.job_runner_mapper.cached_job_destination = JobDestination(id='__fail__') + log.debug( + "(%s) fail(): Job destination raised JobMappingException('%s'), caching fake '__fail__' " + "destination for completion of fail method", + self.get_id_tag(), + unicodify(exc.failure_message), + ) + self.job_runner_mapper.cached_job_destination = JobDestination(id="__fail__") # Might be AssertionError or other exception message = str(message) @@ -1359,30 +1407,38 @@ class JobWrapper(HasResourceParameters): dataset = dataset_assoc.dataset self.sa_session.refresh(dataset) dataset.state = dataset.states.ERROR - dataset.blurb = 'tool error' + dataset.blurb = "tool error" dataset.info = message dataset.set_size() dataset.dataset.set_total_size() dataset.mark_unhidden() - if dataset.ext == 'auto': - dataset.extension = 'data' + if dataset.ext == "auto": + dataset.extension = "data" try: self.__update_output(job, dataset) except Exception: # Failure to update the output of a failed job should not prevent completion of the failure method - log.exception("(%s) fail(): Failed to update job output dataset with id: %s", self.get_id_tag(), - dataset.dataset.id) + log.exception( + "(%s) fail(): Failed to update job output dataset with id: %s", + self.get_id_tag(), + dataset.dataset.id, + ) # Pause any dependent jobs (and those jobs' outputs) for dep_job_assoc in dataset.dependent_jobs: - self.pause(dep_job_assoc.job, "Execution of this dataset's job is paused because its input datasets are in an error state.") - job.set_final_state(job.states.ERROR, supports_skip_locked=self.app.application_stack.supports_skip_locked()) + self.pause( + dep_job_assoc.job, + "Execution of this dataset's job is paused because its input datasets are in an error state.", + ) + job.set_final_state( + job.states.ERROR, supports_skip_locked=self.app.application_stack.supports_skip_locked() + ) job.command_line = self.command_line job.info = message # TODO: Put setting the stdout, stderr, and exit code in one place # (not duplicated with the finish method). job.set_streams(tool_stdout, tool_stderr, job_stdout=job_stdout, job_stderr=job_stderr) # Let the exit code be Null if one is not provided: - if (exit_code is not None): + if exit_code is not None: job.exit_code = exit_code self.sa_session.add(job) @@ -1398,7 +1454,9 @@ class JobWrapper(HasResourceParameters): self._fix_output_permissions() self._report_error() # Perform email action even on failure. - for pja in [pjaa.post_job_action for pjaa in job.post_job_actions if pjaa.post_job_action.action_type == "EmailAction"]: + for pja in [ + pjaa.post_job_action for pjaa in job.post_job_actions if pjaa.post_job_action.action_type == "EmailAction" + ]: ActionBox.execute(self.app, self.sa_session, pja, job) # If the job was deleted, call tool specific fail actions (used for e.g. external metadata) and clean up if self.tool: @@ -1407,7 +1465,7 @@ class JobWrapper(HasResourceParameters): except Exception: log.exception(f"Error occured while calling tool specific fail actions for job {job.id}") cleanup_job = self.cleanup_job - delete_files = cleanup_job == 'always' or (cleanup_job == 'onsuccess' and job.state == job.states.DELETED) + delete_files = cleanup_job == "always" or (cleanup_job == "onsuccess" and job.state == job.states.DELETED) self.cleanup(delete_files=delete_files) def pause(self, job=None, message=None): @@ -1455,8 +1513,12 @@ class JobWrapper(HasResourceParameters): # thread and no other threads are working on the job yet - so don't refresh. if job.state in model.Job.terminal_states: - log.warning("(%s) Ignoring state change from '%s' to '%s' for job " - "that is already terminal", job.id, job.state, state) + log.warning( + "(%s) Ignoring state change from '%s' to '%s' for job " "that is already terminal", + job.id, + job.state, + state, + ) return if info: job.info = info @@ -1472,7 +1534,7 @@ class JobWrapper(HasResourceParameters): return job.state def set_runner(self, runner_url, external_id): - log.warning('set_runner() is deprecated, use set_job_destination()') + log.warning("set_runner() is deprecated, use set_job_destination()") self.set_job_destination(self.job_destination, external_id) def set_job_destination(self, job_destination, external_id=None, flush=True, job=None): @@ -1484,7 +1546,7 @@ class JobWrapper(HasResourceParameters): """ if job is None: job = self.get_job() - log.debug(f'({job.id}) Persisting job destination (destination id: {job_destination.id})') + log.debug(f"({job.id}) Persisting job destination (destination id: {job_destination.id})") job.destination_id = job_destination.id job.destination_params = job_destination.params job.job_runner_name = job_destination.runner @@ -1511,13 +1573,11 @@ class JobWrapper(HasResourceParameters): return self.tool.tmp_target def get_destination_configuration(self, key, default=None): - """ Get a destination parameter that can be defaulted back + """Get a destination parameter that can be defaulted back in app.config if it needs to be applied globally. """ dest_params = self.job_destination.params - return self.get_job().get_destination_configuration( - dest_params, self.app.config, key, default - ) + return self.get_job().get_destination_configuration(dest_params, self.app.config, key, default) def enqueue(self): job = self.get_job() @@ -1566,23 +1626,23 @@ class JobWrapper(HasResourceParameters): trynum = self.app.config.retry_job_output_collection except (OSError, ObjectNotFound) as e: trynum += 1 - log.warning('Error accessing dataset with ID %i, will retry: %s', dataset.dataset.id, unicodify(e)) + log.warning("Error accessing dataset with ID %i, will retry: %s", dataset.dataset.id, unicodify(e)) time.sleep(2) if getattr(dataset, "hidden_beneath_collection_instance", None): dataset.visible = False - dataset.blurb = 'done' - dataset.peek = 'no peek' - dataset.info = (dataset.info or '') - if context['stdout'].strip(): + dataset.blurb = "done" + dataset.peek = "no peek" + dataset.info = dataset.info or "" + if context["stdout"].strip(): # Ensure white space between entries dataset.info = f"{dataset.info.rstrip()}\n{context['stdout'].strip()}" - if context['stderr'].strip(): + if context["stderr"].strip(): # Ensure white space between entries dataset.info = f"{dataset.info.rstrip()}\n{context['stderr'].strip()}" dataset.tool_version = self.version_string dataset.set_size() - if 'uuid' in context: - dataset.dataset.uuid = context['uuid'] + if "uuid" in context: + dataset.dataset.uuid = context["uuid"] self.__update_output(job, dataset) if not purged: collect_extra_files(self.object_store, dataset, self.working_directory) @@ -1593,30 +1653,42 @@ class JobWrapper(HasResourceParameters): dataset.mark_unhidden() elif not purged: # If the tool was expected to set the extension, attempt to retrieve it - if dataset.ext == 'auto': - dataset.extension = context.get('ext', 'data') + if dataset.ext == "auto": + dataset.extension = context.get("ext", "data") dataset.init_meta(copy_from=dataset) # if a dataset was copied, it won't appear in our dictionary: # either use the metadata from originating output dataset, or call set_meta on the copies # it would be quicker to just copy the metadata from the originating output dataset, # but somewhat trickier (need to recurse up the copied_from tree), for now we'll call set_meta() retry_internally = util.asbool(self.get_destination_configuration("retry_metadata_internally", True)) - if not retry_internally and self.tool.tool_type == 'interactive': - retry_internally = util.asbool(self.get_destination_configuration("retry_interactivetool_metadata_internally", retry_internally)) - metadata_set_successfully = self.external_output_metadata.external_metadata_set_successfully(dataset, output_name, self.sa_session, working_directory=self.working_directory) + if not retry_internally and self.tool.tool_type == "interactive": + retry_internally = util.asbool( + self.get_destination_configuration("retry_interactivetool_metadata_internally", retry_internally) + ) + metadata_set_successfully = self.external_output_metadata.external_metadata_set_successfully( + dataset, output_name, self.sa_session, working_directory=self.working_directory + ) if retry_internally and not metadata_set_successfully: # If Galaxy was expected to sniff type and didn't - do so. if dataset.ext == "_sniff_": - extension = sniff.handle_uploaded_dataset_file(dataset.dataset.file_name, self.app.datatypes_registry) + extension = sniff.handle_uploaded_dataset_file( + dataset.dataset.file_name, self.app.datatypes_registry + ) dataset.extension = extension # call datatype.set_meta directly for the initial set_meta call during dataset creation dataset.datatype.set_meta(dataset, overwrite=False) - elif (job.states.ERROR != final_job_state and not metadata_set_successfully): + elif job.states.ERROR != final_job_state and not metadata_set_successfully: dataset._state = model.Dataset.states.FAILED_METADATA else: - self.external_output_metadata.load_metadata(dataset, output_name, self.sa_session, working_directory=self.working_directory, remote_metadata_directory=remote_metadata_directory) - line_count = context.get('line_count', None) + self.external_output_metadata.load_metadata( + dataset, + output_name, + self.sa_session, + working_directory=self.working_directory, + remote_metadata_directory=remote_metadata_directory, + ) + line_count = context.get("line_count", None) try: # Certain datatype's set_peek methods contain a line_count argument dataset.set_peek(line_count=line_count) @@ -1626,8 +1698,8 @@ class JobWrapper(HasResourceParameters): else: # Handle purged datasets. dataset.blurb = "empty" - if dataset.ext == 'auto': - dataset.extension = context.get('ext', 'txt') + if dataset.ext == "auto": + dataset.extension = context.get("ext", "txt") for context_key in TOOL_PROVIDED_JOB_METADATA_KEYS: if context_key in context: @@ -1653,8 +1725,7 @@ class JobWrapper(HasResourceParameters): the contents of the output files. """ finish_timer = self.app.execution_timer_factory.get_timer( - 'internals.galaxy.jobs.job_wrapper_finish', - 'job_wrapper.finish for job ${job_id} executed' + "internals.galaxy.jobs.job_wrapper_finish", "job_wrapper.finish for job ${job_id} executed" ) # default post job setup @@ -1665,13 +1736,21 @@ class JobWrapper(HasResourceParameters): if not isinstance(exception, (AssertionError, MessageException)): # Only attach MessageException and AssertionErrors to job.traceback exception = None - return self.fail(message, tool_stdout=tool_stdout, tool_stderr=tool_stderr, exit_code=tool_exit_code, job_stdout=job_stdout, job_stderr=job_stderr, exception=exception) + return self.fail( + message, + tool_stdout=tool_stdout, + tool_stderr=tool_stderr, + exit_code=tool_exit_code, + job_stdout=job_stdout, + job_stderr=job_stderr, + exception=exception, + ) # TODO: After failing here, consider returning from the function. try: self.reclaim_ownership() except Exception: - log.exception(f'({job.id}) Failed to change ownership of {self.working_directory}, failing') + log.exception(f"({job.id}) Failed to change ownership of {self.working_directory}, failing") return fail() # if the job was deleted, don't finish it @@ -1683,7 +1762,7 @@ class JobWrapper(HasResourceParameters): # the tasks failed. So include the stderr, stdout, and exit code: return fail() - extended_metadata = self.external_output_metadata.extended and not self.tool.tool_type == 'interactive' + extended_metadata = self.external_output_metadata.extended and not self.tool.tool_type == "interactive" # We collect the stderr from tools that write their stderr to galaxy.json tool_provided_metadata = self.get_tool_provided_job_metadata() @@ -1695,7 +1774,14 @@ class JobWrapper(HasResourceParameters): # We set final_job_state to use for dataset management, but *don't* set # job.state until after dataset discovery to prevent history issues if check_output_detected_state is None: - check_output_detected_state = self.check_tool_output(tool_stdout, tool_stderr, tool_exit_code=tool_exit_code, job=job, job_stdout=job_stdout, job_stderr=job_stderr) + check_output_detected_state = self.check_tool_output( + tool_stdout, + tool_stderr, + tool_exit_code=tool_exit_code, + job=job, + job_stdout=job_stdout, + job_stderr=job_stderr, + ) if check_output_detected_state == DETECTED_JOB_STATE.OK and not tool_provided_metadata.has_failed_outputs(): final_job_state = job.states.OK @@ -1713,8 +1799,10 @@ class JobWrapper(HasResourceParameters): # finish method - the false_path file has already moved, # and when the job is recovered, it won't be found. if os.path.exists(dataset_path.real_path) and os.stat(dataset_path.real_path).st_size > 0: - log.warning("finish(): %s not found, but %s is not empty, so it will be used instead" - % (dataset_path.false_path, dataset_path.real_path)) + log.warning( + "finish(): %s not found, but %s is not empty, so it will be used instead" + % (dataset_path.false_path, dataset_path.real_path) + ) else: # Prior to fail we need to set job.state job.set_state(final_job_state) @@ -1725,7 +1813,7 @@ class JobWrapper(HasResourceParameters): try: import_options = store.ImportOptions(allow_dataset_object_edit=True, allow_edit=True) import_model_store = store.get_import_model_store_for_directory( - os.path.join(self.working_directory, 'metadata', 'outputs_populated'), + os.path.join(self.working_directory, "metadata", "outputs_populated"), app=self.app, import_options=import_options, user=job.user, @@ -1756,13 +1844,14 @@ class JobWrapper(HasResourceParameters): # should this also be checking library associations? - can a library item be added from a history before the job has ended? - # lets not allow this to occur # need to update all associated output hdas, i.e. history was shared with job running - for dataset in dataset_assoc.dataset.dataset.history_associations + dataset_assoc.dataset.dataset.library_associations: + for dataset in ( + dataset_assoc.dataset.dataset.history_associations + + dataset_assoc.dataset.dataset.library_associations + ): output_name = dataset_assoc.name # Handles retry internally on error for instance... - self._finish_dataset( - output_name, dataset, job, context, final_job_state, remote_metadata_directory - ) + self._finish_dataset(output_name, dataset, job, context, final_job_state, remote_metadata_directory) if not final_job_state == job.states.ERROR: dataset_assoc.dataset.dataset.state = model.Dataset.states.OK try: @@ -1778,7 +1867,10 @@ class JobWrapper(HasResourceParameters): dataset_assoc.dataset.dataset.state = model.Dataset.states.ERROR # Pause any dependent jobs (and those jobs' outputs) for dep_job_assoc in dataset_assoc.dataset.dependent_jobs: - self.pause(dep_job_assoc.job, "Execution of this dataset's job is paused because its input datasets are in an error state.") + self.pause( + dep_job_assoc.job, + "Execution of this dataset's job is paused because its input datasets are in an error state.", + ) for pja in job.post_job_actions: ActionBox.execute(self.app, self.sa_session, pja.post_job_action, job, final_job_state=final_job_state) @@ -1803,15 +1895,24 @@ class JobWrapper(HasResourceParameters): # ( this used to be performed in the "exec_after_process" hook, but hooks are deprecated ). param_dict = self.get_param_dict(job) try: - self.tool.exec_after_process(self.app, inp_data, out_data, param_dict, job=job, final_job_state=final_job_state) + self.tool.exec_after_process( + self.app, inp_data, out_data, param_dict, job=job, final_job_state=final_job_state + ) except Exception as e: log.exception(f"exec_after_process hook failed for job {self.job_id}") return fail("exec_after_process hook failed", exception=e) # Call 'exec_after_process' hook - self.tool.call_hook('exec_after_process', self.app, inp_data=inp_data, - out_data=out_data, param_dict=param_dict, - tool=self.tool, stdout=job.stdout, stderr=job.stderr) + self.tool.call_hook( + "exec_after_process", + self.app, + inp_data=inp_data, + out_data=out_data, + param_dict=param_dict, + tool=self.tool, + stdout=job.stdout, + stderr=job.stderr, + ) self._fix_output_permissions() @@ -1835,7 +1936,7 @@ class JobWrapper(HasResourceParameters): if job.state == job.states.ERROR: self._report_error() cleanup_job = self.cleanup_job - delete_files = cleanup_job == 'always' or (job.state == job.states.OK and cleanup_job == 'onsuccess') + delete_files = cleanup_job == "always" or (job.state == job.states.OK and cleanup_job == "onsuccess") self.cleanup(delete_files=delete_files) log.debug(finish_timer.to_str(job_id=self.job_id, tool_id=job.tool_id)) @@ -1851,14 +1952,14 @@ class JobWrapper(HasResourceParameters): input_dbkey = loads(input_dbkey) else: # Legacy jobs without __input_ext. - input_ext = 'data' - input_dbkey = '?' + input_ext = "data" + input_dbkey = "?" for _, data in inp_data.items(): # For loop odd, but sort simulating behavior in galaxy.tools.actions if not data: continue input_ext = data.ext - input_dbkey = data.dbkey or '?' + input_dbkey = data.dbkey or "?" # Create generated output children and primary datasets. tool_working_directory = self.tool_working_directory @@ -1880,11 +1981,15 @@ class JobWrapper(HasResourceParameters): if job is not None: job_id_tag = job.get_id_tag() - state, tool_stdout, tool_stderr, job_messages = check_output(self.tool.stdio_regexes, self.tool.stdio_exit_codes, tool_stdout, tool_stderr, tool_exit_code, job_id_tag) + state, tool_stdout, tool_stderr, job_messages = check_output( + self.tool.stdio_regexes, self.tool.stdio_exit_codes, tool_stdout, tool_stderr, tool_exit_code, job_id_tag + ) # Store the modified stdout and stderr in the job: if job is not None: - job.set_streams(tool_stdout, tool_stderr, job_messages=job_messages, job_stdout=job_stdout, job_stderr=job_stderr) + job.set_streams( + tool_stdout, tool_stderr, job_messages=job_messages, job_stdout=job_stdout, job_stderr=job_stderr + ) return state @@ -1902,16 +2007,22 @@ class JobWrapper(HasResourceParameters): raise self.external_output_metadata.cleanup_external_metadata(self.sa_session) if delete_files: - self.object_store.delete(self.get_job(), base_dir='job_work', entire_dir=True, dir_only=True, obj_dir=True) + self.object_store.delete( + self.get_job(), base_dir="job_work", entire_dir=True, dir_only=True, obj_dir=True + ) except Exception: log.exception("Unable to cleanup job %d", self.job_id) def _collect_metrics(self, has_metrics, job_metrics_directory=None): job = has_metrics.get_job() job_metrics_directory = job_metrics_directory or self.working_directory - per_plugin_properties = self.app.job_metrics.collect_properties(job.destination_id, self.job_id, job_metrics_directory) + per_plugin_properties = self.app.job_metrics.collect_properties( + job.destination_id, self.job_id, job_metrics_directory + ) if per_plugin_properties: - log.info(f"Collecting metrics for {type(has_metrics).__name__} {getattr(has_metrics, 'id', None)} in {job_metrics_directory}") + log.info( + f"Collecting metrics for {type(has_metrics).__name__} {getattr(has_metrics, 'id', None)} in {job_metrics_directory}" + ) for plugin, properties in per_plugin_properties.items(): for metric_name, metric_value in properties.items(): if metric_value is not None: @@ -1931,16 +2042,28 @@ class JobWrapper(HasResourceParameters): if self.app.job_config.limits.output_size and self.app.job_config.limits.output_size > 0: for outfile, size in self.get_output_sizes(): if size > self.app.job_config.limits.output_size: - log.warning('(%s) Job output size %s has exceeded the global output size limit', self.get_id_tag(), os.path.basename(outfile)) - return (JobState.runner_states.OUTPUT_SIZE_LIMIT, - 'Job output file grew too large (greater than %s), please try different inputs or parameters' - % util.nice_size(self.app.job_config.limits.output_size)) + log.warning( + "(%s) Job output size %s has exceeded the global output size limit", + self.get_id_tag(), + os.path.basename(outfile), + ) + return ( + JobState.runner_states.OUTPUT_SIZE_LIMIT, + "Job output file grew too large (greater than %s), please try different inputs or parameters" + % util.nice_size(self.app.job_config.limits.output_size), + ) if self.app.job_config.limits.walltime_delta is not None and runtime is not None: if runtime > self.app.job_config.limits.walltime_delta: - log.warning('(%s) Job runtime %s has exceeded the global walltime, it will be terminated', self.get_id_tag(), runtime) - return (JobState.runner_states.GLOBAL_WALLTIME_REACHED, - 'Job ran longer than the maximum allowed execution time (runtime: %s, limit: %s), please try different inputs or parameters' - % (str(runtime).split('.')[0], self.app.job_config.limits.walltime)) + log.warning( + "(%s) Job runtime %s has exceeded the global walltime, it will be terminated", + self.get_id_tag(), + runtime, + ) + return ( + JobState.runner_states.GLOBAL_WALLTIME_REACHED, + "Job ran longer than the maximum allowed execution time (runtime: %s, limit: %s), please try different inputs or parameters" + % (str(runtime).split(".")[0], self.app.job_config.limits.walltime), + ) return None def has_limits(self): @@ -1956,7 +2079,7 @@ class JobWrapper(HasResourceParameters): def get_env_setup_clause(self): if self.app.config.environment_setup_file is None: - return '' + return "" return f'[ -f "{self.app.config.environment_setup_file}" ] && . {self.app.config.environment_setup_file}' @property @@ -1969,7 +2092,9 @@ class JobWrapper(HasResourceParameters): try: if not tmp_dir or util.asbool(tmp_dir): working_directory = self.working_directory - return '''$([ ! -e '{0}/tmp' ] || mv '{0}/tmp' '{0}'/tmp.$(date +%Y%m%d-%H%M%S) ; mkdir '{0}/tmp'; echo '{0}/tmp')'''.format(working_directory) + return """$([ ! -e '{0}/tmp' ] || mv '{0}/tmp' '{0}'/tmp.$(date +%Y%m%d-%H%M%S) ; mkdir '{0}/tmp'; echo '{0}/tmp')""".format( + working_directory + ) else: return tmp_dir except ValueError: @@ -2018,23 +2143,33 @@ class JobWrapper(HasResourceParameters): def invalidate_external_metadata(self): job = self.get_job() - self.external_output_metadata.invalidate_external_metadata([output_dataset_assoc.dataset for - output_dataset_assoc in - job.output_datasets + job.output_library_datasets], - self.sa_session) + self.external_output_metadata.invalidate_external_metadata( + [ + output_dataset_assoc.dataset + for output_dataset_assoc in job.output_datasets + job.output_library_datasets + ], + self.sa_session, + ) - def setup_external_metadata(self, exec_dir=None, tmp_dir=None, - dataset_files_path=None, config_root=None, - config_file=None, datatypes_config=None, - resolve_metadata_dependencies=False, - set_extension=True, **kwds): + def setup_external_metadata( + self, + exec_dir=None, + tmp_dir=None, + dataset_files_path=None, + config_root=None, + config_file=None, + datatypes_config=None, + resolve_metadata_dependencies=False, + set_extension=True, + **kwds, + ): # extension could still be 'auto' if this is the upload tool. job = self.get_job() if set_extension: for output_dataset_assoc in job.output_datasets: - if output_dataset_assoc.dataset.ext == 'auto': + if output_dataset_assoc.dataset.ext == "auto": context = self.get_dataset_finish_context(dict(), output_dataset_assoc) - output_dataset_assoc.dataset.extension = context.get('ext', 'data') + output_dataset_assoc.dataset.extension = context.get("ext", "data") self.sa_session.flush() if tmp_dir is None: # this dir should should relative to the exec_dir @@ -2046,38 +2181,42 @@ class JobWrapper(HasResourceParameters): if config_file is None: config_file = self.app.config.config_file if datatypes_config is None: - datatypes_config = os.path.join(self.working_directory, 'metadata', 'registry.xml') - safe_makedirs(os.path.join(self.working_directory, 'metadata')) + datatypes_config = os.path.join(self.working_directory, "metadata", "registry.xml") + safe_makedirs(os.path.join(self.working_directory, "metadata")) self.app.datatypes_registry.to_xml_file(path=datatypes_config) inp_data, out_data, out_collections = job.io_dicts(exclude_implicit_outputs=True) job_metadata = os.path.join(self.tool_working_directory, self.tool.provided_metadata_file) object_store_conf = self.object_store.to_dict() - command = self.external_output_metadata.setup_external_metadata(out_data, - out_collections, - self.sa_session, - exec_dir=exec_dir, - tmp_dir=tmp_dir, - dataset_files_path=dataset_files_path, - config_root=config_root, - config_file=config_file, - datatypes_config=datatypes_config, - job_metadata=job_metadata, - provided_metadata_style=self.tool.provided_metadata_style, - object_store_conf=object_store_conf, - tool=self.tool, - job=job, - max_metadata_value_size=self.app.config.max_metadata_value_size, - max_discovered_files=self.app.config.max_discovered_files, - validate_outputs=self.validate_outputs, - link_data_only=self.__link_file_check(), - **kwds) + command = self.external_output_metadata.setup_external_metadata( + out_data, + out_collections, + self.sa_session, + exec_dir=exec_dir, + tmp_dir=tmp_dir, + dataset_files_path=dataset_files_path, + config_root=config_root, + config_file=config_file, + datatypes_config=datatypes_config, + job_metadata=job_metadata, + provided_metadata_style=self.tool.provided_metadata_style, + object_store_conf=object_store_conf, + tool=self.tool, + job=job, + max_metadata_value_size=self.app.config.max_metadata_value_size, + max_discovered_files=self.app.config.max_discovered_files, + validate_outputs=self.validate_outputs, + link_data_only=self.__link_file_check(), + **kwds, + ) if resolve_metadata_dependencies: metadata_tool = self.app.toolbox.get_tool("__SET_METADATA__") if metadata_tool is not None: # Due to tool shed hacks for migrate and installed tool tests... # see (``setup_shed_tools_for_test`` in test/base/driver_util.py). - dependency_shell_commands = metadata_tool.build_dependency_shell_commands(job_directory=self.working_directory, metadata=True) + dependency_shell_commands = metadata_tool.build_dependency_shell_commands( + job_directory=self.working_directory, metadata=True + ) if dependency_shell_commands: dependency_shell_commands = "; ".join(dependency_shell_commands) command = f"{dependency_shell_commands}; {command}" @@ -2114,10 +2253,14 @@ class JobWrapper(HasResourceParameters): return True def container_monitor_command(self, container, **kwds): - if not container or not self.tool.produces_entry_points or not self.get_destination_configuration("container_monitor", True): + if ( + not container + or not self.tool.produces_entry_points + or not self.get_destination_configuration("container_monitor", True) + ): return None - exec_dir = kwds.get('exec_dir', os.path.abspath(os.getcwd())) + exec_dir = kwds.get("exec_dir", os.path.abspath(os.getcwd())) work_dir = self.working_directory configs_dir = ensure_configs_directory(work_dir) container_config = os.path.join(configs_dir, "container_config.json") @@ -2136,11 +2279,7 @@ class JobWrapper(HasResourceParameters): encoded_job_id = self.app.security.encode_id(job_id) job_key = self.app.security.encode_id(job_id, kind="jobs_files") endpoint_base = "%s/api/jobs/%s/ports?job_key=%s" - callback_url = endpoint_base % ( - galaxy_url, - encoded_job_id, - job_key - ) + callback_url = endpoint_base % (galaxy_url, encoded_job_id, job_key) container_config_dict["callback_url"] = callback_url with open(container_config, "w") as f: @@ -2157,7 +2296,7 @@ class JobWrapper(HasResourceParameters): elif job.galaxy_session is not None: return f"anonymous@{job.galaxy_session.remote_addr.split()[-1]}" else: - return 'anonymous@unknown' + return "anonymous@unknown" def __update_output(self, job, hda, clean_only=False): """Handle writing outputs to the object store. @@ -2182,7 +2321,7 @@ class JobWrapper(HasResourceParameters): pass def __link_file_check(self): - """ outputs_to_working_directory breaks library uploads where data is + """outputs_to_working_directory breaks library uploads where data is linked. This method is a hack that solves that problem, but is specific to the upload tool and relies on an injected job param. This method should be removed ASAP and replaced with some properly generic @@ -2191,7 +2330,7 @@ class JobWrapper(HasResourceParameters): if self.tool: job = self.get_job() param_dict = job.get_param_values(self.app) - return self.tool.id == 'upload1' and param_dict.get('link_data_only', None) == 'link_to_files' + return self.tool.id == "upload1" and param_dict.get("link_data_only", None) == "link_to_files" else: # The tool is unavailable, we try to move the outputs. return False @@ -2200,8 +2339,9 @@ class JobWrapper(HasResourceParameters): job = self.get_job() external_chown_script = self.get_destination_configuration("external_chown_script", None) if job.user is not None and external_chown_script: - ret = external_chown(self.working_directory, self.user_system_pwent, - external_chown_script, description="working directory") + ret = external_chown( + self.working_directory, self.user_system_pwent, external_chown_script, description="working directory" + ) if not ret: os.chmod(self.working_directory, RWXRWXRWX) @@ -2209,8 +2349,9 @@ class JobWrapper(HasResourceParameters): job = self.get_job() external_chown_script = self.get_destination_configuration("external_chown_script", None) if job.user is not None and external_chown_script: - external_chown(self.working_directory, self.galaxy_system_pwent, - external_chown_script, description="working directory") + external_chown( + self.working_directory, self.galaxy_system_pwent, external_chown_script, description="working directory" + ) @property def user_system_pwent(self): @@ -2246,7 +2387,12 @@ class JobWrapper(HasResourceParameters): def set_container(self, container): if container: - cont = model.JobContainerAssociation(job=self.get_job(), container_type=container.container_type, container_name=container.container_name, container_info=container.container_info) + cont = model.JobContainerAssociation( + job=self.get_job(), + container_type=container.container_type, + container_name=container.container_name, + container_info=container.container_info, + ) self.sa_session.add(cont) self.sa_session.flush() @@ -2256,6 +2402,7 @@ class TaskWrapper(JobWrapper): Extension of JobWrapper intended for running tasks. Should be refactored into a generalized executable unit wrapper parent, then jobs and tasks. """ + # Abstract this to be more useful for running tasks that *don't* necessarily compose a job. is_task = True @@ -2317,12 +2464,12 @@ class TaskWrapper(JobWrapper): self.sa_session.add(task) self.sa_session.flush() - self.status = 'prepared' + self.status = "prepared" return self.extra_filenames def fail(self, message, exception=False): log.error(f"TaskWrapper Failure {message}") - self.status = 'error' + self.status = "error" # How do we want to handle task failure? Fail the job and let it clean up? def change_state(self, state, info=False, flush=True, job=None): @@ -2363,16 +2510,17 @@ class TaskWrapper(JobWrapper): """ # This may have ended too soon - log.debug('task %s for job %d ended; exit code: %d' - % (self.task_id, self.job_id, - tool_exit_code if tool_exit_code is not None else -256)) + log.debug( + "task %s for job %d ended; exit code: %d" + % (self.task_id, self.job_id, tool_exit_code if tool_exit_code is not None else -256) + ) # default post job setup_external_metadata self.sa_session.expunge_all() task = self.get_task() # if the job was deleted, don't finish it if task.state == task.states.DELETED: # Job was deleted by an administrator - delete_files = self.cleanup_job in ('always', 'onsuccess') + delete_files = self.cleanup_job in ("always", "onsuccess") self.cleanup(delete_files=delete_files) return elif task.state == task.states.ERROR: @@ -2416,9 +2564,17 @@ class TaskWrapper(JobWrapper): # Handled at the parent job level. Do nothing here. pass - def setup_external_metadata(self, exec_dir=None, tmp_dir=None, dataset_files_path=None, - config_root=None, config_file=None, datatypes_config=None, - set_extension=True, **kwds): + def setup_external_metadata( + self, + exec_dir=None, + tmp_dir=None, + dataset_files_path=None, + config_root=None, + config_file=None, + datatypes_config=None, + set_extension=True, + **kwds, + ): # There is no metadata setting for tasks. This is handled after the merge, at the job level. return "" diff --git a/lib/galaxy/jobs/command_factory.py b/lib/galaxy/jobs/command_factory.py index eb76d84c39f..5b8a90dd5bb 100644 --- a/lib/galaxy/jobs/command_factory.py +++ b/lib/galaxy/jobs/command_factory.py @@ -2,7 +2,7 @@ from logging import getLogger from os import getcwd from os.path import ( abspath, - join + join, ) from galaxy import util @@ -77,7 +77,9 @@ def build_command( external_command_shell = container.shell else: external_command_shell = shell - externalized_commands = __externalize_commands(job_wrapper, external_command_shell, commands_builder, remote_command_params, container=container) + externalized_commands = __externalize_commands( + job_wrapper, external_command_shell, commands_builder, remote_command_params, container=container + ) if container and modify_command_for_container: # Stop now and build command before handling metadata and copying # working directory files back. These should always happen outside @@ -85,9 +87,7 @@ def build_command( # metadata and means no need for Galaxy to be available to container # and not copying workdir outputs back means on can be more restrictive # of where container can write to in some circumstances. - run_in_container_command = container.containerize_command( - externalized_commands - ) + run_in_container_command = container.containerize_command(externalized_commands) commands_builder = CommandsBuilder(run_in_container_command) else: commands_builder = CommandsBuilder(externalized_commands) @@ -102,7 +102,7 @@ def build_command( # xref https://github.com/galaxyproject/galaxy/issues/3289 commands_builder.prepend_command(PREPARE_DIRS) - for_pulsar = 'script_directory' in remote_command_params + for_pulsar = "script_directory" in remote_command_params __handle_remote_command_line_building(commands_builder, job_wrapper, for_pulsar=for_pulsar) container_monitor_command = job_wrapper.container_monitor_command(container) @@ -124,14 +124,16 @@ def build_command( return commands_builder.build() -def __externalize_commands(job_wrapper, shell, commands_builder, remote_command_params, script_name="tool_script.sh", container=None): +def __externalize_commands( + job_wrapper, shell, commands_builder, remote_command_params, script_name="tool_script.sh", container=None +): local_container_script = join(job_wrapper.working_directory, script_name) tool_commands = commands_builder.build() integrity_injection = "" # Setting shell to none in job_conf.xml disables creating a tool command script, # set -e doesn't work for composite commands but this is necessary for Windows jobs # for instance. - if shell and shell.lower() == 'none': + if shell and shell.lower() == "none": return tool_commands if job_wrapper.job_io.check_job_script_integrity: integrity_injection = INTEGRITY_INJECTION @@ -162,7 +164,7 @@ def __externalize_commands(job_wrapper, shell, commands_builder, remote_command_ # doesn't need to mount the job directory (rw) and then eliminate this hack # (or restrict to older Pulsar versions). # https://github.com/galaxyproject/galaxy/pull/8449 - for_pulsar = 'script_directory' in remote_command_params + for_pulsar = "script_directory" in remote_command_params if for_pulsar: commands = f"{shell} {join(remote_command_params['script_directory'], script_name)}" else: @@ -172,7 +174,7 @@ def __externalize_commands(job_wrapper, shell, commands_builder, remote_command_ def __handle_remote_command_line_building(commands_builder, job_wrapper, for_pulsar=False): - if getattr(job_wrapper, 'remote_command_line', False): + if getattr(job_wrapper, "remote_command_line", False): sep = "" if for_pulsar else "&&" command = 'PYTHONPATH="$GALAXY_LIB:$PYTHONPATH" python "$GALAXY_LIB"/galaxy/tools/remote_tool_eval.py' if for_pulsar: @@ -184,7 +186,7 @@ def __handle_remote_command_line_building(commands_builder, job_wrapper, for_pul def __handle_task_splitting(commands_builder, job_wrapper): # prepend getting input files (if defined) - if getattr(job_wrapper, 'prepare_input_files_cmds', None): + if getattr(job_wrapper, "prepare_input_files_cmds", None): commands_builder.prepend_commands(job_wrapper.prepare_input_files_cmds) @@ -198,8 +200,8 @@ def __handle_dependency_resolution(commands_builder, job_wrapper, remote_command def __handle_work_dir_outputs(commands_builder, job_wrapper, runner, remote_command_params): # Append commands to copy job outputs based on from_work_dir attribute. work_dir_outputs_kwds = {} - if 'working_directory' in remote_command_params: - work_dir_outputs_kwds['job_working_directory'] = remote_command_params['working_directory'] + if "working_directory" in remote_command_params: + work_dir_outputs_kwds["job_working_directory"] = remote_command_params["working_directory"] work_dir_outputs = runner.get_work_dir_outputs(job_wrapper, **work_dir_outputs_kwds) if work_dir_outputs: commands_builder.capture_return_code() @@ -210,30 +212,33 @@ def __handle_work_dir_outputs(commands_builder, job_wrapper, runner, remote_comm def __handle_metadata(commands_builder, job_wrapper, runner, remote_command_params): # Append metadata setting commands, we don't want to overwrite metadata # that was copied over in init_meta(), as per established behavior - metadata_kwds = remote_command_params.get('metadata_kwds', {}) - exec_dir = metadata_kwds.get('exec_dir', abspath(getcwd())) - tmp_dir = metadata_kwds.get('tmp_dir', job_wrapper.working_directory) - dataset_files_path = metadata_kwds.get('dataset_files_path', runner.app.model.Dataset.file_path) - output_fnames = metadata_kwds.get('output_fnames', job_wrapper.job_io.get_output_fnames()) - config_root = metadata_kwds.get('config_root', None) - config_file = metadata_kwds.get('config_file', None) - datatypes_config = metadata_kwds.get('datatypes_config', None) - compute_tmp_dir = metadata_kwds.get('compute_tmp_dir', None) + metadata_kwds = remote_command_params.get("metadata_kwds", {}) + exec_dir = metadata_kwds.get("exec_dir", abspath(getcwd())) + tmp_dir = metadata_kwds.get("tmp_dir", job_wrapper.working_directory) + dataset_files_path = metadata_kwds.get("dataset_files_path", runner.app.model.Dataset.file_path) + output_fnames = metadata_kwds.get("output_fnames", job_wrapper.job_io.get_output_fnames()) + config_root = metadata_kwds.get("config_root", None) + config_file = metadata_kwds.get("config_file", None) + datatypes_config = metadata_kwds.get("datatypes_config", None) + compute_tmp_dir = metadata_kwds.get("compute_tmp_dir", None) resolve_metadata_dependencies = job_wrapper.commands_in_new_shell - metadata_command = job_wrapper.setup_external_metadata( - exec_dir=exec_dir, - tmp_dir=tmp_dir, - dataset_files_path=dataset_files_path, - output_fnames=output_fnames, - set_extension=False, - config_root=config_root, - config_file=config_file, - datatypes_config=datatypes_config, - compute_tmp_dir=compute_tmp_dir, - resolve_metadata_dependencies=resolve_metadata_dependencies, - use_bin=job_wrapper.use_metadata_binary, - kwds={'overwrite': False} - ) or '' + metadata_command = ( + job_wrapper.setup_external_metadata( + exec_dir=exec_dir, + tmp_dir=tmp_dir, + dataset_files_path=dataset_files_path, + output_fnames=output_fnames, + set_extension=False, + config_root=config_root, + config_file=config_file, + datatypes_config=datatypes_config, + compute_tmp_dir=compute_tmp_dir, + resolve_metadata_dependencies=resolve_metadata_dependencies, + use_bin=job_wrapper.use_metadata_binary, + kwds={"overwrite": False}, + ) + or "" + ) metadata_command = metadata_command.strip() if metadata_command: # Place Galaxy and its dependencies in environment for metadata regardless of tool. @@ -244,18 +249,17 @@ def __handle_metadata(commands_builder, job_wrapper, runner, remote_command_para def __copy_if_exists_command(work_dir_output): source_file, destination = work_dir_output - if '?' in source_file or '*' in source_file: - source_file = source_file.replace('*', '"*"').replace('?', '"?"') + if "?" in source_file or "*" in source_file: + source_file = source_file.replace("*", '"*"').replace("?", '"?"') return f'\nif [ -f "{source_file}" ] ; then cp "{source_file}" "{destination}" ; fi' class CommandsBuilder: - - def __init__(self, initial_command=''): + def __init__(self, initial_command=""): # Remove trailing semi-colon so we can start hacking up this command. # TODO: Refactor to compose a list and join with ';', would be more clean. self.raw_command = initial_command - initial_command = util.unicodify(initial_command or '') + initial_command = util.unicodify(initial_command or "") commands = initial_command.rstrip("; ") self.commands = commands @@ -272,7 +276,7 @@ class CommandsBuilder: def prepend_commands(self, commands): return self.prepend_command("; ".join(c for c in commands if c)) - def append_command(self, command, sep=';'): + def append_command(self, command, sep=";"): if command: self.commands = f"{self.commands}{sep} {command}" return self @@ -281,14 +285,15 @@ class CommandsBuilder: self.append_command("; ".join(c for c in commands if c)) def capture_stdout_stderr(self, stdout_file, stderr_file): - self.prepend_command("""out="${TMPDIR:-/tmp}/out.$$" err="${TMPDIR:-/tmp}/err.$$" + self.prepend_command( + """out="${TMPDIR:-/tmp}/out.$$" err="${TMPDIR:-/tmp}/err.$$" mkfifo "$out" "$err" trap 'rm "$out" "$err"' EXIT tee -a stdout.log < "$out" & tee -a stderr.log < "$err" >&2 &""", - sep="") - self.append_command(f"> '{stdout_file}' 2> '{stderr_file}'", - sep="") + sep="", + ) + self.append_command(f"> '{stdout_file}' 2> '{stderr_file}'", sep="") def capture_return_code(self): if not self.return_code_captured: @@ -301,4 +306,4 @@ tee -a stderr.log < "$err" >&2 &""", return self.commands -__all__ = ("build_command", ) +__all__ = ("build_command",) diff --git a/lib/galaxy/jobs/dynamic_tool_destination.py b/lib/galaxy/jobs/dynamic_tool_destination.py index 3ab28227a22..e327c5eee39 100755 --- a/lib/galaxy/jobs/dynamic_tool_destination.py +++ b/lib/galaxy/jobs/dynamic_tool_destination.py @@ -15,7 +15,7 @@ import yaml from galaxy.util import parse_xml -__version__ = '1.1.0' +__version__ = "1.1.0" # log to galaxy's logger log = logging.getLogger(__name__) @@ -46,13 +46,16 @@ max_edit_dist = 2 """ List of valid categories that can be expected in the configuration. """ -valid_categories = ['verbose', 'tools', 'default_destination', - 'users', 'default_priority'] +valid_categories = ["verbose", "tools", "default_destination", "users", "default_priority"] # --- destination validation error messages --- # dest_err_default_dest = "Default destination '%s' does not appear in the job configuration." # destination -dest_err_tool_default_dest = "Default destination for '%s': '%s' does not appear in the job configuration." # tool, destination -dest_err_tool_rule_dest = "Destination for '%s', rule %s: '%s' does not exist in job configuration." # tool, counter, destination +dest_err_tool_default_dest = ( + "Default destination for '%s': '%s' does not appear in the job configuration." # tool, destination +) +dest_err_tool_rule_dest = ( + "Destination for '%s', rule %s: '%s' does not exist in job configuration." # tool, counter, destination +) class MalformedYMLException(Exception): @@ -96,21 +99,20 @@ class RuleValidator: :returns: validated rule or result of validation (depending on return_bool) """ - if rule_type == 'file_size': + if rule_type == "file_size": return cls.__validate_file_size_rule(app, return_bool, *args, **kwargs) - elif rule_type == 'num_input_datasets': + elif rule_type == "num_input_datasets": return cls.__validate_num_input_datasets_rule(app, return_bool, *args, **kwargs) - elif rule_type == 'records': + elif rule_type == "records": return cls.__validate_records_rule(app, return_bool, *args, **kwargs) - elif rule_type == 'arguments': + elif rule_type == "arguments": return cls.__validate_arguments_rule(app, return_bool, *args, **kwargs) @classmethod - def __validate_file_size_rule( - cls, app, return_bool, original_rule, counter, tool): + def __validate_file_size_rule(cls, app, return_bool, original_rule, counter, tool): """ This function is responsible for validating 'file_size' rules. @@ -138,23 +140,19 @@ class RuleValidator: # Users Verification # if rule is not None: - valid_rule, rule = cls.__validate_users( - valid_rule, return_bool, rule, tool, counter) + valid_rule, rule = cls.__validate_users(valid_rule, return_bool, rule, tool, counter) # Nice_value Verification # if rule is not None: - valid_rule, rule = cls.__validate_nice_value( - valid_rule, return_bool, rule, tool, counter) + valid_rule, rule = cls.__validate_nice_value(valid_rule, return_bool, rule, tool, counter) # Destination Verification # if rule is not None: - valid_rule, rule = cls.__validate_destination( - valid_rule, app, return_bool, rule, tool, counter) + valid_rule, rule = cls.__validate_destination(valid_rule, app, return_bool, rule, tool, counter) # Bounds Verification # if rule is not None: - valid_rule, rule = cls.__validate_bounds( - valid_rule, return_bool, rule, tool, counter) + valid_rule, rule = cls.__validate_bounds(valid_rule, return_bool, rule, tool, counter) if return_bool: return valid_rule @@ -163,8 +161,7 @@ class RuleValidator: return rule @classmethod - def __validate_num_input_datasets_rule( - cls, app, return_bool, original_rule, counter, tool): + def __validate_num_input_datasets_rule(cls, app, return_bool, original_rule, counter, tool): """ This function is responsible for validating 'num_input_datasets' rules. @@ -192,23 +189,19 @@ class RuleValidator: # Users Verification # if rule is not None: - valid_rule, rule = cls.__validate_users( - valid_rule, return_bool, rule, tool, counter) + valid_rule, rule = cls.__validate_users(valid_rule, return_bool, rule, tool, counter) # Nice_value Verification # if rule is not None: - valid_rule, rule = cls.__validate_nice_value( - valid_rule, return_bool, rule, tool, counter) + valid_rule, rule = cls.__validate_nice_value(valid_rule, return_bool, rule, tool, counter) # Destination Verification # if rule is not None: - valid_rule, rule = cls.__validate_destination( - valid_rule, app, return_bool, rule, tool, counter) + valid_rule, rule = cls.__validate_destination(valid_rule, app, return_bool, rule, tool, counter) # Bounds Verification # if rule is not None: - valid_rule, rule = cls.__validate_bounds( - valid_rule, return_bool, rule, tool, counter) + valid_rule, rule = cls.__validate_bounds(valid_rule, return_bool, rule, tool, counter) if return_bool: return valid_rule @@ -245,23 +238,19 @@ class RuleValidator: # Users Verification # if rule is not None: - valid_rule, rule = cls.__validate_users( - valid_rule, return_bool, rule, tool, counter) + valid_rule, rule = cls.__validate_users(valid_rule, return_bool, rule, tool, counter) # Nice_value Verification # if rule is not None: - valid_rule, rule = cls.__validate_nice_value( - valid_rule, return_bool, rule, tool, counter) + valid_rule, rule = cls.__validate_nice_value(valid_rule, return_bool, rule, tool, counter) # Destination Verification # if rule is not None: - valid_rule, rule = cls.__validate_destination( - valid_rule, app, return_bool, rule, tool, counter) + valid_rule, rule = cls.__validate_destination(valid_rule, app, return_bool, rule, tool, counter) # Bounds Verification # if rule is not None: - valid_rule, rule = cls.__validate_bounds( - valid_rule, return_bool, rule, tool, counter) + valid_rule, rule = cls.__validate_bounds(valid_rule, return_bool, rule, tool, counter) if return_bool: return valid_rule @@ -270,8 +259,7 @@ class RuleValidator: return rule @classmethod - def __validate_arguments_rule( - cls, app, return_bool, original_rule, counter, tool): + def __validate_arguments_rule(cls, app, return_bool, original_rule, counter, tool): """ This is responsible for validating 'arguments' rules. @@ -299,24 +287,20 @@ class RuleValidator: # Users Verification # if rule is not None: - valid_rule, rule = cls.__validate_users( - valid_rule, return_bool, rule, tool, counter) + valid_rule, rule = cls.__validate_users(valid_rule, return_bool, rule, tool, counter) # Nice_value Verification # if rule is not None: - valid_rule, rule = cls.__validate_nice_value( - valid_rule, return_bool, rule, tool, counter) + valid_rule, rule = cls.__validate_nice_value(valid_rule, return_bool, rule, tool, counter) # Destination Verification # if rule is not None: - valid_rule, rule = cls.__validate_destination( - valid_rule, app, return_bool, rule, tool, counter) + valid_rule, rule = cls.__validate_destination(valid_rule, app, return_bool, rule, tool, counter) # Arguments Verification (for rule_type arguments; read comment block at top # of function for clarification. if rule is not None: - valid_rule, rule = cls.__validate_arguments( - valid_rule, return_bool, rule, tool, counter) + valid_rule, rule = cls.__validate_arguments(valid_rule, return_bool, rule, tool, counter) if return_bool: return valid_rule @@ -394,7 +378,7 @@ class RuleValidator: """ if "fail_message" in rule: - if "destination" not in rule or rule['destination'] != "fail": + if "destination" not in rule or rule["destination"] != "fail": error = f"Found a fail_message for rule {str(counter)}" error += f" in '{str(tool)}', but destination is not 'fail'!" if not return_bool: @@ -421,22 +405,24 @@ class RuleValidator: log.debug(error) valid_rule = False else: - is_valid = validate_destination(app, rule["destination"], - dest_err_tool_rule_dest, (tool, counter, rule["destination"]), - return_bool) + is_valid = validate_destination( + app, + rule["destination"], + dest_err_tool_rule_dest, + (tool, counter, rule["destination"]), + return_bool, + ) if not is_valid: valid_rule = False elif isinstance(rule["destination"], dict): - if ("priority" in rule["destination"] - and isinstance(rule["destination"]["priority"], dict)): + if "priority" in rule["destination"] and isinstance(rule["destination"]["priority"], dict): for priority in rule["destination"]["priority"]: if priority not in priority_list: error = "Invalid priority '" error += f"{str(priority)}' for rule " error += f"{str(counter)} in '{str(tool)}'." - suggestion = get_typo_correction(priority, - priority_list, max_edit_dist) + suggestion = get_typo_correction(priority, priority_list, max_edit_dist) if suggestion: error += f" Did you mean '{str(suggestion)}'?" if not return_bool: @@ -456,11 +442,13 @@ class RuleValidator: log.debug(error) valid_rule = False else: - is_valid = validate_destination(app, + is_valid = validate_destination( + app, rule["destination"]["priority"][priority], dest_err_tool_rule_dest, (tool, counter, rule["destination"]["priority"][priority]), - return_bool) + return_bool, + ) if not is_valid: valid_rule = False else: @@ -685,9 +673,13 @@ class RuleValidator: return valid_rule, rule -def parse_yaml(path: str = "/config/tool_destinations.yml", - job_conf_path: str = "/config/job_conf.xml", app=None, test: bool = False, - return_bool: bool = False): +def parse_yaml( + path: str = "/config/tool_destinations.yml", + job_conf_path: str = "/config/job_conf.xml", + app=None, + test: bool = False, + return_bool: bool = False, +): """ Get a yaml file from path and send it to validate_config for validation. @@ -715,8 +707,8 @@ def parse_yaml(path: str = "/config/tool_destinations.yml", # os.path.realpath gets the path of DynamicToolDestination.py # and then os.path.join is used to go back four directories config_directory = os.path.join( - os.path.dirname(os.path.realpath(__file__)), os.pardir, - os.pardir, os.pardir, os.pardir) + os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, os.pardir, os.pardir + ) opt_file = config_directory + path @@ -748,8 +740,7 @@ def parse_yaml(path: str = "/config/tool_destinations.yml", return config -def validate_destination(app, destination: str, err_message: str, err_message_contents, - return_bool: bool = True): +def validate_destination(app, destination: str, err_message: str, err_message_contents, return_bool: bool = True): """ Validate received destination id. @@ -773,14 +764,15 @@ def validate_destination(app, destination: str, err_message: str, err_message_co valid_destination = False suggestion = None - if destination == 'fail' and err_message is dest_err_tool_rule_dest: # It's a tool rule that is set to fail. It's valid + if ( + destination == "fail" and err_message is dest_err_tool_rule_dest + ): # It's a tool rule that is set to fail. It's valid valid_destination = True elif app is None: if destination in destination_list: valid_destination = True else: - suggestion = get_typo_correction(destination, - destination_list, max_edit_dist) + suggestion = get_typo_correction(destination, destination_list, max_edit_dist) elif app.job_config.get_destination(destination): valid_destination = True @@ -824,8 +816,8 @@ def validate_config(obj: dict, app=None, return_bool: bool = False): if return_bool: verbose = True - elif obj is not None and 'verbose' in obj and isinstance(obj['verbose'], bool): - verbose = obj['verbose'] + elif obj is not None and "verbose" in obj and isinstance(obj["verbose"], bool): + verbose = obj["verbose"] else: valid_config = False if obj: @@ -840,57 +832,64 @@ def validate_config(obj: dict, app=None, return_bool: bool = False): log.debug("Missing mandatory field 'verbose' in config!") # a list with the available rule_types. Can be expanded on easily in the future - available_rule_types = ['file_size', 'num_input_datasets', 'records', 'arguments'] + available_rule_types = ["file_size", "num_input_datasets", "records", "arguments"] if obj is not None: # in obj, there should always be only 5 categories: tools, default_destination, # default_priority, users, and verbose - if 'default_destination' in obj: + if "default_destination" in obj: suggestion = None - if isinstance(obj['default_destination'], str): - is_valid = validate_destination(app, obj['default_destination'], - dest_err_default_dest, - (obj['default_destination'])) + if isinstance(obj["default_destination"], str): + is_valid = validate_destination( + app, obj["default_destination"], dest_err_default_dest, (obj["default_destination"]) + ) if is_valid: - new_config["default_destination"] = obj['default_destination'] + new_config["default_destination"] = obj["default_destination"] else: valid_config = False - elif isinstance(obj['default_destination'], dict): + elif isinstance(obj["default_destination"], dict): - if ('priority' in obj['default_destination'] - and isinstance(obj['default_destination']['priority'], dict)): + if "priority" in obj["default_destination"] and isinstance( + obj["default_destination"]["priority"], dict + ): - for priority in obj['default_destination']['priority']: - if isinstance(obj['default_destination']['priority'][priority], - str): + for priority in obj["default_destination"]["priority"]: + if isinstance(obj["default_destination"]["priority"][priority], str): priority_list.add(priority) is_valid = validate_destination( - app, obj['default_destination']['priority'][priority], + app, + obj["default_destination"]["priority"][priority], dest_err_default_dest, - (obj['default_destination']['priority'][priority])) + (obj["default_destination"]["priority"][priority]), + ) if is_valid: - new_config["default_destination"]['priority'][priority] = ( - obj['default_destination']['priority'][priority]) + new_config["default_destination"]["priority"][priority] = obj["default_destination"][ + "priority" + ][priority] else: valid_config = False if len(priority_list) < 1: - error = ("No valid priorities found!") + error = "No valid priorities found!" if verbose: log.debug(error) valid_config = False else: - if 'default_priority' in obj: - if isinstance(obj['default_priority'], str): - if obj['default_priority'] in priority_list: - new_config['default_priority'] = obj['default_priority'] + if "default_priority" in obj: + if isinstance(obj["default_priority"], str): + if obj["default_priority"] in priority_list: + new_config["default_priority"] = obj["default_priority"] else: - error = ("Default priority '" + str(obj['default_priority']) - + "' is not a valid priority.") - suggestion = get_typo_correction(obj['default_priority'], - priority_list, max_edit_dist) + error = ( + "Default priority '" + + str(obj["default_priority"]) + + "' is not a valid priority." + ) + suggestion = get_typo_correction( + obj["default_priority"], priority_list, max_edit_dist + ) if suggestion: error += f" Did you mean '{str(suggestion)}'?" if verbose: @@ -902,11 +901,11 @@ def validate_config(obj: dict, app=None, return_bool: bool = False): valid_config = False else: error = "No default_priority section found in config." - if 'med' in priority_list: + if "med" in priority_list: # set 'med' as fallback default priority, so # old tool_destination.yml configs still work error += " Setting 'med' as default priority." - new_config['default_priority'] = 'med' + new_config["default_priority"] = "med" else: error += " Things may not run as expected!" valid_config = False @@ -930,22 +929,26 @@ def validate_config(obj: dict, app=None, return_bool: bool = False): log.debug(error) valid_config = False - if 'users' in obj: - if isinstance(obj['users'], dict): - for user in obj['users']: - curr = obj['users'][user] + if "users" in obj: + if isinstance(obj["users"], dict): + for user in obj["users"]: + curr = obj["users"][user] if isinstance(curr, dict): - if 'priority' in curr and isinstance(curr['priority'], str): + if "priority" in curr and isinstance(curr["priority"], str): - if curr['priority'] in priority_list: - new_config['users'][user]['priority'] = curr['priority'] + if curr["priority"] in priority_list: + new_config["users"][user]["priority"] = curr["priority"] else: - error = ("User '" + user + "', priority '" - + str(curr['priority']) + "' is not defined " - + "in the global default_destination section") - suggestion = get_typo_correction(curr['priority'], - priority_list, max_edit_dist) + error = ( + "User '" + + user + + "', priority '" + + str(curr["priority"]) + + "' is not defined " + + "in the global default_destination section" + ) + suggestion = get_typo_correction(curr["priority"], priority_list, max_edit_dist) if suggestion: error += f" Did you mean '{str(suggestion)}'?" if verbose: @@ -967,9 +970,9 @@ def validate_config(obj: dict, app=None, return_bool: bool = False): log.debug(error) valid_config = False - if 'tools' in obj: - for tool in obj['tools']: - curr = obj['tools'][tool] + if "tools" in obj: + for tool in obj["tools"]: + curr = obj["tools"][tool] # This check is to make sure we have a tool name, and not just # rules right way. @@ -982,51 +985,64 @@ def validate_config(obj: dict, app=None, return_bool: bool = False): # default_destination (not mandatory) and rules (mandatory) if "default_destination" in curr: suggestion = None - if isinstance(curr['default_destination'], str): - is_valid = validate_destination(app, - curr['default_destination'], + if isinstance(curr["default_destination"], str): + is_valid = validate_destination( + app, + curr["default_destination"], dest_err_tool_default_dest, - (tool, curr['default_destination'])) + (tool, curr["default_destination"]), + ) if is_valid: - new_config['tools'][tool]['default_destination'] = ( - curr['default_destination']) + new_config["tools"][tool]["default_destination"] = curr["default_destination"] tool_has_default = True else: valid_config = False - elif isinstance(curr['default_destination'], dict): + elif isinstance(curr["default_destination"], dict): - if ('priority' in curr['default_destination'] - and isinstance(curr['default_destination']['priority'], dict)): + if "priority" in curr["default_destination"] and isinstance( + curr["default_destination"]["priority"], dict + ): - for priority in curr['default_destination']['priority']: - destination = curr['default_destination']['priority'][priority] + for priority in curr["default_destination"]["priority"]: + destination = curr["default_destination"]["priority"][priority] if priority in priority_list: if isinstance(destination, str): is_valid = validate_destination( - app, destination, + app, + destination, dest_err_tool_default_dest, - (tool, curr['default_destination']['priority'][priority])) + (tool, curr["default_destination"]["priority"][priority]), + ) if is_valid: - new_config['tools'][tool]['default_destination']['priority'][priority] = destination + new_config["tools"][tool]["default_destination"]["priority"][ + priority + ] = destination tool_has_default = True else: valid_config = False else: - error = ("No default '" + str(priority) - + "' priority destination for tool " - + str(tool) + " in config!") + error = ( + "No default '" + + str(priority) + + "' priority destination for tool " + + str(tool) + + " in config!" + ) if verbose: log.debug(error) valid_config = False else: - error = ("Invalid default destination priority '" - + str(priority) + "' for '" + str(tool) - + "'.") - suggestion = get_typo_correction(priority, - priority_list, max_edit_dist) + error = ( + "Invalid default destination priority '" + + str(priority) + + "' for '" + + str(tool) + + "'." + ) + suggestion = get_typo_correction(priority, priority_list, max_edit_dist) if suggestion: error += f" Did you mean '{str(suggestion)}'?" if verbose: @@ -1039,14 +1055,14 @@ def validate_config(obj: dict, app=None, return_bool: bool = False): log.debug(error) valid_config = False - if "rules" in curr and isinstance(curr['rules'], list): + if "rules" in curr and isinstance(curr["rules"], list): # under rules, there should only be a list of rules curr_tool = curr counter = 0 - for rule in curr_tool['rules']: + for rule in curr_tool["rules"]: if "rule_type" in rule: - if rule['rule_type'] in available_rule_types: + if rule["rule_type"] in available_rule_types: validated_rule = None counter += 1 @@ -1055,16 +1071,14 @@ def validate_config(obj: dict, app=None, return_bool: bool = False): # result if return_bool: valid_rule = RuleValidator.validate_rule( - rule['rule_type'], app, return_bool, - rule, counter, tool) + rule["rule_type"], app, return_bool, rule, counter, tool + ) # otherwise, retrieve the processed rule else: - validated_rule = ( - RuleValidator.validate_rule( - rule['rule_type'], - app, return_bool, - rule, counter, tool)) + validated_rule = RuleValidator.validate_rule( + rule["rule_type"], app, return_bool, rule, counter, tool + ) # if the result we get is False, then # indicate that the whole config is invalid @@ -1074,10 +1088,8 @@ def validate_config(obj: dict, app=None, return_bool: bool = False): # if we got a rule back that seems to be # valid (or was fixable) then append it to # list of ready-to-use tools - if (not return_bool - and validated_rule is not None): - curr_tool_rules.append( - copy.deepcopy(validated_rule)) + if not return_bool and validated_rule is not None: + curr_tool_rules.append(copy.deepcopy(validated_rule)) # if rule['rule_type'] in available_rule_types else: @@ -1116,7 +1128,7 @@ def validate_config(obj: dict, app=None, return_bool: bool = False): log.debug(error) if curr_tool_rules: - new_config['tools'][str(tool)]['rules'] = curr_tool_rules + new_config["tools"][str(tool)]["rules"] = curr_tool_rules # if not isinstance(curr, list) else: @@ -1153,7 +1165,7 @@ def validate_config(obj: dict, app=None, return_bool: bool = False): def bytes_to_str(size, unit="YB"): - ''' + """ Uses the bi convention: 1024 B = 1 KB since this method primarily has inputs of bytes for RAM @@ -1162,7 +1174,7 @@ def bytes_to_str(size, unit="YB"): @rtype: str @return return_str: the resulting string - ''' + """ # converts size in bytes to most readable unit units = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"] i = 0 @@ -1196,7 +1208,7 @@ def bytes_to_str(size, unit="YB"): def str_to_bytes(size): - ''' + """ Uses the bi convention: 1024 B = 1 KB since this method primarily has inputs of bytes for RAM @@ -1205,7 +1217,7 @@ def str_to_bytes(size): @rtype: int @return curr_size: the resulting size converted from str - ''' + """ units = ["", "b", "kb", "mb", "gb", "tb", "pb", "eb", "zb", "yb"] curr_size = size @@ -1258,19 +1270,20 @@ def importer(test): global JobDestination global JobMappingException if test: + class JobDestination: def __init__(self, *kwd): - self.id = kwd.get('id') - self.nativeSpec = kwd.get('params')['nativeSpecification'] - self.runner = kwd.get('runner') + self.id = kwd.get("id") + self.nativeSpec = kwd.get("params")["nativeSpecification"] + self.runner = kwd.get("runner") + from galaxy.jobs.mapper import JobMappingException else: from galaxy.jobs import JobDestination from galaxy.jobs.mapper import JobMappingException -def map_tool_to_destination( - job, app, tool, user_email, test=False, path=None, job_conf_path=None): +def map_tool_to_destination(job, app, tool, user_email, test=False, path=None, job_conf_path=None): """ Dynamically allocate resources @@ -1312,9 +1325,9 @@ def map_tool_to_destination( inp_data = {da.name: da.dataset for da in job.input_datasets} inp_data.update([(da.name, da.dataset) for da in job.input_library_datasets]) - if config is not None and str(tool.old_id) in config['tools']: - if 'rules' in config['tools'][str(tool.old_id)]: - for rule in config['tools'][str(tool.old_id)]['rules']: + if config is not None and str(tool.old_id) in config["tools"]: + if "rules" in config["tools"][str(tool.old_id)]: + for rule in config["tools"][str(tool.old_id)]["rules"]: if rule["rule_type"] == "file_size": filesize_rule_present = True @@ -1392,18 +1405,21 @@ def map_tool_to_destination( # Get the default priority from the config if necessary. # If there isn't one, choose an arbitrary one as a fallback if "default_destination" in config: - if isinstance(config['default_destination'], dict): - if 'default_priority' in config: - default_priority = config['default_priority'] + if isinstance(config["default_destination"], dict): + if "default_priority" in config: + default_priority = config["default_priority"] priority = default_priority else: if len(priority_list) > 0: default_priority = next(iter(priority_list)) priority = default_priority - error = ("No default priority found, arbitrarily setting '" - + default_priority + "' as the default priority." - + " Things may not work as expected!") + error = ( + "No default priority found, arbitrarily setting '" + + default_priority + + "' as the default priority." + + " Things may not work as expected!" + ) if verbose: log.debug(error) @@ -1421,17 +1437,17 @@ def map_tool_to_destination( # Priority coming from workflow invocation takes precedence over job specific priorities if workflow_params is not None: resource_params = json.loads(workflow_params) - if 'priority' in resource_params: + if "priority" in resource_params: # For by_group mapping, this priority has already been validated when the # request was created. - if resource_params['priority'] is not None: - priority = resource_params['priority'] + if resource_params["priority"] is not None: + priority = resource_params["priority"] elif job_params is not None: resource_params = json.loads(job_params) - if 'priority' in resource_params: - if resource_params['priority'] is not None: - priority = resource_params['priority'] + if "priority" in resource_params: + if resource_params["priority"] is not None: + priority = resource_params["priority"] # get the user's priority if "users" in config: @@ -1439,21 +1455,21 @@ def map_tool_to_destination( priority = config["users"][user_email]["priority"] if "default_destination" in config: - if isinstance(config['default_destination'], str): - destination = config['default_destination'] + if isinstance(config["default_destination"], str): + destination = config["default_destination"] else: - if priority in config['default_destination']['priority']: - destination = config['default_destination']['priority'][priority] - elif default_priority in config['default_destination']['priority']: - destination = (config['default_destination']['priority'][default_priority]) - config = config['tools'] + if priority in config["default_destination"]["priority"]: + destination = config["default_destination"]["priority"][priority] + elif default_priority in config["default_destination"]["priority"]: + destination = config["default_destination"]["priority"][default_priority] + config = config["tools"] if str(tool.old_id) in config: - if 'rules' in config[str(tool.old_id)]: - for rule in config[str(tool.old_id)]['rules']: + if "rules" in config[str(tool.old_id)]: + for rule in config[str(tool.old_id)]["rules"]: rule_counter += 1 user_authorized = False - if 'users' in rule and isinstance(rule['users'], list): - if user_email in rule['users']: + if "users" in rule and isinstance(rule["users"], list): + if user_email in rule["users"]: user_authorized = True else: user_authorized = True @@ -1471,7 +1487,7 @@ def map_tool_to_destination( matched = True else: - if (lower_bound <= file_size and file_size < upper_bound): + if lower_bound <= file_size and file_size < upper_bound: matched = True elif rule["rule_type"] == "num_input_datasets": @@ -1484,7 +1500,7 @@ def map_tool_to_destination( if lower_bound <= num_input_datasets: matched = True else: - if (lower_bound <= num_input_datasets and num_input_datasets < upper_bound): + if lower_bound <= num_input_datasets and num_input_datasets < upper_bound: matched = True elif rule["rule_type"] == "records": @@ -1512,7 +1528,7 @@ def map_tool_to_destination( try: options_value = reduce(dict.__getitem__, arg_keys_list, options) arg_value = reduce(dict.__getitem__, arg_keys_list, arg_dict) - if (arg_value != options_value): + if arg_value != options_value: matched = False except KeyError: matched = False @@ -1523,8 +1539,7 @@ def map_tool_to_destination( # if we matched a rule if matched: - if (matched_rule is None or rule["nice_value"] - < matched_rule["nice_value"]): + if matched_rule is None or rule["nice_value"] < matched_rule["nice_value"]: matched_rule = rule # if user_authorized else: @@ -1544,14 +1559,14 @@ def map_tool_to_destination( if matched_rule is None: if "default_destination" in config[str(tool.old_id)]: - default_tool_destination = (config[str(tool.old_id)]['default_destination']) + default_tool_destination = config[str(tool.old_id)]["default_destination"] if isinstance(default_tool_destination, str): destination = default_tool_destination else: - if priority in default_tool_destination['priority']: - destination = default_tool_destination['priority'][priority] - elif default_priority in default_tool_destination['priority']: - destination = (default_tool_destination['priority'][default_priority]) + if priority in default_tool_destination["priority"]: + destination = default_tool_destination["priority"][priority] + elif default_priority in default_tool_destination["priority"]: + destination = default_tool_destination["priority"][default_priority] # else global default destination is used else: if isinstance(matched_rule["destination"], str): @@ -1560,7 +1575,7 @@ def map_tool_to_destination( if priority in matched_rule["destination"]["priority"]: destination = matched_rule["destination"]["priority"][priority] elif default_priority in matched_rule["destination"]["priority"]: - destination = (matched_rule["destination"]["priority"][default_priority]) + destination = matched_rule["destination"]["priority"][default_priority] # else global default destination is used # if "default_destination" in config @@ -1615,11 +1630,10 @@ def get_destination_list_from_job_config(job_config_location) -> set: # os.path.realpath gets the path of DynamicToolDestination.py # and then os.path.join is used to go back four directories - config_location = os.path.join( - os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, os.pardir) + config_location = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, os.pardir) if job_config_location: - local_path = re.compile('^/config/.+$') + local_path = re.compile("^/config/.+$") if local_path.match(job_config_location): job_config_location = os.path.join(config_location, job_config_location) else: # Pick one of the default ones @@ -1636,8 +1650,7 @@ def get_destination_list_from_job_config(job_config_location) -> set: message += f"using '{f}'. *" break else: - message += ("and no default job configs in 'config/'. " - + "Expect lots of failures. *") + message += "and no default job configs in 'config/'. " + "Expect lots of failures. *" if verbose: log.debug(message) @@ -1698,14 +1711,10 @@ def get_edit_distance(source, target): # Substitution or matching: # Target and source items are aligned, and either # are different (cost of 1), or are the same (cost of 0). - current_row[1:] = np.minimum( - current_row[1:], - np.add(previous_row[:-1], target != s)) + current_row[1:] = np.minimum(current_row[1:], np.add(previous_row[:-1], target != s)) # Deletion (target grows shorter than source): - current_row[1:] = np.minimum( - current_row[1:], - current_row[0:-1] + 1) + current_row[1:] = np.minimum(current_row[1:], current_row[0:-1] + 1) previous_row = current_row @@ -1757,7 +1766,7 @@ def get_typo_correction(typo_str, word_set, max_dist): return suggestion -if __name__ == '__main__': +if __name__ == "__main__": """ This function is responsible for running the app if directly run through the commandline. It offers the ability to specify a config through the @@ -1773,18 +1782,20 @@ if __name__ == '__main__': logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) parser.add_argument( - '-c', '--check-config', dest='check_config', nargs='?', - help='Use this option to validate tool_destinations.yml.' - + ' Optionally, provide the path to the tool_destinations.yml' - + ' that you would like to check, and/or the path to the related' - + ' job_conf.xml. Default: galaxy/config/tool_destinations.yml' - + 'and galaxy/config/job_conf.xml') + "-c", + "--check-config", + dest="check_config", + nargs="?", + help="Use this option to validate tool_destinations.yml." + + " Optionally, provide the path to the tool_destinations.yml" + + " that you would like to check, and/or the path to the related" + + " job_conf.xml. Default: galaxy/config/tool_destinations.yml" + + "and galaxy/config/job_conf.xml", + ) - parser.add_argument( - '-j', '--job-config', dest='job_config') + parser.add_argument("-j", "--job-config", dest="job_config") - parser.add_argument( - '-V', '--version', action='version', version=f"%(prog)s {__version__}") + parser.add_argument("-V", "--version", action="version", version=f"%(prog)s {__version__}") args = parser.parse_args() @@ -1796,13 +1807,11 @@ if __name__ == '__main__': job_config_location = args.job_config if args.check_config: - valid_config = parse_yaml(path=args.check_config, - job_conf_path=job_config_location, - return_bool=True) + valid_config = parse_yaml(path=args.check_config, job_conf_path=job_config_location, return_bool=True) else: - valid_config = parse_yaml(path="/config/tool_destinations.yml", - job_conf_path=job_config_location, - return_bool=True) + valid_config = parse_yaml( + path="/config/tool_destinations.yml", job_conf_path=job_config_location, return_bool=True + ) if valid_config: print("Configuration is valid!") diff --git a/lib/galaxy/jobs/handler.py b/lib/galaxy/jobs/handler.py index 0085a74cfa5..58aaecd5ea2 100644 --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -9,7 +9,11 @@ from queue import ( Empty, Queue, ) -from typing import Dict, List, Tuple +from typing import ( + Dict, + List, + Tuple, +) from sqlalchemy.exc import OperationalError from sqlalchemy.sql.expression import ( @@ -18,7 +22,7 @@ from sqlalchemy.sql.expression import ( null, or_, select, - true + true, ) from galaxy import model @@ -26,7 +30,7 @@ from galaxy.exceptions import ObjectNotFound from galaxy.jobs import ( JobDestination, JobWrapper, - TaskWrapper + TaskWrapper, ) from galaxy.jobs.mapper import JobNotReadyException from galaxy.structured_app import MinimalManagerApp @@ -39,12 +43,31 @@ from galaxy.web_stack.message import JobHandlerMessage log = get_logger(__name__) # States for running a job. These are NOT the same as data states -JOB_WAIT, JOB_ERROR, JOB_INPUT_ERROR, JOB_INPUT_DELETED, JOB_READY, JOB_DELETED, JOB_ADMIN_DELETED, JOB_USER_OVER_QUOTA, JOB_USER_OVER_TOTAL_WALLTIME = 'wait', 'error', 'input_error', 'input_deleted', 'ready', 'deleted', 'admin_deleted', 'user_over_quota', 'user_over_total_walltime' -DEFAULT_JOB_PUT_FAILURE_MESSAGE = 'Unable to run job due to a misconfiguration of the Galaxy job running system. Please contact a site administrator.' +( + JOB_WAIT, + JOB_ERROR, + JOB_INPUT_ERROR, + JOB_INPUT_DELETED, + JOB_READY, + JOB_DELETED, + JOB_ADMIN_DELETED, + JOB_USER_OVER_QUOTA, + JOB_USER_OVER_TOTAL_WALLTIME, +) = ( + "wait", + "error", + "input_error", + "input_deleted", + "ready", + "deleted", + "admin_deleted", + "user_over_quota", + "user_over_total_walltime", +) +DEFAULT_JOB_PUT_FAILURE_MESSAGE = "Unable to run job due to a misconfiguration of the Galaxy job running system. Please contact a site administrator." class JobHandlerI: - def start(self): pass @@ -76,8 +99,15 @@ class JobHandler(JobHandlerI): class ItemGrabber: - - def __init__(self, app, grab_type='Job', handler_assignment_method=None, max_grab=None, self_handler_tags=None, handler_tags=None): + def __init__( + self, + app, + grab_type="Job", + handler_assignment_method=None, + max_grab=None, + self_handler_tags=None, + handler_tags=None, + ): self.app = app self.sa_session = app.model.context self.grab_this = getattr(model, grab_type) @@ -86,30 +116,39 @@ class ItemGrabber: self.self_handler_tags = self_handler_tags self.max_grab = max_grab self.handler_tags = handler_tags - self._grab_conn_opts = {'autocommit': False} + self._grab_conn_opts = {"autocommit": False} self._grab_query = None self._supports_returning = self.app.application_stack.supports_returning() def setup_query(self): - subq = select([self.grab_this.id]) \ - .where(and_( - self.grab_this.table.c.handler.in_(self.self_handler_tags), - self.grab_this.table.c.state == self.grab_this.states.NEW)) \ + subq = ( + select([self.grab_this.id]) + .where( + and_( + self.grab_this.table.c.handler.in_(self.self_handler_tags), + self.grab_this.table.c.state == self.grab_this.states.NEW, + ) + ) .order_by(self.grab_this.table.c.id) + ) if self.max_grab: subq = subq.limit(self.max_grab) if self.handler_assignment_method == HANDLER_ASSIGNMENT_METHODS.DB_SKIP_LOCKED: subq = subq.with_for_update(skip_locked=True) - self._grab_query = self.grab_this.table.update() \ - .where(self.grab_this.table.c.id.in_(subq)) \ + self._grab_query = ( + self.grab_this.table.update() + .where(self.grab_this.table.c.id.in_(subq)) .values(handler=self.app.config.server_name) + ) if self._supports_returning: self._grab_query = self._grab_query.returning(self.grab_this.table.c.id) if self.handler_assignment_method == HANDLER_ASSIGNMENT_METHODS.DB_TRANSACTION_ISOLATION: - self._grab_conn_opts['isolation_level'] = 'SERIALIZABLE' + self._grab_conn_opts["isolation_level"] = "SERIALIZABLE" log.info( - "Handler job grabber initialized with '%s' assignment method for handler '%s', tag(s): %s", self.handler_assignment_method, - self.app.config.server_name, ', '.join(str(x) for x in self.handler_tags) + "Handler job grabber initialized with '%s' assignment method for handler '%s', tag(s): %s", + self.handler_assignment_method, + self.app.config.server_name, + ", ".join(str(x) for x in self.handler_tags), ) @staticmethod @@ -151,8 +190,8 @@ class ItemGrabber: except OperationalError as e: # If this is a serialization failure on PostgreSQL, then e.orig is a psycopg2 TransactionRollbackError # and should have attribute `code`. Other engines should just report the message and move on. - if int(getattr(e.orig, 'pgcode', -1)) != 40001: - log.debug('Grabbing %s failed (serialization failures are ok): %s', self.grab_type, unicodify(e)) + if int(getattr(e.orig, "pgcode", -1)) != 40001: + log.debug("Grabbing %s failed (serialization failures are ok): %s", self.grab_type, unicodify(e)) trans.rollback() @@ -161,6 +200,7 @@ class JobHandlerQueue(Monitors): Job Handler's Internal Queue, this is what actually implements waiting for jobs to be runnable and dispatching to a JobRunner. """ + STOP_SIGNAL = object() def __init__(self, app: MinimalManagerApp, dispatcher): @@ -186,11 +226,13 @@ class JobHandlerQueue(Monitors): name = "JobHandlerQueue.monitor_thread" self._init_monitor_thread(name, target=self.__monitor, config=app.config) self.job_grabber = None - handler_assignment_method = ItemGrabber.get_grabbable_handler_assignment_method(self.app.job_config.handler_assignment_methods) + handler_assignment_method = ItemGrabber.get_grabbable_handler_assignment_method( + self.app.job_config.handler_assignment_methods + ) if handler_assignment_method: self.job_grabber = ItemGrabber( app=app, - grab_type='Job', + grab_type="Job", handler_assignment_method=handler_assignment_method, max_grab=self.app.job_config.handler_max_grab, self_handler_tags=self.app.job_config.self_handler_tags, @@ -201,13 +243,13 @@ class JobHandlerQueue(Monitors): """ Starts the JobHandler's thread after checking for any unhandled jobs. """ - log.debug('Handler queue starting for jobs assigned to handler: %s', self.app.config.server_name) + log.debug("Handler queue starting for jobs assigned to handler: %s", self.app.config.server_name) # Recover jobs at startup self.__check_jobs_at_startup() # Start the queue self.monitor_thread.start() # The stack code is initialized in the application - JobHandlerMessage().bind_default_handler(self, '_handle_message') + JobHandlerMessage().bind_default_handler(self, "_handle_message") self.app.application_stack.register_message_handler(self._handle_message, name=JobHandlerMessage.target) log.info("job handler queue started") @@ -227,48 +269,63 @@ class JobHandlerQueue(Monitors): """ jobs_at_startup = [] if self.track_jobs_in_database: - in_list = (model.Job.states.QUEUED, - model.Job.states.RUNNING, - model.Job.states.STOPPED) + in_list = (model.Job.states.QUEUED, model.Job.states.RUNNING, model.Job.states.STOPPED) else: - in_list = (model.Job.states.NEW, - model.Job.states.QUEUED, - model.Job.states.RUNNING) + in_list = (model.Job.states.NEW, model.Job.states.QUEUED, model.Job.states.RUNNING) if self.app.config.user_activation_on: - jobs_at_startup = self.sa_session.query(model.Job).enable_eagerloads(False) \ - .outerjoin(model.User) \ - .filter(model.Job.state.in_(in_list) - & (model.Job.handler == self.app.config.server_name) - & or_((model.Job.user_id == null()), (model.User.active == true()))).yield_per(model.YIELD_PER_ROWS) + jobs_at_startup = ( + self.sa_session.query(model.Job) + .enable_eagerloads(False) + .outerjoin(model.User) + .filter( + model.Job.state.in_(in_list) + & (model.Job.handler == self.app.config.server_name) + & or_((model.Job.user_id == null()), (model.User.active == true())) + ) + .yield_per(model.YIELD_PER_ROWS) + ) else: - jobs_at_startup = self.sa_session.query(model.Job).enable_eagerloads(False) \ - .filter(model.Job.state.in_(in_list) - & (model.Job.handler == self.app.config.server_name)).yield_per(model.YIELD_PER_ROWS) + jobs_at_startup = ( + self.sa_session.query(model.Job) + .enable_eagerloads(False) + .filter(model.Job.state.in_(in_list) & (model.Job.handler == self.app.config.server_name)) + .yield_per(model.YIELD_PER_ROWS) + ) for job in jobs_at_startup: if not self.app.toolbox.has_tool(job.tool_id, job.tool_version, exact=True): log.warning(f"({job.id}) Tool '{job.tool_id}' removed from tool config, unable to recover job") - self.job_wrapper(job).fail('This tool was disabled before the job completed. Please contact your Galaxy administrator.') + self.job_wrapper(job).fail( + "This tool was disabled before the job completed. Please contact your Galaxy administrator." + ) elif job.job_runner_name is not None and job.job_runner_external_id is None: # This could happen during certain revisions of Galaxy where a runner URL was persisted before the job was dispatched to a runner. - log.debug(f"({job.id}) Job runner assigned but no external ID recorded, adding to the job handler queue") + log.debug( + f"({job.id}) Job runner assigned but no external ID recorded, adding to the job handler queue" + ) job.job_runner_name = None if self.track_jobs_in_database: job.set_state(model.Job.states.NEW) else: self.queue.put((job.id, job.tool_id)) - elif job.job_runner_name is not None and job.job_runner_external_id is not None and job.destination_id is None: + elif ( + job.job_runner_name is not None + and job.job_runner_external_id is not None + and job.destination_id is None + ): # This is the first start after upgrading from URLs to destinations, convert the URL to a destination and persist job_wrapper = self.job_wrapper(job) job_destination = self.dispatcher.url_to_destination(job.job_runner_name) if job_destination.id is None: - job_destination.id = 'legacy_url' + job_destination.id = "legacy_url" job_wrapper.set_job_destination(job_destination, job.job_runner_external_id) self.dispatcher.recover(job, job_wrapper) - log.info(f'({job.id}) Converted job from a URL to a destination and recovered') + log.info(f"({job.id}) Converted job from a URL to a destination and recovered") elif job.job_runner_name is None: # Never (fully) dispatched - log.debug(f"({job.id}) No job runner assigned and job still in '{job.state}' state, adding to the job handler queue") + log.debug( + f"({job.id}) No job runner assigned and job still in '{job.state}' state, adding to the job handler queue" + ) if self.track_jobs_in_database: job.set_state(model.Job.states.NEW) else: @@ -285,14 +342,20 @@ class JobHandlerQueue(Monitors): job_wrapper = self.job_wrapper(job) # Use the persisted destination as its params may differ from # what's in the job_conf xml - job_destination = JobDestination(id=job.destination_id, runner=job.job_runner_name, params=job.destination_params) + job_destination = JobDestination( + id=job.destination_id, runner=job.job_runner_name, params=job.destination_params + ) # resubmits are not persisted (it's a good thing) so they # should be added back to the in-memory destination on startup try: config_job_destination = self.app.job_config.get_destination(job.destination_id) job_destination.resubmit = config_job_destination.resubmit except KeyError: - log.debug('(%s) Recovered destination id (%s) does not exist in job config (but this may be normal in the case of a dynamically generated destination)', job.id, job.destination_id) + log.debug( + "(%s) Recovered destination id (%s) does not exist in job config (but this may be normal in the case of a dynamically generated destination)", + job.id, + job.destination_id, + ) job_wrapper.job_runner_mapper.cached_job_destination = job_destination return job_wrapper @@ -319,8 +382,7 @@ class JobHandlerQueue(Monitors): Called repeatedly by `monitor` to process waiting jobs. """ monitor_step_timer = self.app.execution_timer_factory.get_timer( - 'internal.galaxy.jobs.handlers.monitor_step', - 'Job handler monitor step complete.' + "internal.galaxy.jobs.handlers.monitor_step", "Job handler monitor step complete." ) if self.job_grabber is not None: self.job_grabber.grab_unhandled_items() @@ -341,50 +403,79 @@ class JobHandlerQueue(Monitors): # Clear the session so we get fresh states for job and all datasets self.sa_session.expunge_all() # Fetch all new jobs - hda_not_ready = self.sa_session.query(model.Job.id).enable_eagerloads(False) \ - .join(model.JobToInputDatasetAssociation) \ - .join(model.HistoryDatasetAssociation) \ - .join(model.Dataset) \ - .filter(and_(model.Job.state == model.Job.states.NEW, - model.Dataset.state.in_(model.Dataset.non_ready_states))).subquery() - ldda_not_ready = self.sa_session.query(model.Job.id).enable_eagerloads(False) \ - .join(model.JobToInputLibraryDatasetAssociation) \ - .join(model.LibraryDatasetDatasetAssociation) \ - .join(model.Dataset) \ - .filter(and_(model.Job.state == model.Job.states.NEW, - model.Dataset.state.in_(model.Dataset.non_ready_states))).subquery() - rank = func.rank().over(partition_by=model.Job.table.c.user_id, - order_by=model.Job.table.c.id).label('rank') + hda_not_ready = ( + self.sa_session.query(model.Job.id) + .enable_eagerloads(False) + .join(model.JobToInputDatasetAssociation) + .join(model.HistoryDatasetAssociation) + .join(model.Dataset) + .filter( + and_( + model.Job.state == model.Job.states.NEW, model.Dataset.state.in_(model.Dataset.non_ready_states) + ) + ) + .subquery() + ) + ldda_not_ready = ( + self.sa_session.query(model.Job.id) + .enable_eagerloads(False) + .join(model.JobToInputLibraryDatasetAssociation) + .join(model.LibraryDatasetDatasetAssociation) + .join(model.Dataset) + .filter( + and_( + model.Job.state == model.Job.states.NEW, model.Dataset.state.in_(model.Dataset.non_ready_states) + ) + ) + .subquery() + ) + rank = func.rank().over(partition_by=model.Job.table.c.user_id, order_by=model.Job.table.c.id).label("rank") job_filter_conditions = ( (model.Job.state == model.Job.states.NEW), (model.Job.handler == self.app.config.server_name), ~model.Job.table.c.id.in_(select(hda_not_ready)), - ~model.Job.table.c.id.in_(select(ldda_not_ready))) + ~model.Job.table.c.id.in_(select(ldda_not_ready)), + ) if self.app.config.user_activation_on: job_filter_conditions = job_filter_conditions + ( - or_((model.Job.user_id == null()), (model.User.active == true())),) - if self.sa_session.bind.name == 'sqlite': + or_((model.Job.user_id == null()), (model.User.active == true())), + ) + if self.sa_session.bind.name == "sqlite": query_objects = (model.Job,) else: query_objects = (model.Job, rank) - ready_query = self.sa_session.query(*query_objects).enable_eagerloads(False) \ - .outerjoin(model.User) \ - .filter(and_(*job_filter_conditions)) \ + ready_query = ( + self.sa_session.query(*query_objects) + .enable_eagerloads(False) + .outerjoin(model.User) + .filter(and_(*job_filter_conditions)) .order_by(model.Job.id) - if self.sa_session.bind.name == 'sqlite': + ) + if self.sa_session.bind.name == "sqlite": jobs_to_check = ready_query.all() else: ranked = ready_query.subquery() - jobs_to_check = self.sa_session.query(model.Job) \ - .join(ranked, model.Job.id == ranked.c.id) \ - .filter(ranked.c.rank <= self.app.job_config.handler_ready_window_size).all() + jobs_to_check = ( + self.sa_session.query(model.Job) + .join(ranked, model.Job.id == ranked.c.id) + .filter(ranked.c.rank <= self.app.job_config.handler_ready_window_size) + .all() + ) # Filter jobs with invalid input states jobs_to_check = self.__filter_jobs_with_invalid_input_states(jobs_to_check) # Fetch all "resubmit" jobs - resubmit_jobs = self.sa_session.query(model.Job).enable_eagerloads(False) \ - .filter(and_((model.Job.state == model.Job.states.RESUBMITTED), - (model.Job.handler == self.app.config.server_name))) \ - .order_by(model.Job.id).all() + resubmit_jobs = ( + self.sa_session.query(model.Job) + .enable_eagerloads(False) + .filter( + and_( + (model.Job.state == model.Job.states.RESUBMITTED), + (model.Job.handler == self.app.config.server_name), + ) + ) + .order_by(model.Job.id) + .all() + ) else: # Get job objects and append to watch queue for any which were # previously waiting @@ -405,7 +496,7 @@ class JobHandlerQueue(Monitors): self.__clear_job_count() # Check resubmit jobs first so that limits of new jobs will still be enforced for job in resubmit_jobs: - log.debug('(%s) Job was resubmitted and is being dispatched immediately', job.id) + log.debug("(%s) Job was resubmitted and is being dispatched immediately", job.id) # Reassemble resubmit job destination from persisted value jw = self.__recover_job_wrapper(job) if jw.is_ready_for_resubmission(job): @@ -449,8 +540,7 @@ class JobHandlerQueue(Monitors): log.info("(%d) Job deleted by user while still queued" % job.id) elif job_state == JOB_ADMIN_DELETED: log.info("(%d) Job deleted by admin while still queued" % job.id) - elif job_state in (JOB_USER_OVER_QUOTA, - JOB_USER_OVER_TOTAL_WALLTIME): + elif job_state in (JOB_USER_OVER_QUOTA, JOB_USER_OVER_TOTAL_WALLTIME): if job_state == JOB_USER_OVER_QUOTA: log.info("(%d) User (%s) is over quota: job paused" % (job.id, job.user_id)) what = "your disk quota" @@ -489,25 +579,34 @@ class JobHandlerQueue(Monitors): """ job_ids_to_check = [j.id for j in jobs] queries = [] - for job_to_input, input_association in [(model.JobToInputDatasetAssociation, model.HistoryDatasetAssociation), - (model.JobToInputLibraryDatasetAssociation, model.LibraryDatasetDatasetAssociation)]: - q = self.sa_session.query( - model.Job.id, - input_association.deleted, - input_association._state, - input_association.name, - model.Dataset.deleted, - model.Dataset.purged, - model.Dataset.state, - ).join(job_to_input.job) \ - .join(input_association) \ - .join(model.Dataset) \ - .filter(model.Job.id.in_(job_ids_to_check)) \ - .filter(or_(model.Dataset.deleted == true(), - model.Dataset.state != model.Dataset.states.OK, - input_association.deleted == true(), - input_association._state == input_association.states.FAILED_METADATA - )).all() + for job_to_input, input_association in [ + (model.JobToInputDatasetAssociation, model.HistoryDatasetAssociation), + (model.JobToInputLibraryDatasetAssociation, model.LibraryDatasetDatasetAssociation), + ]: + q = ( + self.sa_session.query( + model.Job.id, + input_association.deleted, + input_association._state, + input_association.name, + model.Dataset.deleted, + model.Dataset.purged, + model.Dataset.state, + ) + .join(job_to_input.job) + .join(input_association) + .join(model.Dataset) + .filter(model.Job.id.in_(job_ids_to_check)) + .filter( + or_( + model.Dataset.deleted == true(), + model.Dataset.state != model.Dataset.states.OK, + input_association.deleted == true(), + input_association._state == input_association.states.FAILED_METADATA, + ) + ) + .all() + ) queries.extend(q) jobs_to_pause = defaultdict(list) jobs_to_fail = defaultdict(list) @@ -586,14 +685,16 @@ class JobHandlerQueue(Monitors): return state def __verify_job_ready(self, job, job_wrapper): - """ Compute job destination and verify job is ready at that + """Compute job destination and verify job is ready at that destination by checking job limits and quota. If this method return a job state of JOB_READY - it MUST also return a job destination. """ job_destination = None try: - assert job_wrapper.tool is not None, 'This tool was disabled before the job completed. Please contact your Galaxy administrator.' + assert ( + job_wrapper.tool is not None + ), "This tool was disabled before the job completed. Please contact your Galaxy administrator." # Cause the job_destination to be set and cached by the mapper job_destination = job_wrapper.job_destination except AssertionError as e: @@ -604,9 +705,9 @@ class JobHandlerQueue(Monitors): job_state = e.job_state or JOB_WAIT return job_state, None except Exception as e: - failure_message = getattr(e, 'failure_message', DEFAULT_JOB_PUT_FAILURE_MESSAGE) + failure_message = getattr(e, "failure_message", DEFAULT_JOB_PUT_FAILURE_MESSAGE) if failure_message == DEFAULT_JOB_PUT_FAILURE_MESSAGE: - log.exception('Failed to generate job destination') + log.exception("Failed to generate job destination") else: log.debug(f"Intentionally failing job with message ({failure_message})") job_wrapper.fail(failure_message) @@ -620,10 +721,11 @@ class JobHandlerQueue(Monitors): if state == JOB_READY and self.app.quota_agent.is_over_quota(self.app, job, job_destination): return JOB_USER_OVER_QUOTA, job_destination # Check total walltime limits - if (state == JOB_READY and "delta" in self.app.job_config.limits.total_walltime): + if state == JOB_READY and "delta" in self.app.job_config.limits.total_walltime: jobs_to_check = self.sa_session.query(model.Job).filter( - model.Job.update_time >= datetime.datetime.now() - datetime.timedelta(self.app.job_config.limits.total_walltime["window"]), - model.Job.state == 'ok' + model.Job.update_time + >= datetime.datetime.now() - datetime.timedelta(self.app.job_config.limits.total_walltime["window"]), + model.Job.state == "ok", ) if job.user_id: jobs_to_check = jobs_to_check.filter(model.Job.user_id == job.user_id) @@ -634,9 +736,7 @@ class JobHandlerQueue(Monitors): # History is job.state_history started = None finished = None - for history in sorted( - job.state_history, - key=lambda history: history.update_time): + for history in sorted(job.state_history, key=lambda history: history.update_time): if history.state == "running": started = history.create_time elif history.state == "ok": @@ -645,8 +745,12 @@ class JobHandlerQueue(Monitors): if started is not None and finished is not None: time_spent += finished - started else: - log.warning("Unable to calculate time spent for job %s; started: %s, finished: %s", - job.id, started, finished) + log.warning( + "Unable to calculate time spent for job %s; started: %s, finished: %s", + job.id, + started, + finished, + ) if time_spent > self.app.job_config.limits.total_walltime["delta"]: return JOB_USER_OVER_TOTAL_WALLTIME, job_destination @@ -654,7 +758,7 @@ class JobHandlerQueue(Monitors): return state, job_destination def __verify_in_memory_job_inputs(self, job): - """ Perform the same checks that happen via SQL for in-memory managed + """Perform the same checks that happen via SQL for in-memory managed jobs. """ if job.state == model.Job.states.DELETED: @@ -667,16 +771,24 @@ class JobHandlerQueue(Monitors): continue # don't run jobs for which the input dataset was deleted if idata.deleted: - self.job_wrappers.pop(job.id, self.job_wrapper(job)).fail(f"input data {idata.hid} (file: {idata.file_name}) was deleted before the job started") + self.job_wrappers.pop(job.id, self.job_wrapper(job)).fail( + f"input data {idata.hid} (file: {idata.file_name}) was deleted before the job started" + ) return JOB_INPUT_DELETED # an error in the input data causes us to bail immediately elif idata.state == idata.states.ERROR: self.job_wrappers.pop(job.id, self.job_wrapper(job)).fail(f"input data {idata.hid} is in error state") return JOB_INPUT_ERROR elif idata.state == idata.states.FAILED_METADATA: - self.job_wrappers.pop(job.id, self.job_wrapper(job)).fail(f"input data {idata.hid} failed to properly set metadata") + self.job_wrappers.pop(job.id, self.job_wrapper(job)).fail( + f"input data {idata.hid} failed to properly set metadata" + ) return JOB_INPUT_ERROR - elif idata.state != idata.states.OK and not (idata.state == idata.states.SETTING_METADATA and job.tool_id is not None and job.tool_id == self.app.datatypes_registry.set_external_metadata_tool.id): + elif idata.state != idata.states.OK and not ( + idata.state == idata.states.SETTING_METADATA + and job.tool_id is not None + and job.tool_id == self.app.datatypes_registry.set_external_metadata_tool.id + ): # need to requeue return JOB_WAIT @@ -693,11 +805,16 @@ class JobHandlerQueue(Monitors): # This could have been incremented by a previous job dispatched on this iteration, even if we're not caching rval = self.user_job_count.get(user_id, 0) if not self.app.config.cache_user_job_count: - result = self.sa_session.execute(select([func.count(model.Job.table.c.id)]) - .where(and_(model.Job.table.c.state.in_((model.Job.states.QUEUED, - model.Job.states.RUNNING, - model.Job.states.RESUBMITTED)), - (model.Job.table.c.user_id == user_id)))) + result = self.sa_session.execute( + select([func.count(model.Job.table.c.id)]).where( + and_( + model.Job.table.c.state.in_( + (model.Job.states.QUEUED, model.Job.states.RUNNING, model.Job.states.RESUBMITTED) + ), + (model.Job.table.c.user_id == user_id), + ) + ) + ) for row in result: # there should only be one row rval += row[0] @@ -707,12 +824,18 @@ class JobHandlerQueue(Monitors): # Cache the job count if necessary if self.user_job_count is None and self.app.config.cache_user_job_count: self.user_job_count = {} - query = self.sa_session.execute(select([model.Job.table.c.user_id, func.count(model.Job.table.c.user_id)]) - .where(and_(model.Job.table.c.state.in_((model.Job.states.QUEUED, - model.Job.states.RUNNING, - model.Job.states.RESUBMITTED)), - (model.Job.table.c.user_id != null()))) - .group_by(model.Job.table.c.user_id)) + query = self.sa_session.execute( + select([model.Job.table.c.user_id, func.count(model.Job.table.c.user_id)]) + .where( + and_( + model.Job.table.c.state.in_( + (model.Job.states.QUEUED, model.Job.states.RUNNING, model.Job.states.RESUBMITTED) + ), + (model.Job.table.c.user_id != null()), + ) + ) + .group_by(model.Job.table.c.user_id) + ) for row in query: self.user_job_count[row[0]] = row[1] elif self.user_job_count is None: @@ -730,32 +853,51 @@ class JobHandlerQueue(Monitors): # queue. rval = {} rval.update(cached) - result = self.sa_session.execute(select([model.Job.table.c.destination_id, func.count(model.Job.table.c.destination_id).label('job_count')]) - .where(and_(model.Job.table.c.state.in_((model.Job.states.QUEUED, model.Job.states.RUNNING)), (model.Job.table.c.user_id == user_id))) - .group_by(model.Job.table.c.destination_id)) + result = self.sa_session.execute( + select( + [model.Job.table.c.destination_id, func.count(model.Job.table.c.destination_id).label("job_count")] + ) + .where( + and_( + model.Job.table.c.state.in_((model.Job.states.QUEUED, model.Job.states.RUNNING)), + (model.Job.table.c.user_id == user_id), + ) + ) + .group_by(model.Job.table.c.destination_id) + ) for row in result: # Add the count from the database to the cached count - rval[row['destination_id']] = rval.get(row['destination_id'], 0) + row['job_count'] + rval[row["destination_id"]] = rval.get(row["destination_id"], 0) + row["job_count"] return rval def __cache_user_job_count_per_destination(self): # Cache the job count if necessary if self.user_job_count_per_destination is None and self.app.config.cache_user_job_count: self.user_job_count_per_destination = {} - result = self.sa_session.execute(select([model.Job.table.c.user_id, model.Job.table.c.destination_id, func.count(model.Job.table.c.user_id).label('job_count')]) - .where(and_(model.Job.table.c.state.in_((model.Job.states.QUEUED, model.Job.states.RUNNING)))) - .group_by(model.Job.table.c.user_id, model.Job.table.c.destination_id)) + result = self.sa_session.execute( + select( + [ + model.Job.table.c.user_id, + model.Job.table.c.destination_id, + func.count(model.Job.table.c.user_id).label("job_count"), + ] + ) + .where(and_(model.Job.table.c.state.in_((model.Job.states.QUEUED, model.Job.states.RUNNING)))) + .group_by(model.Job.table.c.user_id, model.Job.table.c.destination_id) + ) for row in result: - if row['user_id'] not in self.user_job_count_per_destination: - self.user_job_count_per_destination[row['user_id']] = {} - self.user_job_count_per_destination[row['user_id']][row['destination_id']] = row['job_count'] + if row["user_id"] not in self.user_job_count_per_destination: + self.user_job_count_per_destination[row["user_id"]] = {} + self.user_job_count_per_destination[row["user_id"]][row["destination_id"]] = row["job_count"] elif self.user_job_count_per_destination is None: self.user_job_count_per_destination = {} def increase_running_job_count(self, user_id, destination_id): - if self.app.job_config.limits.registered_user_concurrent_jobs or \ - self.app.job_config.limits.anonymous_user_concurrent_jobs or \ - self.app.job_config.limits.destination_user_concurrent_jobs: + if ( + self.app.job_config.limits.registered_user_concurrent_jobs + or self.app.job_config.limits.anonymous_user_concurrent_jobs + or self.app.job_config.limits.destination_user_concurrent_jobs + ): if self.user_job_count is None: self.user_job_count = {} if self.user_job_count_per_destination is None: @@ -763,11 +905,15 @@ class JobHandlerQueue(Monitors): self.user_job_count[user_id] = self.user_job_count.get(user_id, 0) + 1 if user_id not in self.user_job_count_per_destination: self.user_job_count_per_destination[user_id] = {} - self.user_job_count_per_destination[user_id][destination_id] = self.user_job_count_per_destination[user_id].get(destination_id, 0) + 1 + self.user_job_count_per_destination[user_id][destination_id] = ( + self.user_job_count_per_destination[user_id].get(destination_id, 0) + 1 + ) if self.app.job_config.limits.destination_total_concurrent_jobs: if self.total_job_count_per_destination is None: self.total_job_count_per_destination = {} - self.total_job_count_per_destination[destination_id] = self.total_job_count_per_destination.get(destination_id, 0) + 1 + self.total_job_count_per_destination[destination_id] = ( + self.total_job_count_per_destination.get(destination_id, 0) + 1 + ) def __check_user_jobs(self, job, job_wrapper): # TODO: Update output datasets' _state = LIMITED or some such new @@ -803,25 +949,40 @@ class JobHandlerQueue(Monitors): elif job.galaxy_session: # Anonymous users only get the hard limit if self.app.job_config.limits.anonymous_user_concurrent_jobs: - count = self.sa_session.query(model.Job).enable_eagerloads(False) \ - .filter(and_(model.Job.session_id == job.galaxy_session.id, - or_(model.Job.state == model.Job.states.RUNNING, - model.Job.state == model.Job.states.QUEUED))).count() + count = ( + self.sa_session.query(model.Job) + .enable_eagerloads(False) + .filter( + and_( + model.Job.session_id == job.galaxy_session.id, + or_( + model.Job.state == model.Job.states.RUNNING, model.Job.state == model.Job.states.QUEUED + ), + ) + ) + .count() + ) if count >= self.app.job_config.limits.anonymous_user_concurrent_jobs: return JOB_WAIT else: - log.warning(f'Job {job.id} is not associated with a user or session so job concurrency limit cannot be checked.') + log.warning( + f"Job {job.id} is not associated with a user or session so job concurrency limit cannot be checked." + ) return JOB_READY def __cache_total_job_count_per_destination(self): # Cache the job count if necessary if self.total_job_count_per_destination is None: self.total_job_count_per_destination = {} - result = self.sa_session.execute(select([model.Job.table.c.destination_id, func.count(model.Job.table.c.destination_id).label('job_count')]) - .where(and_(model.Job.table.c.state.in_((model.Job.states.QUEUED, model.Job.states.RUNNING)))) - .group_by(model.Job.table.c.destination_id)) + result = self.sa_session.execute( + select( + [model.Job.table.c.destination_id, func.count(model.Job.table.c.destination_id).label("job_count")] + ) + .where(and_(model.Job.table.c.state.in_((model.Job.states.QUEUED, model.Job.states.RUNNING)))) + .group_by(model.Job.table.c.destination_id) + ) for row in result: - self.total_job_count_per_destination[row['destination_id']] = row['job_count'] + self.total_job_count_per_destination[row["destination_id"]] = row["job_count"] def get_total_job_count_per_destination(self): self.__cache_total_job_count_per_destination() @@ -862,7 +1023,12 @@ class JobHandlerQueue(Monitors): # If not tracking jobs in the database self.put(job.id, job.tool_id) else: - log.warning("(%s) Handler '%s' received setup message but handler '%s' is already assigned, ignoring", job.id, self.app.config.server_name, job.handler) + log.warning( + "(%s) Handler '%s' received setup message but handler '%s' is already assigned, ignoring", + job.id, + self.app.config.server_name, + job.handler, + ) def put(self, job_id, tool_id): """Add a job to the queue (by job identifier)""" @@ -892,6 +1058,7 @@ class JobHandlerStopQueue(Monitors): """ A queue for jobs which need to be terminated prematurely. """ + STOP_SIGNAL = object() def __init__(self, app: MinimalManagerApp, dispatcher): @@ -957,11 +1124,19 @@ class JobHandlerStopQueue(Monitors): # Clear the session so we get fresh states for job and all datasets self.sa_session.expunge_all() # Fetch all new jobs - newly_deleted_jobs = self.sa_session.query(model.Job).enable_eagerloads(False) \ - .filter((model.Job.state.in_((model.Job.states.DELETED_NEW, - model.Job.states.DELETING, - model.Job.states.STOPPING))) - & (model.Job.handler == self.app.config.server_name)).all() + newly_deleted_jobs = ( + self.sa_session.query(model.Job) + .enable_eagerloads(False) + .filter( + ( + model.Job.state.in_( + (model.Job.states.DELETED_NEW, model.Job.states.DELETING, model.Job.states.STOPPING) + ) + ) + & (model.Job.handler == self.app.config.server_name) + ) + .all() + ) for job in newly_deleted_jobs: # job.stderr is always a string (job.job_stderr + job.tool_stderr, possibly `''`), # while any `not None` message returned in self.queue.get_nowait() is interpreted @@ -980,15 +1155,19 @@ class JobHandlerStopQueue(Monitors): except Empty: pass for job, error_msg in jobs_to_check: - if (job.state not in - (job.states.DELETED_NEW, - job.states.DELETING, - job.states.DELETED, - job.states.STOPPING, - job.states.STOPPED) - and job.finished): + if ( + job.state + not in ( + job.states.DELETED_NEW, + job.states.DELETING, + job.states.DELETED, + job.states.STOPPING, + job.states.STOPPED, + ) + and job.finished + ): # terminated before it got here - log.debug('Job %s already finished, not deleting or stopping', job.id) + log.debug("Job %s already finished, not deleting or stopping", job.id) continue if job.state in (job.states.DELETED_NEW, job.states.DELETING): self.__delete(job, error_msg) @@ -1018,7 +1197,6 @@ class JobHandlerStopQueue(Monitors): class DefaultJobDispatcher: - def __init__(self, app): self.app = app self.job_runners = self.app.job_config.get_job_runner_plugins(self.app.config.server_name) @@ -1045,11 +1223,15 @@ class DefaultJobDispatcher: New-style runner plugin IDs must match the URL's scheme for this to work. """ - runner_name = url.split(':', 1)[0] + runner_name = url.split(":", 1)[0] try: return self.job_runners[runner_name].url_to_destination(url) except Exception: - log.exception("Unable to convert legacy job runner URL '%s' to job destination, destination will be the '%s' runner with no params", url, runner_name) + log.exception( + "Unable to convert legacy job runner URL '%s' to job destination, destination will be the '%s' runner with no params", + url, + runner_name, + ) return JobDestination(runner=runner_name) def put(self, job_wrapper): @@ -1062,7 +1244,7 @@ class DefaultJobDispatcher: log.debug(f"({job_wrapper.job_id}) Dispatching to {runner_name} runner") self.job_runners[runner_name].put(job_wrapper) except KeyError: - log.error(f'put(): ({job_wrapper.job_id}) Invalid job runner: {runner_name}') + log.error(f"put(): ({job_wrapper.job_id}) Invalid job runner: {runner_name}") job_wrapper.fail(DEFAULT_JOB_PUT_FAILURE_MESSAGE) def stop(self, job, job_wrapper): @@ -1084,7 +1266,7 @@ class DefaultJobDispatcher: try: self.job_runners[runner_name].stop_job(job_wrapper) except KeyError: - log.error(f'stop(): ({job_wrapper.get_id_tag()}) Invalid job runner: {runner_name}') + log.error(f"stop(): ({job_wrapper.get_id_tag()}) Invalid job runner: {runner_name}") # Job and output dataset states have already been updated, so nothing is done here. def recover(self, job, job_wrapper): @@ -1093,7 +1275,7 @@ class DefaultJobDispatcher: try: self.job_runners[runner_name].recover(job, job_wrapper) except KeyError: - log.error(f'recover(): ({job_wrapper.job_id}) Invalid job runner: {runner_name}') + log.error(f"recover(): ({job_wrapper.job_id}) Invalid job runner: {runner_name}") job_wrapper.fail(DEFAULT_JOB_PUT_FAILURE_MESSAGE) except ObjectNotFound: msg = "Could not recover job working directory after Galaxy restart" diff --git a/lib/galaxy/jobs/manager.py b/lib/galaxy/jobs/manager.py index 7184e6a4771..4d58a001f35 100644 --- a/lib/galaxy/jobs/manager.py +++ b/lib/galaxy/jobs/manager.py @@ -6,8 +6,14 @@ from functools import partial from sqlalchemy.sql.expression import null -from galaxy.exceptions import HandlerAssignmentError, ToolExecutionError -from galaxy.jobs import handler, NoopQueue +from galaxy.exceptions import ( + HandlerAssignmentError, + ToolExecutionError, +) +from galaxy.jobs import ( + handler, + NoopQueue, +) from galaxy.model import Job from galaxy.structured_app import MinimalManagerApp from galaxy.web_stack.message import JobHandlerMessage @@ -19,6 +25,7 @@ class JobManager: """ Highest level interface to job management. """ + job_handler: handler.JobHandlerI def __init__(self, app: MinimalManagerApp): @@ -32,12 +39,17 @@ class JobManager: def __check_jobs_at_startup(self): if self.app.job_config.use_messaging: - jobs_at_startup = self.app.model.context.query(Job).enable_eagerloads(False) \ - .filter((Job.state == Job.states.NEW) & (Job.handler == null())).all() + jobs_at_startup = ( + self.app.model.context.query(Job) + .enable_eagerloads(False) + .filter((Job.state == Job.states.NEW) & (Job.handler == null())) + .all() + ) if jobs_at_startup: log.info( - 'No handler assigned at startup for the following jobs, will dispatch via message: %s', - ', '.join(str(j.id) for j in jobs_at_startup)) + "No handler assigned at startup for the following jobs, will dispatch via message: %s", + ", ".join(str(j.id) for j in jobs_at_startup), + ) for job in jobs_at_startup: tool = self.app.toolbox.get_tool(job.tool_id, job.tool_version, exact=True) self.enqueue(job, tool) @@ -52,7 +64,7 @@ class JobManager: self.job_handler.job_queue.put(job.id, tool_id) def _message_callback(self, job): - return JobHandlerMessage(task='setup', job_id=job.id) + return JobHandlerMessage(task="setup", job_id=job.id) def enqueue(self, job, tool=None, flush=True): """Queue a job for execution. @@ -76,12 +88,19 @@ class JobManager: configured_handler = tool.get_configured_job_handler(job.params) if configured_handler is not None: p = f" (with job params: {str(job.params)})" if job.params else "" - log.debug("(%s) Configured job handler for tool '%s'%s is: %s", job.log_str(), tool_id, p, configured_handler) + log.debug( + "(%s) Configured job handler for tool '%s'%s is: %s", job.log_str(), tool_id, p, configured_handler + ) queue_callback = partial(self._queue_callback, job, tool_id) message_callback = partial(self._message_callback, job) try: return self.app.job_config.assign_handler( - job, configured=configured_handler, queue_callback=queue_callback, message_callback=message_callback, flush=flush) + job, + configured=configured_handler, + queue_callback=queue_callback, + message_callback=message_callback, + flush=flush, + ) except HandlerAssignmentError as exc: raise ToolExecutionError(exc.args[0], job=exc.obj) @@ -105,6 +124,7 @@ class NoopManager: """ Implements the JobManager interface but does nothing """ + def __init__(self, *args, **kwargs): self.job_handler = NoopHandler() @@ -119,6 +139,7 @@ class NoopHandler(handler.JobHandlerI): """ Implements the JobHandler interface but does nothing """ + def __init__(self, *args, **kwargs): self.job_queue = NoopQueue() self.job_stop_queue = NoopQueue() diff --git a/lib/galaxy/jobs/mapper.py b/lib/galaxy/jobs/mapper.py index cf762ed9b45..93222311836 100644 --- a/lib/galaxy/jobs/mapper.py +++ b/lib/galaxy/jobs/mapper.py @@ -15,7 +15,9 @@ DYNAMIC_RUNNER_NAME = "dynamic" DYNAMIC_DESTINATION_ID = "dynamic_legacy_from_url" ERROR_MESSAGE_NO_RULE_FUNCTION = "Galaxy misconfigured - cannot find dynamic rule function name for destination %s." -ERROR_MESSAGE_RULE_FUNCTION_NOT_FOUND = "Galaxy misconfigured - no rule function named %s found in dynamic rule modules." +ERROR_MESSAGE_RULE_FUNCTION_NOT_FOUND = ( + "Galaxy misconfigured - no rule function named %s found in dynamic rule modules." +) ERROR_MESSAGE_RULE_EXCEPTION = "Encountered an unhandled exception while caching job destination dynamic rule." @@ -24,13 +26,11 @@ class JobMappingConfigurationException(Exception): class JobMappingException(Exception): - def __init__(self, failure_message): self.failure_message = failure_message class JobNotReadyException(Exception): - def __init__(self, job_state=None, message=None): self.job_state = job_state self.message = message @@ -49,6 +49,7 @@ class JobRunnerMapper: This class is responsible to managing the mapping of jobs (in the form of job_wrappers) to job runner url strings. """ + rules_module: ModuleType def __init__(self, job_wrapper, url_to_destination, job_config): @@ -59,7 +60,7 @@ class JobRunnerMapper: self.rules_module = galaxy.jobs.rules if job_config.dynamic_params is not None: - module_name = job_config.dynamic_params['rules_module'] + module_name = job_config.dynamic_params["rules_module"] self.rules_module = importlib.import_module(module_name) def __invoke_expand_function(self, expand_function, destination): @@ -72,7 +73,7 @@ class JobRunnerMapper: "job_wrapper": self.job_wrapper, "rule_helper": RuleHelper(app), "app": app, - "referrer": destination + "referrer": destination, } actual_args = {} @@ -138,7 +139,7 @@ class JobRunnerMapper: calls the url_to_destination method for the appropriate runner. """ dest = self.url_to_destination(url) - dest['id'] = DYNAMIC_DESTINATION_ID + dest["id"] = DYNAMIC_DESTINATION_ID return dest def __find_function_by_tool_id(self, rule_modules): @@ -155,13 +156,12 @@ class JobRunnerMapper: is specified, search within that rules_module, or default to the plugin's top level rules_module. """ - rules_module_name = destination.params.get('rules_module') + rules_module_name = destination.params.get("rules_module") rule_modules = self.__get_rule_modules_or_defaults(rules_module_name) expand_function = None - expand_function_name = destination.params.get('function') + expand_function_name = destination.params.get("function") if expand_function_name: - expand_function = self.__last_matching_function_in_modules( - rule_modules, expand_function_name) + expand_function = self.__last_matching_function_in_modules(rule_modules, expand_function_name) if not expand_function: message = ERROR_MESSAGE_RULE_FUNCTION_NOT_FOUND % expand_function_name raise JobMappingConfigurationException(message) @@ -192,7 +192,7 @@ class JobRunnerMapper: return None def __handle_dynamic_job_destination(self, destination): - expand_type = destination.params.get('type', "python") + expand_type = destination.params.get("type", "python") expand_function = None if expand_type == "python": expand_function = self.__get_expand_function(destination) @@ -207,7 +207,7 @@ class JobRunnerMapper: job_destination = self.__invoke_expand_function(rule_function, destination) if not isinstance(job_destination, galaxy.jobs.JobDestination): job_destination_rep = str(job_destination) # Should be either id or url - if '://' in job_destination_rep: + if "://" in job_destination_rep: job_destination = self.__convert_url_to_destination(job_destination_rep) else: job_destination = self.job_config.get_destination(job_destination_rep) @@ -233,7 +233,9 @@ class JobRunnerMapper: def __cache_job_destination(self, params, raw_job_destination=None): try: - self.cached_job_destination = self.__determine_job_destination(params, raw_job_destination=raw_job_destination) + self.cached_job_destination = self.__determine_job_destination( + params, raw_job_destination=raw_job_destination + ) except (JobMappingConfigurationException, JobMappingException, JobNotReadyException): raise except Exception: @@ -249,7 +251,7 @@ class JobRunnerMapper: externally set to short-circuit the mapper, such as during resubmits. get_job_destination will respect that and not run the mapper if so. """ - if not hasattr(self, 'cached_job_destination'): + if not hasattr(self, "cached_job_destination"): return self.__cache_job_destination(params) return self.cached_job_destination @@ -258,5 +260,4 @@ class JobRunnerMapper: Force update of cached_job_destination to mapper determined job destination, overwriting any externally set cached_job_destination """ - return self.__cache_job_destination( - None, raw_job_destination=raw_job_destination) + return self.__cache_job_destination(None, raw_job_destination=raw_job_destination) diff --git a/lib/galaxy/jobs/rule_helper.py b/lib/galaxy/jobs/rule_helper.py index b70b183ddad..677eba7077a 100644 --- a/lib/galaxy/jobs/rule_helper.py +++ b/lib/galaxy/jobs/rule_helper.py @@ -7,7 +7,7 @@ from sqlalchemy import func from galaxy import ( model, - util + util, ) from galaxy.tool_util.deps.dependencies import ToolInfo @@ -17,7 +17,7 @@ VALID_JOB_HASH_STRATEGIES = ["job", "user", "history", "workflow_invocation"] class RuleHelper: - """ Utility to allow job rules to interface cleanly with the rest of + """Utility to allow job rules to interface cleanly with the rest of Galaxy and shield them from low-level details of models, metrics, etc.... Currently focus is on figuring out job statistics for a given user, but @@ -40,16 +40,17 @@ class RuleHelper: # developers from the details and they shouldn't have to know how to # interrogate tool or job to figure out if it can be run in a # container. - if hasattr(job_or_tool, 'containers'): + if hasattr(job_or_tool, "containers"): tool = job_or_tool - elif hasattr(job_or_tool, 'tool'): + elif hasattr(job_or_tool, "tool"): # Have a JobWrapper-like tool = job_or_tool.tool else: # Have a Job object. tool = self.app.toolbox.get_tool(job_or_tool.tool_id, tool_version=job_or_tool.tool_version) - tool_info = ToolInfo(tool.containers, tool.requirements, tool.requires_galaxy_python_environment, - tool.docker_env_pass_through) + tool_info = ToolInfo( + tool.containers, tool.requirements, tool.requires_galaxy_python_environment, tool.docker_env_pass_through + ) container_description = self.app.container_finder.find_best_container_description([container_type], tool_info) return container_description is not None @@ -71,17 +72,11 @@ class RuleHelper: """ return self.supports_container(job_or_tool, container_type="singularity") - def job_count( - self, - **kwds - ): + def job_count(self, **kwds): query = self.query(model.Job) return self._filter_job_query(query, **kwds).count() - def sum_job_runtime( - self, - **kwds - ): + def sum_job_runtime(self, **kwds): # TODO: Consider sum_core_hours or something that scales runtime by # by calculated cores per job. query = self.metric_query( @@ -147,7 +142,7 @@ class RuleHelper: return query def should_burst(self, destination_ids, num_jobs, job_states=None): - """ Check if the specified destinations ``destination_ids`` have at + """Check if the specified destinations ``destination_ids`` have at least ``num_jobs`` assigned to it - send in ``job_state`` as ``queued`` to limit this check to number of jobs queued. @@ -159,8 +154,7 @@ class RuleHelper: if job_states is None: job_states = "queued,running" from_destination_job_count = self.job_count( - for_destinations=destination_ids, - for_job_states=util.listify(job_states) + for_destinations=destination_ids, for_job_states=util.listify(job_states) ) # Would this job push us over maximum job count before requiring # bursting (roughly... very roughly given many handler threads may be @@ -168,7 +162,7 @@ class RuleHelper: return (from_destination_job_count + 1) > int(num_jobs) def choose_one(self, lst, hash_value=None): - """ Choose a random value from supplied list. If hash_value is passed + """Choose a random value from supplied list. If hash_value is passed in then every request with that same hash_value would produce the same choice from the supplied list. """ @@ -184,7 +178,7 @@ class RuleHelper: return lst[random_index] def job_hash(self, job, hash_by=None): - """ Produce a reproducible hash for the given job on various + """Produce a reproducible hash for the given job on various criteria - for instance if hash_by is "workflow_invocation,history" - all jobs within the same workflow invocation will receive the same hash - for jobs outside of workflows all jobs within the same history @@ -206,7 +200,7 @@ class RuleHelper: return self._try_hash_for_job(job, "job") def _try_hash_for_job(self, job, hash_by): - """ May return False or None if hash type is invalid for that job - + """May return False or None if hash type is invalid for that job - e.g. attempting to hash by user for anonymous job or by workflow invocation for jobs outside of workflows. """ diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py index 2d9dbbf2273..61859b77741 100644 --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -16,17 +16,20 @@ from queue import ( import galaxy.jobs from galaxy import model -from galaxy.job_execution.output_collect import default_exit_code_file, read_exit_code_from +from galaxy.job_execution.output_collect import ( + default_exit_code_file, + read_exit_code_from, +) from galaxy.jobs.command_factory import build_command from galaxy.jobs.runners.util import runner_states from galaxy.jobs.runners.util.env import env_to_statement from galaxy.jobs.runners.util.job_script import ( job_script, - write_script + write_script, ) from galaxy.tool_util.deps.dependencies import ( JobInfo, - ToolInfo + ToolInfo, ) from galaxy.tool_util.output_checker import DETECTED_JOB_STATE from galaxy.util import ( @@ -47,7 +50,9 @@ STOP_SIGNAL = object() JOB_RUNNER_PARAMETER_UNKNOWN_MESSAGE = "Invalid job runner parameter for this plugin: %s" -JOB_RUNNER_PARAMETER_MAP_PROBLEM_MESSAGE = "Job runner parameter '%s' value '%s' could not be converted to the correct type" +JOB_RUNNER_PARAMETER_MAP_PROBLEM_MESSAGE = ( + "Job runner parameter '%s' value '%s' could not be converted to the correct type" +) JOB_RUNNER_PARAMETER_VALIDATION_FAILED_MESSAGE = "Job runner parameter %s failed validation" GALAXY_LIB_ADJUST_TEMPLATE = """GALAXY_LIB="%s"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi;""" @@ -55,7 +60,6 @@ GALAXY_VENV_TEMPLATE = """GALAXY_VIRTUAL_ENV="%s"; if [ "$GALAXY_VIRTUAL_ENV" != class RunnerParams(ParamsWithSpecs): - def _param_unknown_error(self, name): raise Exception(JOB_RUNNER_PARAMETER_UNKNOWN_MESSAGE % name) @@ -68,21 +72,20 @@ class RunnerParams(ParamsWithSpecs): class BaseJobRunner: - start_methods = ['_init_monitor_thread', '_init_worker_threads'] + start_methods = ["_init_monitor_thread", "_init_worker_threads"] DEFAULT_SPECS = dict(recheck_missing_job_retries=dict(map=int, valid=lambda x: int(x) >= 0, default=0)) def __init__(self, app, nworkers, **kwargs): - """Start the job runner - """ + """Start the job runner""" self.app = app self.redact_email_in_job_name = self.app.config.redact_email_in_job_name self.sa_session = app.model.context self.nworkers = nworkers runner_param_specs = self.DEFAULT_SPECS.copy() - if 'runner_param_specs' in kwargs: - runner_param_specs.update(kwargs.pop('runner_param_specs')) + if "runner_param_specs" in kwargs: + runner_param_specs.update(kwargs.pop("runner_param_specs")) if kwargs: - log.debug('Loading %s with params: %s', self.runner_name, kwargs) + log.debug("Loading %s with params: %s", self.runner_name, kwargs) self.runner_params = RunnerParams(specs=runner_param_specs, params=kwargs) self.runner_state_handlers = build_state_handlers() self._should_stop = False @@ -92,11 +95,10 @@ class BaseJobRunner: getattr(self, start_method, lambda: None)() def _init_worker_threads(self): - """Start ``nworkers`` worker threads. - """ + """Start ``nworkers`` worker threads.""" self.work_queue = Queue() self.work_threads = [] - log.debug(f'Starting {self.nworkers} {self.runner_name} workers') + log.debug(f"Starting {self.nworkers} {self.runner_name} workers") for i in range(self.nworkers): worker = threading.Thread(name="%s.work_thread-%d" % (self.runner_name, i), target=self.run_next) worker.daemon = True @@ -115,8 +117,7 @@ class BaseJobRunner: yield thread def run_next(self): - """Run the next item in the work queue (a job waiting to run) - """ + """Run the next item in the work queue (a job waiting to run)""" while self._should_stop is False: try: (method, arg) = self.work_queue.get(timeout=1) @@ -132,16 +133,15 @@ class BaseJobRunner: # arg should be a JobWrapper/TaskWrapper job_id = arg.get_id_tag() except Exception: - job_id = 'unknown' + job_id = "unknown" try: name = method.__name__ except Exception: - name = 'unknown' + name = "unknown" try: - action_str = f'galaxy.jobs.runners.{self.__class__.__name__.lower()}.{name}' + action_str = f"galaxy.jobs.runners.{self.__class__.__name__.lower()}.{name}" action_timer = self.app.execution_timer_factory.get_timer( - f'internals.{action_str}', - 'job runner action %s for job ${job_id} executed' % (action_str) + f"internals.{action_str}", "job runner action %s for job ${job_id} executed" % (action_str) ) method(arg) log.trace(action_timer.to_str(job_id=job_id)) @@ -157,8 +157,7 @@ class BaseJobRunner: # Causes a runner's `queue_job` method to be called from a worker thread def put(self, job_wrapper): - """Add a job to the queue (by job identifier), indicate that the job is ready to run. - """ + """Add a job to the queue (by job identifier), indicate that the job is ready to run.""" put_timer = ExecutionTimer() job_wrapper.enqueue() self.mark_as_queued(job_wrapper) @@ -168,8 +167,7 @@ class BaseJobRunner: self.work_queue.put((self.queue_job, job_wrapper)) def shutdown(self): - """Attempts to gracefully shut down the worker threads - """ + """Attempts to gracefully shut down the worker threads""" log.info("%s: Sending stop signal to %s job worker threads", self.runner_name, len(self.work_threads)) self._should_stop = True for _ in range(len(self.work_threads)): @@ -199,8 +197,12 @@ class BaseJobRunner: except KeyError: # thread is now stopped continue - log.warning("Timed out waiting for job worker thread %s to terminate, shutdown will be unclean! Thread " - "stack is:\n%s", thread.name, ''.join(traceback.format_stack(frame))) + log.warning( + "Timed out waiting for job worker thread %s to terminate, shutdown will be unclean! Thread " + "stack is:\n%s", + thread.name, + "".join(traceback.format_stack(frame)), + ) # Most runners should override the legacy URL handler methods and destination param method def url_to_destination(self, url): @@ -211,22 +213,22 @@ class BaseJobRunner: This base class method converts from a URL to a very basic JobDestination without destination params. """ - return galaxy.jobs.JobDestination(runner=url.split(':')[0]) + return galaxy.jobs.JobDestination(runner=url.split(":")[0]) def parse_destination_params(self, params): - """Parse the JobDestination ``params`` dict and return the runner's native representation of those params. - """ + """Parse the JobDestination ``params`` dict and return the runner's native representation of those params.""" raise NotImplementedError() - def prepare_job(self, - job_wrapper, - include_metadata=False, - include_work_dir_outputs=True, - modify_command_for_container=True, - stdout_file=None, - stderr_file=None): - """Some sanity checks that all runners' queue_job() methods are likely to want to do - """ + def prepare_job( + self, + job_wrapper, + include_metadata=False, + include_work_dir_outputs=True, + modify_command_for_container=True, + stdout_file=None, + stderr_file=None, + ): + """Some sanity checks that all runners' queue_job() methods are likely to want to do""" job_id = job_wrapper.get_id_tag() job_state = job_wrapper.get_state() job_wrapper.is_ready = False @@ -260,7 +262,7 @@ class BaseJobRunner: return False if not job_wrapper.runner_command_line: - job_wrapper.finish('', '') + job_wrapper.finish("", "") return False return True @@ -275,13 +277,15 @@ class BaseJobRunner: def recover(self, job, job_wrapper): raise NotImplementedError() - def build_command_line(self, - job_wrapper, - include_metadata=False, - include_work_dir_outputs=True, - modify_command_for_container=True, - stdout_file=None, - stderr_file=None): + def build_command_line( + self, + job_wrapper, + include_metadata=False, + include_work_dir_outputs=True, + modify_command_for_container=True, + stdout_file=None, + stderr_file=None, + ): container = self._find_container(job_wrapper) if not container and job_wrapper.requires_containerization: raise Exception("Failed to find a container when required, contact Galaxy admin.") @@ -302,7 +306,9 @@ class BaseJobRunner: to work_dir output file and ultimate destination. """ if tool_working_directory is not None and job_working_directory is not None: - raise Exception("get_work_dir_outputs called with both a job and tool working directory, only one may be specified") + raise Exception( + "get_work_dir_outputs called with both a job and tool working directory, only one may be specified" + ) if tool_working_directory is None: if not job_working_directory: @@ -334,14 +340,24 @@ class BaseJobRunner: output_pairs.append((source_file, destination)) else: # Security violation. - log.exception("from_work_dir specified a location not in the working directory: %s, %s", source_file, job_wrapper.working_directory) + log.exception( + "from_work_dir specified a location not in the working directory: %s, %s", + source_file, + job_wrapper.working_directory, + ) return output_pairs def _walk_dataset_outputs(self, job): for dataset_assoc in job.output_datasets + job.output_library_datasets: - for dataset in dataset_assoc.dataset.dataset.history_associations + dataset_assoc.dataset.dataset.library_associations: + for dataset in ( + dataset_assoc.dataset.dataset.history_associations + dataset_assoc.dataset.dataset.library_associations + ): if isinstance(dataset, self.app.model.HistoryDatasetAssociation): - joda = self.sa_session.query(self.app.model.JobToOutputDatasetAssociation).filter_by(job=job, dataset=dataset).first() + joda = ( + self.sa_session.query(self.app.model.JobToOutputDatasetAssociation) + .filter_by(job=job, dataset=dataset) + .first() + ) yield (joda, dataset) # TODO: why is this not just something easy like: # for dataset_assoc in job.output_datasets + job.output_library_datasets: @@ -356,37 +372,52 @@ class BaseJobRunner: # run the metadata setting script here # this is terminate-able when output dataset/job is deleted # so that long running set_meta()s can be canceled without having to reboot the server - if job_wrapper.get_state() not in [model.Job.states.ERROR, model.Job.states.DELETED] and job_wrapper.job_io.output_paths: + if ( + job_wrapper.get_state() not in [model.Job.states.ERROR, model.Job.states.DELETED] + and job_wrapper.job_io.output_paths + ): lib_adjust = GALAXY_LIB_ADJUST_TEMPLATE % job_wrapper.galaxy_lib_dir venv = GALAXY_VENV_TEMPLATE % job_wrapper.galaxy_virtual_env - external_metadata_script = job_wrapper.setup_external_metadata(output_fnames=job_wrapper.job_io.get_output_fnames(), - set_extension=True, - tmp_dir=job_wrapper.working_directory, - # We don't want to overwrite metadata that was copied over in init_meta(), as per established behavior - kwds={'overwrite': False}) + external_metadata_script = job_wrapper.setup_external_metadata( + output_fnames=job_wrapper.job_io.get_output_fnames(), + set_extension=True, + tmp_dir=job_wrapper.working_directory, + # We don't want to overwrite metadata that was copied over in init_meta(), as per established behavior + kwds={"overwrite": False}, + ) external_metadata_script = f"{lib_adjust} {venv} {external_metadata_script}" if resolve_requirements: - dependency_shell_commands = self.app.datatypes_registry.set_external_metadata_tool.build_dependency_shell_commands(job_directory=job_wrapper.working_directory) + dependency_shell_commands = ( + self.app.datatypes_registry.set_external_metadata_tool.build_dependency_shell_commands( + job_directory=job_wrapper.working_directory + ) + ) if dependency_shell_commands: if isinstance(dependency_shell_commands, list): dependency_shell_commands = "&&".join(dependency_shell_commands) external_metadata_script = f"{dependency_shell_commands}&&{external_metadata_script}" - log.debug('executing external set_meta script for job %d: %s' % (job_wrapper.job_id, external_metadata_script)) - external_metadata_proc = subprocess.Popen(args=external_metadata_script, - shell=True, - cwd=job_wrapper.working_directory, - env=os.environ, - preexec_fn=os.setpgrp) - job_wrapper.external_output_metadata.set_job_runner_external_pid(external_metadata_proc.pid, self.sa_session) + log.debug( + "executing external set_meta script for job %d: %s" % (job_wrapper.job_id, external_metadata_script) + ) + external_metadata_proc = subprocess.Popen( + args=external_metadata_script, + shell=True, + cwd=job_wrapper.working_directory, + env=os.environ, + preexec_fn=os.setpgrp, + ) + job_wrapper.external_output_metadata.set_job_runner_external_pid( + external_metadata_proc.pid, self.sa_session + ) external_metadata_proc.wait() - log.debug('execution of external set_meta for job %d finished' % job_wrapper.job_id) + log.debug("execution of external set_meta for job %d finished" % job_wrapper.job_id) def get_job_file(self, job_wrapper, **kwds): job_metrics = job_wrapper.app.job_metrics job_instrumenter = job_metrics.job_instrumenters[job_wrapper.job_destination.id] - env_setup_commands = kwds.get('env_setup_commands', []) - env_setup_commands.append(job_wrapper.get_env_setup_clause() or '') + env_setup_commands = kwds.get("env_setup_commands", []) + env_setup_commands.append(job_wrapper.get_env_setup_clause() or "") destination = job_wrapper.job_destination or {} envs = destination.get("env", []) envs.extend(job_wrapper.environment_variables) @@ -408,7 +439,7 @@ class BaseJobRunner: # Additional logging to enable if debugging from_work_dir handling, metadata # commands, etc... (or just peak in the job script.) job_id = job_wrapper.job_id - log.debug(f'({job_id}) command is: {command_line}') + log.debug(f"({job_id}) command is: {command_line}") options.update(**kwds) return job_script(**options) @@ -458,11 +489,7 @@ class BaseJobRunner: ) destination_info = job_wrapper.job_destination.params - container = self.app.container_finder.find_container( - tool_info, - destination_info, - job_info - ) + container = self.app.container_finder.find_container(tool_info, destination_info, job_info) if container: job_wrapper.set_container(container) return container @@ -474,17 +501,17 @@ class BaseJobRunner: if job_state.runner_state_handled: break except Exception: - log.exception('Caught exception in runner state handler') + log.exception("Caught exception in runner state handler") def fail_job(self, job_state, exception=False): - if getattr(job_state, 'stop_job', True): + if getattr(job_state, "stop_job", True): self.stop_job(job_state.job_wrapper) job_state.job_wrapper.reclaim_ownership() - self._handle_runner_state('failure', job_state) + self._handle_runner_state("failure", job_state) # Not convinced this is the best way to indicate this state, but # something necessary if not job_state.runner_state_handled: - job_state.job_wrapper.fail(getattr(job_state, 'fail_message', 'Job failed'), exception=exception) + job_state.job_wrapper.fail(getattr(job_state, "fail_message", "Job failed"), exception=exception) if job_state.job_wrapper.cleanup_job == "always": job_state.cleanup() @@ -495,7 +522,9 @@ class BaseJobRunner: self.app.job_manager.job_handler.dispatcher.put(job_state.job_wrapper) def _job_io_for_db(self, stream): - return shrink_stream_by_size(stream, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True) + return shrink_stream_by_size( + stream, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True + ) def _finish_or_resubmit_job(self, job_state, job_stdout, job_stderr, job_id=None, external_job_id=None): job_wrapper = job_state.job_wrapper @@ -527,7 +556,14 @@ class BaseJobRunner: tool_stderr = job_stderr job_stderr = None - check_output_detected_state = job_wrapper.check_tool_output(tool_stdout, tool_stderr, tool_exit_code=exit_code, job=job, job_stdout=job_stdout, job_stderr=job_stderr) + check_output_detected_state = job_wrapper.check_tool_output( + tool_stdout, + tool_stderr, + tool_exit_code=exit_code, + job=job, + job_stdout=job_stdout, + job_stderr=job_stderr, + ) job_ok = check_output_detected_state == DETECTED_JOB_STATE.OK # clean up the job files @@ -544,12 +580,19 @@ class BaseJobRunner: if check_output_detected_state == DETECTED_JOB_STATE.OUT_OF_MEMORY_ERROR: job_runner_state = JobState.runner_states.MEMORY_LIMIT_REACHED job_state.runner_state = job_runner_state - self._handle_runner_state('failure', job_state) + self._handle_runner_state("failure", job_state) # Was resubmitted or something - I think we are done with it. if job_state.runner_state_handled: return - job_wrapper.finish(tool_stdout, tool_stderr, exit_code, check_output_detected_state=check_output_detected_state, job_stdout=job_stdout, job_stderr=job_stderr) + job_wrapper.finish( + tool_stdout, + tool_stderr, + exit_code, + check_output_detected_state=check_output_detected_state, + job_stdout=job_stdout, + job_stderr=job_stderr, + ) except Exception: log.exception(f"({job_id or ''}/{external_job_id or ''}) Job wrapper finish method failed") job_wrapper.fail("Unable to finish job", exception=True) @@ -559,6 +602,7 @@ class JobState: """ Encapsulate state of jobs. """ + runner_states = runner_states def __init__(self, job_wrapper, job_destination): @@ -570,26 +614,26 @@ class JobState: if self.job_wrapper: self.redact_email_in_job_name = self.job_wrapper.app.config.redact_email_in_job_name - self.cleanup_file_attributes = ['job_file', 'output_file', 'error_file', 'exit_code_file'] + self.cleanup_file_attributes = ["job_file", "output_file", "error_file", "exit_code_file"] def set_defaults(self, files_dir): if self.job_wrapper is not None: id_tag = self.job_wrapper.get_id_tag() if files_dir is not None: self.job_file = JobState.default_job_file(files_dir, id_tag) - self.output_file = os.path.join(files_dir, f'galaxy_{id_tag}.o') - self.error_file = os.path.join(files_dir, f'galaxy_{id_tag}.e') + self.output_file = os.path.join(files_dir, f"galaxy_{id_tag}.o") + self.error_file = os.path.join(files_dir, f"galaxy_{id_tag}.e") self.exit_code_file = default_exit_code_file(files_dir, id_tag) - job_name = f'g{id_tag}' + job_name = f"g{id_tag}" if self.job_wrapper.tool.old_id: - job_name += f'_{self.job_wrapper.tool.old_id}' + job_name += f"_{self.job_wrapper.tool.old_id}" if not self.redact_email_in_job_name and self.job_wrapper.user: - job_name += f'_{self.job_wrapper.user}' - self.job_name = ''.join(x if x in (f"{string.ascii_letters + string.digits}_") else '_' for x in job_name) + job_name += f"_{self.job_wrapper.user}" + self.job_name = "".join(x if x in (f"{string.ascii_letters + string.digits}_") else "_" for x in job_name) @staticmethod def default_job_file(files_dir, id_tag): - return os.path.join(files_dir, f'galaxy_{id_tag}.sh') + return os.path.join(files_dir, f"galaxy_{id_tag}.sh") def read_exit_code(self): return read_exit_code_from(self.exit_code_file, self.job_wrapper.get_id_tag()) @@ -615,7 +659,18 @@ class AsynchronousJobState(JobState): to communicate with distributed resource manager. """ - def __init__(self, files_dir=None, job_wrapper=None, job_id=None, job_file=None, output_file=None, error_file=None, exit_code_file=None, job_name=None, job_destination=None): + def __init__( + self, + files_dir=None, + job_wrapper=None, + job_id=None, + job_file=None, + output_file=None, + error_file=None, + exit_code_file=None, + job_name=None, + job_destination=None, + ): super().__init__(job_wrapper, job_destination) self.old_state = None self._running = False @@ -711,7 +766,7 @@ class AsynchronousJobRunner(BaseJobRunner, Monitors): try: self.check_watched_items() except Exception: - log.exception('Unhandled exception checking active jobs') + log.exception("Unhandled exception checking active jobs") # Sleep a bit before the next state check time.sleep(self.app.config.job_runner_monitor_sleep) @@ -761,15 +816,15 @@ class AsynchronousJobRunner(BaseJobRunner, Monitors): collect_output_success = True while which_try < self.app.config.retry_job_output_collection + 1: try: - with open(job_state.output_file, "rb") as stdout_file, open(job_state.error_file, 'rb') as stderr_file: + with open(job_state.output_file, "rb") as stdout_file, open(job_state.error_file, "rb") as stderr_file: stdout = self._job_io_for_db(stdout_file) stderr = self._job_io_for_db(stderr_file) break except Exception as e: if which_try == self.app.config.retry_job_output_collection: - stdout = '' + stdout = "" stderr = job_state.runner_states.JOB_OUTPUT_NOT_RETURNED_FROM_CLUSTER - log.error('(%s/%s) %s: %s', galaxy_id_tag, external_job_id, stderr, unicodify(e)) + log.error("(%s/%s) %s: %s", galaxy_id_tag, external_job_id, stderr, unicodify(e)) collect_output_success = False else: time.sleep(1) diff --git a/lib/galaxy/jobs/runners/chronos.py b/lib/galaxy/jobs/runners/chronos.py index ae161b4b772..297726fb1da 100644 --- a/lib/galaxy/jobs/runners/chronos.py +++ b/lib/galaxy/jobs/runners/chronos.py @@ -3,15 +3,21 @@ import logging import os from galaxy import model -from galaxy.jobs.runners import AsynchronousJobRunner, AsynchronousJobState +from galaxy.jobs.runners import ( + AsynchronousJobRunner, + AsynchronousJobState, +) from galaxy.util import unicodify -CHRONOS_IMPORT_MSG = ('The Python \'chronos\' package is required to use ' - 'this feature, please install it or correct the ' - 'following error:\nImportError {msg!s}') +CHRONOS_IMPORT_MSG = ( + "The Python 'chronos' package is required to use " + "this feature, please install it or correct the " + "following error:\nImportError {msg!s}" +) try: import chronos + chronos_exceptions = ( chronos.ChronosAPIError, chronos.UnauthorizedError, @@ -23,7 +29,7 @@ except ImportError as e: CHRONOS_IMPORT_MSG.format(msg=unicodify(e)) -__all__ = ('ChronosJobRunner',) +__all__ = ("ChronosJobRunner",) LOGGER = logging.getLogger(__name__) @@ -52,19 +58,19 @@ def to_dict(segments, v): def _write_logfile(logfile, msg): - with open(logfile, 'w') as fil: + with open(logfile, "w") as fil: fil.write(msg) def _parse_job_volumes_list(li): # Convert comma separated string to list - volume_list = list(li.split(',')) + volume_list = list(li.split(",")) # Create the list with right mountpoint and permissions mountpoint_list = [] # Convert each element to right format for i in volume_list: - hpath, cpath, mode = i.split(':') - mountpoint_list.append({'hostPath': hpath, 'containerPath': cpath, 'mode': mode}) + hpath, cpath, mode = i.split(":") + mountpoint_list.append({"hostPath": hpath, "containerPath": cpath, "mode": mode}) return mountpoint_list @@ -72,60 +78,56 @@ def _add_galaxy_environment_variables(cpus, memory): # Set: # GALAXY_SLOTS: to docker_cpu # GALAXY_MEMORY_MB to docker_memory - return [{'name': 'GALAXY_SLOTS', 'value': cpus}, {'name': 'GALAXY_MEMORY_MB', 'value': memory}] + return [{"name": "GALAXY_SLOTS", "value": cpus}, {"name": "GALAXY_MEMORY_MB", "value": memory}] class ChronosJobRunner(AsynchronousJobRunner): - runner_name = 'ChronosRunner' - RUNNER_PARAM_SPEC_KEY = 'runner_param_specs' - JOB_NAME_PREFIX = 'galaxy-chronos-' + runner_name = "ChronosRunner" + RUNNER_PARAM_SPEC_KEY = "runner_param_specs" + JOB_NAME_PREFIX = "galaxy-chronos-" RUNNER_PARAM_SPEC = { - 'chronos': { - 'map': str, + "chronos": { + "map": str, }, - 'insecure': { - 'map': lambda x: x in ['true', 'True', 'TRUE'], - 'default': True, + "insecure": { + "map": lambda x: x in ["true", "True", "TRUE"], + "default": True, }, - 'username': { - 'map': str, + "username": { + "map": str, }, - 'password': { - 'map': str, - }, - 'owner': { - 'map': str + "password": { + "map": str, }, + "owner": {"map": str}, } DESTINATION_PARAMS_SPEC = { - 'docker_cpu': { - 'default': 0.1, - 'map_name': 'cpus', - 'map': float, + "docker_cpu": { + "default": 0.1, + "map_name": "cpus", + "map": float, }, - 'docker_memory': { - 'default': 128, - 'map_name': 'mem', - 'map': int, + "docker_memory": { + "default": 128, + "map_name": "mem", + "map": int, }, - 'docker_disk': { - 'default': 256, - 'map_name': 'disk', - 'map': int, + "docker_disk": { + "default": 256, + "map_name": "disk", + "map": int, }, - 'volumes': { - 'default': None, - 'map_name': 'container/volumes', - 'map': ( - lambda x: _parse_job_volumes_list(x) - if x is not None else []) + "volumes": { + "default": None, + "map_name": "container/volumes", + "map": (lambda x: _parse_job_volumes_list(x) if x is not None else []), }, - 'max_retries': { - 'default': 2, - 'map_name': 'retries', - 'map': int, + "max_retries": { + "default": 2, + "map_name": "retries", + "map": int, }, } @@ -136,28 +138,30 @@ class ChronosJobRunner(AsynchronousJobRunner): kwargs[self.RUNNER_PARAM_SPEC_KEY] = {} kwargs[self.RUNNER_PARAM_SPEC_KEY].update(self.RUNNER_PARAM_SPEC) super().__init__(app, nworkers, **kwargs) - protocol = 'http' if self.runner_params.get('insecure', True) else 'https' + protocol = "http" if self.runner_params.get("insecure", True) else "https" self._chronos_client = chronos.connect( - self.runner_params['chronos'], - username=self.runner_params.get('username'), - password=self.runner_params.get('password'), - proto=protocol) + self.runner_params["chronos"], + username=self.runner_params.get("username"), + password=self.runner_params.get("password"), + proto=protocol, + ) @handle_exception_call def queue_job(self, job_wrapper): LOGGER.debug(f"Starting queue_job for job {job_wrapper.get_id_tag()}") - if not self.prepare_job(job_wrapper, include_metadata=False, - modify_command_for_container=False): + if not self.prepare_job(job_wrapper, include_metadata=False, modify_command_for_container=False): LOGGER.debug(f"Not ready {job_wrapper.get_id_tag()}") return job_destination = job_wrapper.job_destination chronos_job_spec = self._get_job_spec(job_wrapper) - job_name = chronos_job_spec['name'] + job_name = chronos_job_spec["name"] self._chronos_client.add(chronos_job_spec) - ajs = AsynchronousJobState(files_dir=job_wrapper.working_directory, - job_wrapper=job_wrapper, - job_id=job_name, - job_destination=job_destination) + ajs = AsynchronousJobState( + files_dir=job_wrapper.working_directory, + job_wrapper=job_wrapper, + job_id=job_name, + job_destination=job_destination, + ) self.monitor_queue.put(ajs) @handle_exception_call @@ -166,46 +170,40 @@ class ChronosJobRunner(AsynchronousJobRunner): job_name = self.JOB_NAME_PREFIX + job_id job = self._retrieve_job(job_name) if job: - msg = 'Job {name!r} is terminated' + msg = "Job {name!r} is terminated" self._chronos_client.delete(job_name) LOGGER.debug(msg.format(name=job_name)) else: - msg = 'Job {name!r} not found. It cannot be terminated.' + msg = "Job {name!r} not found. It cannot be terminated." LOGGER.error(msg.format(name=job_name)) def recover(self, job, job_wrapper): - msg = ('(name!r/runner!r) is still in {state!s} state, adding to' - ' the runner monitor queue') + msg = "(name!r/runner!r) is still in {state!s} state, adding to" " the runner monitor queue" job_id = job.get_job_runner_external_id() - ajs = AsynchronousJobState(files_dir=job_wrapper.working_directory, - job_wrapper=job_wrapper) + ajs = AsynchronousJobState(files_dir=job_wrapper.working_directory, job_wrapper=job_wrapper) ajs.job_id = self.JOB_NAME_PREFIX + str(job_id) ajs.command_line = job.command_line ajs.job_wrapper = job_wrapper ajs.job_destination = job_wrapper.job_destination if job.state in (model.Job.states.RUNNING, model.Job.states.STOPPED): - LOGGER.debug(msg.format( - name=job.id, runner=job.job_runner_external_id, - state=job.state)) + LOGGER.debug(msg.format(name=job.id, runner=job.job_runner_external_id, state=job.state)) ajs.old_state = model.Job.states.RUNNING ajs.running = True self.monitor_queue.put(ajs) elif job.state == model.Job.states.QUEUED: - LOGGER.debug(msg.format( - name=job.id, runner=job.job_runner_external_id, - state='queued')) + LOGGER.debug(msg.format(name=job.id, runner=job.job_runner_external_id, state="queued")) ajs.old_state = model.Job.states.QUEUED ajs.running = False self.monitor_queue.put(ajs) def fail_job(self, job_state, exception=False): - if getattr(job_state, 'stop_job', True): + if getattr(job_state, "stop_job", True): self.stop_job(job_state.job_wrapper) job_state.job_wrapper.reclaim_ownership() - self._handle_runner_state('failure', job_state) + self._handle_runner_state("failure", job_state) if not job_state.runner_state_handled: - job_state.job_wrapper.fail(getattr(job_state, 'fail_message', 'Job failed'), exception=exception) - self._finish_or_resubmit_job(job_state, '', job_state.fail_message, job_id=job_state.job_id) + job_state.job_wrapper.fail(getattr(job_state, "fail_message", "Job failed"), exception=exception) + self._finish_or_resubmit_job(job_state, "", job_state.fail_message, job_id=job_state.job_id) if job_state.job_wrapper.cleanup_job == "always": job_state.cleanup() @@ -215,28 +213,27 @@ class ChronosJobRunner(AsynchronousJobRunner): job = self._retrieve_job(job_name) # TODO: how can stopped GxIT jobs be handled here? if job: - succeeded = job['successCount'] - errors = job['errorCount'] + succeeded = job["successCount"] + errors = job["errorCount"] if succeeded > 0: return self._mark_as_successful(job_state) elif not succeeded and not errors: return self._mark_as_active(job_state) elif errors: - max_retries = job['retries'] + max_retries = job["retries"] if max_retries == 0: - msg = 'Job {name!r} failed. No retries performed.' + msg = "Job {name!r} failed. No retries performed." else: - msg = 'Job {name!r} failed more than {retries!s} times.' + msg = "Job {name!r} failed more than {retries!s} times." reason = msg.format(name=job_name, retries=str(max_retries)) return self._mark_as_failed(job_state, reason) - reason = f'Job {job_name!r} not found' + reason = f"Job {job_name!r} not found" return self._mark_as_failed(job_state, reason) def _mark_as_successful(self, job_state): - msg = 'Job {name!r} finished successfully' - _write_logfile(job_state.output_file, - msg.format(name=job_state.job_id)) - _write_logfile(job_state.error_file, '') + msg = "Job {name!r} finished successfully" + _write_logfile(job_state.output_file, msg.format(name=job_state.job_id)) + _write_logfile(job_state.error_file, "") job_state.running = False job_state.job_wrapper.change_state(model.Job.states.OK) self.mark_as_finished(job_state) @@ -264,10 +261,10 @@ class ChronosJobRunner(AsynchronousJobRunner): def parse_destination_params(self, params): parsed_params = {} for k, spec in self.DESTINATION_PARAMS_SPEC.items(): - value = params.get(k, spec.get('default')) - map_to = spec.get('map_name') - mapper = spec.get('map') - segments = map_to.split('/') + value = params.get(k, spec.get("default")) + map_to = spec.get("map_name") + mapper = spec.get("map") + segments = map_to.split("/") parsed_params.update(to_dict(segments, mapper(value))) return parsed_params @@ -280,8 +277,8 @@ class ChronosJobRunner(AsynchronousJobRunner): path = f"{job_wrapper.working_directory}/chronos_{job_wrapper.get_id_tag()}.sh" mode = 0o755 - with open(path, 'w', encoding='utf-8') as f: - f.write('#!/bin/bash\n') + with open(path, "w", encoding="utf-8") as f: + f.write("#!/bin/bash\n") f.write(job_wrapper.runner_command_line) os.chmod(path, mode) return path @@ -291,34 +288,33 @@ class ChronosJobRunner(AsynchronousJobRunner): job_destination = job_wrapper.job_destination command_script_path = self.write_command(job_wrapper) template = { - 'async': False, + "async": False, # 'command': job_wrapper.runner_command_line, - 'command': f"$SHELL {command_script_path}", - 'owner': self.runner_params['owner'], - 'disabled': False, - 'schedule': 'R1//PT1S', - 'name': job_name, + "command": f"$SHELL {command_script_path}", + "owner": self.runner_params["owner"], + "disabled": False, + "schedule": "R1//PT1S", + "name": job_name, # Add Galaxy environemnt variables to json - 'environmentVariables': _add_galaxy_environment_variables(job_destination.params.get('docker_cpu'), job_destination.params.get('docker_memory')), + "environmentVariables": _add_galaxy_environment_variables( + job_destination.params.get("docker_cpu"), job_destination.params.get("docker_memory") + ), } - if not job_destination.params.get('docker_enabled'): - raise ChronosRunnerException( - 'ChronosJobRunner needs \'docker_enabled\' to be set as True') - destination_params = self.parse_destination_params( - job_destination.params) + if not job_destination.params.get("docker_enabled"): + raise ChronosRunnerException("ChronosJobRunner needs 'docker_enabled' to be set as True") + destination_params = self.parse_destination_params(job_destination.params) template.update(destination_params) - template['container']['type'] = 'DOCKER' - template['container']['image'] = self._find_container( - job_wrapper).container_id + template["container"]["type"] = "DOCKER" + template["container"]["image"] = self._find_container(job_wrapper).container_id # Fix the working directory inside the container - template['container']['parameters'] = [{"key": "workdir", "value": job_wrapper.working_directory}] + template["container"]["parameters"] = [{"key": "workdir", "value": job_wrapper.working_directory}] return template def _retrieve_job(self, job_id): jobs = self._chronos_client.list() - job = [x for x in jobs if x['name'] == job_id] + job = [x for x in jobs if x["name"] == job_id] if len(job) > 1: - msg = f'Multiple jobs found with name {job_id!r}' + msg = f"Multiple jobs found with name {job_id!r}" LOGGER.error(msg) raise ChronosRunnerException(msg) return job[0] if job else None diff --git a/lib/galaxy/jobs/runners/cli.py b/lib/galaxy/jobs/runners/cli.py index 0f064e2304b..14605ce7d8c 100644 --- a/lib/galaxy/jobs/runners/cli.py +++ b/lib/galaxy/jobs/runners/cli.py @@ -10,13 +10,17 @@ from galaxy.jobs import JobDestination from galaxy.jobs.runners import ( AsynchronousJobRunner, AsynchronousJobState, - JobState) + JobState, +) from galaxy.util import asbool -from .util.cli import CliInterface, split_params +from .util.cli import ( + CliInterface, + split_params, +) log = logging.getLogger(__name__) -__all__ = ('ShellJobRunner', ) +__all__ = ("ShellJobRunner",) DEFAULT_EMBED_METADATA_IN_JOB = True MAX_SUBMIT_RETRY = 3 @@ -26,10 +30,11 @@ class ShellJobRunner(AsynchronousJobRunner): """ Job runner backed by a finite pool of worker threads. FIFO scheduling """ + runner_name = "ShellRunner" def __init__(self, app, nworkers): - """Start the job runner """ + """Start the job runner""" super().__init__(app, nworkers) self.cli_interface = CliInterface() @@ -39,15 +44,15 @@ class ShellJobRunner(AsynchronousJobRunner): def url_to_destination(self, url): params = {} - shell_params, job_params = url.split('/')[2:4] + shell_params, job_params = url.split("/")[2:4] # split 'foo=bar&baz=quux' into { 'foo' : 'bar', 'baz' : 'quux' } - shell_params = {f"shell_{k}": v for k, v in [kv.split('=', 1) for kv in shell_params.split('&')]} - job_params = {f"job_{k}": v for k, v in [kv.split('=', 1) for kv in job_params.split('&')]} + shell_params = {f"shell_{k}": v for k, v in [kv.split("=", 1) for kv in shell_params.split("&")]} + job_params = {f"job_{k}": v for k, v in [kv.split("=", 1) for kv in job_params.split("&")]} params.update(shell_params) params.update(job_params) log.debug(f"Converted URL '{url}' to destination runner=cli, params={params}") # Create a dynamic JobDestination - return JobDestination(runner='cli', params=params) + return JobDestination(runner="cli", params=params) def parse_destination_params(self, params): return split_params(params) @@ -55,7 +60,9 @@ class ShellJobRunner(AsynchronousJobRunner): def queue_job(self, job_wrapper): """Create job script and submit it to the DRM""" # prepare the job - include_metadata = asbool(job_wrapper.job_destination.params.get("embed_metadata_in_job", DEFAULT_EMBED_METADATA_IN_JOB)) + include_metadata = asbool( + job_wrapper.job_destination.params.get("embed_metadata_in_job", DEFAULT_EMBED_METADATA_IN_JOB) + ) if not self.prepare_job(job_wrapper, include_metadata=include_metadata): return @@ -72,10 +79,7 @@ class ShellJobRunner(AsynchronousJobRunner): job_file_kwargs = job_interface.job_script_kwargs(ajs.output_file, ajs.error_file, ajs.job_name) script = self.get_job_file( - job_wrapper, - exit_code_path=ajs.exit_code_file, - shell=job_wrapper.shell, - **job_file_kwargs + job_wrapper, exit_code_path=ajs.exit_code_file, shell=job_wrapper.shell, **job_file_kwargs ) try: @@ -102,7 +106,7 @@ class ShellJobRunner(AsynchronousJobRunner): # Strip and split to get job ID. external_job_id = stdout.strip().split()[-1] if not external_job_id: - log.error(f'({galaxy_id_tag}) submission did not return a job identifier, failing job') + log.error(f"({galaxy_id_tag}) submission did not return a job identifier, failing job") job_wrapper.fail("failure submitting job") return @@ -113,7 +117,7 @@ class ShellJobRunner(AsynchronousJobRunner): # Store state information for job ajs.job_id = external_job_id - ajs.old_state = 'new' + ajs.old_state = "new" ajs.job_destination = job_destination # Add to our 'queue' of jobs to monitor @@ -129,8 +133,8 @@ class ShellJobRunner(AsynchronousJobRunner): cmd_out = shell.execute(job_interface.submit(job_file)) if cmd_out.returncode == 0: return cmd_out.returncode, cmd_out.stdout - stdout = f'({galaxy_id_tag}) submission failed (stdout): {cmd_out.stdout}' - stderr = f'({galaxy_id_tag}) submission failed (stderr): {cmd_out.stderr}' + stdout = f"({galaxy_id_tag}) submission failed (stdout): {cmd_out.stdout}" + stderr = f"({galaxy_id_tag}) submission failed (stderr): {cmd_out.stderr}" if retry > 0: log.debug("%s, retrying in %s seconds", stdout, timeout) log.debug("%s, retrying in %s seconds", stderr, timeout) @@ -165,7 +169,9 @@ class ShellJobRunner(AsynchronousJobRunner): cmd_out = shell.execute(job_interface.get_single_status(external_job_id)) state = job_interface.parse_single_status(cmd_out.stdout, external_job_id) if not state == model.Job.states.OK: - log.warning(f'({id_tag}/{external_job_id}) job not found in batch state check, but found in individual state check') + log.warning( + f"({id_tag}/{external_job_id}) job not found in batch state check, but found in individual state check" + ) job_state = ajs.job_wrapper.get_state() if state != old_state: log.debug(f"({id_tag}/{external_job_id}) state change: from {old_state} to {state}") @@ -183,10 +189,12 @@ class ShellJobRunner(AsynchronousJobRunner): ajs.running = True ajs.old_state = state if state == model.Job.states.OK or job_state == model.Job.states.STOPPED: - external_metadata = not asbool(ajs.job_wrapper.job_destination.params.get("embed_metadata_in_job", DEFAULT_EMBED_METADATA_IN_JOB)) + external_metadata = not asbool( + ajs.job_wrapper.job_destination.params.get("embed_metadata_in_job", DEFAULT_EMBED_METADATA_IN_JOB) + ) if external_metadata: self.work_queue.put((self.handle_metadata_externally, ajs)) - log.debug(f'({id_tag}/{external_job_id}) job execution finished, running job wrapper finish method') + log.debug(f"({id_tag}/{external_job_id}) job execution finished, running job wrapper finish method") self.work_queue.put((self.finish_job, ajs)) else: new_watched.append(ajs) @@ -201,8 +209,10 @@ class ShellJobRunner(AsynchronousJobRunner): shell, job_interface = self.get_cli_plugins(shell_params, job_params) cmd_out = shell.execute(job_interface.get_failure_reason(external_job_id)) if cmd_out is not None: - if job_interface.parse_failure_reason(cmd_out.stdout, external_job_id) \ - == JobState.runner_states.MEMORY_LIMIT_REACHED: + if ( + job_interface.parse_failure_reason(cmd_out.stdout, external_job_id) + == JobState.runner_states.MEMORY_LIMIT_REACHED + ): ajs.runner_state = JobState.runner_states.MEMORY_LIMIT_REACHED ajs.fail_message = "Tool failed due to insufficient memory. Try with more memory." @@ -212,13 +222,15 @@ class ShellJobRunner(AsynchronousJobRunner): # unique the list of destinations for ajs in self.watched: if ajs.job_destination.id not in job_destinations: - job_destinations[ajs.job_destination.id] = dict(job_destination=ajs.job_destination, job_ids=[ajs.job_id]) + job_destinations[ajs.job_destination.id] = dict( + job_destination=ajs.job_destination, job_ids=[ajs.job_id] + ) else: - job_destinations[ajs.job_destination.id]['job_ids'].append(ajs.job_id) + job_destinations[ajs.job_destination.id]["job_ids"].append(ajs.job_id) # check each destination for the listed job ids for v in job_destinations.values(): - job_destination = v['job_destination'] - job_ids = v['job_ids'] + job_destination = v["job_destination"] + job_ids = v["job_ids"] shell_params, job_params = self.parse_destination_params(job_destination.params) shell, job_interface = self.get_cli_plugins(shell_params, job_params) cmd_out = shell.execute(job_interface.get_status(job_ids)) @@ -236,7 +248,9 @@ class ShellJobRunner(AsynchronousJobRunner): assert cmd_out.returncode == 0, cmd_out.stderr log.debug(f"({job.id}/{job.job_runner_external_id}) Terminated at user's request") except Exception as e: - log.debug(f"({job.id}/{job.job_runner_external_id}) User killed running job, but error encountered during termination: {e}") + log.debug( + f"({job.id}/{job.job_runner_external_id}) User killed running job, but error encountered during termination: {e}" + ) def recover(self, job, job_wrapper): """Recovers jobs stuck in the queued/running state when Galaxy started""" @@ -250,12 +264,16 @@ class ShellJobRunner(AsynchronousJobRunner): ajs.job_wrapper = job_wrapper ajs.job_destination = job_wrapper.job_destination if job.state in (model.Job.states.RUNNING, model.Job.states.STOPPED): - log.debug(f"({job.id}/{job.job_runner_external_id}) is still in {job.state} state, adding to the runner monitor queue") + log.debug( + f"({job.id}/{job.job_runner_external_id}) is still in {job.state} state, adding to the runner monitor queue" + ) ajs.old_state = model.Job.states.RUNNING ajs.running = True self.monitor_queue.put(ajs) elif job.state == model.Job.states.QUEUED: - log.debug(f"({job.id}/{job.job_runner_external_id}) is still in queued state, adding to the runner monitor queue") + log.debug( + f"({job.id}/{job.job_runner_external_id}) is still in queued state, adding to the runner monitor queue" + ) ajs.old_state = model.Job.states.QUEUED ajs.running = False self.monitor_queue.put(ajs) diff --git a/lib/galaxy/jobs/runners/condor.py b/lib/galaxy/jobs/runners/condor.py index 41dcab79a53..da7b5556959 100644 --- a/lib/galaxy/jobs/runners/condor.py +++ b/lib/galaxy/jobs/runners/condor.py @@ -17,20 +17,20 @@ import subprocess from galaxy import model from galaxy.jobs.runners import ( AsynchronousJobRunner, - AsynchronousJobState + AsynchronousJobState, ) from galaxy.jobs.runners.util.condor import ( build_submit_description, condor_stop, condor_submit, submission_params, - summarize_condor_log + summarize_condor_log, ) from galaxy.util import asbool log = logging.getLogger(__name__) -__all__ = ('CondorJobRunner', ) +__all__ = ("CondorJobRunner",) class CondorJobState(AsynchronousJobState): @@ -49,6 +49,7 @@ class CondorJobRunner(AsynchronousJobRunner): """ Job runner backed by a finite pool of worker threads. FIFO scheduling """ + runner_name = "CondorRunner" def queue_job(self, job_wrapper): @@ -68,28 +69,25 @@ class CondorJobRunner(AsynchronousJobRunner): # get destination params query_params = submission_params(prefix="", **job_destination.params) container = None - universe = query_params.get('universe', None) - if universe and universe.strip().lower() == 'docker': + universe = query_params.get("universe", None) + if universe and universe.strip().lower() == "docker": container = self._find_container(job_wrapper) if container: # HTCondor needs the image as 'docker_image' - query_params.update({'docker_image': container.container_id}) + query_params.update({"docker_image": container.container_id}) - galaxy_slots = query_params.get('request_cpus', None) + galaxy_slots = query_params.get("request_cpus", None) if galaxy_slots: galaxy_slots_statement = f'GALAXY_SLOTS="{galaxy_slots}"; export GALAXY_SLOTS; GALAXY_SLOTS_CONFIGURED="1"; export GALAXY_SLOTS_CONFIGURED;' else: galaxy_slots_statement = 'GALAXY_SLOTS="1"; export GALAXY_SLOTS;' # define job attributes - cjs = CondorJobState( - files_dir=job_wrapper.working_directory, - job_wrapper=job_wrapper - ) + cjs = CondorJobState(files_dir=job_wrapper.working_directory, job_wrapper=job_wrapper) - cjs.user_log = os.path.join(job_wrapper.working_directory, f'galaxy_{galaxy_id_tag}.condor.log') - cjs.register_cleanup_file_attribute('user_log') - submit_file = os.path.join(job_wrapper.working_directory, f'galaxy_{galaxy_id_tag}.condor.desc') + cjs.user_log = os.path.join(job_wrapper.working_directory, f"galaxy_{galaxy_id_tag}.condor.log") + cjs.register_cleanup_file_attribute("user_log") + submit_file = os.path.join(job_wrapper.working_directory, f"galaxy_{galaxy_id_tag}.condor.desc") executable = cjs.job_file build_submit_params = dict( @@ -169,7 +167,10 @@ class CondorJobRunner(AsynchronousJobRunner): job_id = cjs.job_id galaxy_id_tag = cjs.job_wrapper.get_id_tag() try: - if cjs.job_wrapper.tool.tool_type != 'interactive' and os.stat(cjs.user_log).st_size == cjs.user_log_size: + if ( + cjs.job_wrapper.tool.tool_type != "interactive" + and os.stat(cjs.user_log).st_size == cjs.user_log_size + ): new_watched.append(cjs) continue s1, s4, s7, s5, s9, log_size = summarize_condor_log(cjs.user_log, job_id) @@ -199,7 +200,9 @@ class CondorJobRunner(AsynchronousJobRunner): job_state = cjs.job_wrapper.get_state() if job_complete or job_state == model.Job.states.STOPPED: if job_state != model.Job.states.DELETED: - external_metadata = not asbool(cjs.job_wrapper.job_destination.params.get("embed_metadata_in_job", True)) + external_metadata = not asbool( + cjs.job_wrapper.job_destination.params.get("embed_metadata_in_job", True) + ) if external_metadata: self._handle_metadata_externally(cjs.job_wrapper, resolve_requirements=True) log.debug(f"({galaxy_id_tag}/{job_id}) job has completed") @@ -236,7 +239,9 @@ class CondorJobRunner(AsynchronousJobRunner): self._stop_container(job_wrapper) # self.watched.append(cjs) if cjs.job_wrapper.get_state() != model.Job.states.DELETED: - external_metadata = not asbool(cjs.job_wrapper.job_destination.params.get("embed_metadata_in_job", True)) + external_metadata = not asbool( + cjs.job_wrapper.job_destination.params.get("embed_metadata_in_job", True) + ) if external_metadata: self._handle_metadata_externally(cjs.job_wrapper, resolve_requirements=True) log.debug(f"({galaxy_id_tag}/{external_id}) job has completed") @@ -268,10 +273,12 @@ class CondorJobRunner(AsynchronousJobRunner): cjs.command_line = job.get_command_line() cjs.job_wrapper = job_wrapper cjs.job_destination = job_wrapper.job_destination - cjs.user_log = os.path.join(job_wrapper.working_directory, f'galaxy_{galaxy_id_tag}.condor.log') - cjs.register_cleanup_file_attribute('user_log') + cjs.user_log = os.path.join(job_wrapper.working_directory, f"galaxy_{galaxy_id_tag}.condor.log") + cjs.register_cleanup_file_attribute("user_log") if job.state in (model.Job.states.RUNNING, model.Job.states.STOPPED): - log.debug(f"({job.id}/{job.get_job_runner_external_id()}) is still in {job.state} state, adding to the DRM queue") + log.debug( + f"({job.id}/{job.get_job_runner_external_id()}) is still in {job.state} state, adding to the DRM queue" + ) cjs.running = True self.monitor_queue.put(cjs) elif job.state == model.Job.states.QUEUED: @@ -280,10 +287,10 @@ class CondorJobRunner(AsynchronousJobRunner): self.monitor_queue.put(cjs) def _stop_container(self, job_wrapper): - return self._run_container_command(job_wrapper, 'stop') + return self._run_container_command(job_wrapper, "stop") def _kill_container(self, job_wrapper): - return self._run_container_command(job_wrapper, 'kill') + return self._run_container_command(job_wrapper, "kill") def _run_container_command(self, job_wrapper, command): job = job_wrapper.get_job() @@ -291,13 +298,15 @@ class CondorJobRunner(AsynchronousJobRunner): if job: cont = job.container if cont: - if cont.container_type == 'docker': - return self._run_command(cont.container_info['commands'][command], external_id)[0] + if cont.container_type == "docker": + return self._run_command(cont.container_info["commands"][command], external_id)[0] def _run_command(self, command, external_job_id): - command = f'condor_ssh_to_job {external_job_id} {command}' + command = f"condor_ssh_to_job {external_job_id} {command}" - p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, close_fds=True, preexec_fn=os.setpgrp) + p = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, close_fds=True, preexec_fn=os.setpgrp + ) stdout, stderr = p.communicate() exit_code = p.returncode ret = None @@ -308,5 +317,5 @@ class CondorJobRunner(AsynchronousJobRunner): # exit_code = subprocess.call(command, # shell=True, # preexec_fn=os.setpgrp) - log.debug('_run_command(%s) exit code (%s) and failure: %s', command, exit_code, stderr) + log.debug("_run_command(%s) exit code (%s) and failure: %s", command, exit_code, stderr) return (exit_code, ret) diff --git a/lib/galaxy/jobs/runners/drmaa.py b/lib/galaxy/jobs/runners/drmaa.py index 081a22fd9fc..903a5688489 100644 --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -14,27 +14,28 @@ from galaxy.jobs import JobDestination from galaxy.jobs.handler import DEFAULT_JOB_PUT_FAILURE_MESSAGE from galaxy.jobs.runners import ( AsynchronousJobRunner, - AsynchronousJobState + AsynchronousJobState, ) from galaxy.util import ( asbool, commands, - unicodify + unicodify, ) drmaa = None log = logging.getLogger(__name__) -__all__ = ('DRMAAJobRunner',) +__all__ = ("DRMAAJobRunner",) -RETRY_EXCEPTIONS_LOWER = frozenset({'invalidjobexception', 'internalexception'}) +RETRY_EXCEPTIONS_LOWER = frozenset({"invalidjobexception", "internalexception"}) class DRMAAJobRunner(AsynchronousJobRunner): """ Job runner backed by a finite pool of worker threads. FIFO scheduling """ + runner_name = "DRMAARunner" restrict_job_name_length = 15 @@ -42,32 +43,37 @@ class DRMAAJobRunner(AsynchronousJobRunner): """Start the job runner""" global drmaa - runner_param_specs = { - 'drmaa_library_path': dict(map=str, default=os.environ.get('DRMAA_LIBRARY_PATH', None))} + runner_param_specs = {"drmaa_library_path": dict(map=str, default=os.environ.get("DRMAA_LIBRARY_PATH", None))} for retry_exception in RETRY_EXCEPTIONS_LOWER: - runner_param_specs[f"{retry_exception}_state"] = dict(map=str, valid=lambda x: x in (model.Job.states.OK, model.Job.states.ERROR), default=model.Job.states.OK) + runner_param_specs[f"{retry_exception}_state"] = dict( + map=str, valid=lambda x: x in (model.Job.states.OK, model.Job.states.ERROR), default=model.Job.states.OK + ) runner_param_specs[f"{retry_exception}_retries"] = dict(map=int, valid=lambda x: int(x) >= 0, default=0) - if 'runner_param_specs' not in kwargs: - kwargs['runner_param_specs'] = dict() - kwargs['runner_param_specs'].update(runner_param_specs) + if "runner_param_specs" not in kwargs: + kwargs["runner_param_specs"] = dict() + kwargs["runner_param_specs"].update(runner_param_specs) super().__init__(app, nworkers, **kwargs) # This allows multiple drmaa runners (although only one per handler) in the same job config file - if 'drmaa_library_path' in kwargs: - log.info('Overriding DRMAA_LIBRARY_PATH due to runner plugin parameter: %s', self.runner_params.drmaa_library_path) - os.environ['DRMAA_LIBRARY_PATH'] = self.runner_params.drmaa_library_path + if "drmaa_library_path" in kwargs: + log.info( + "Overriding DRMAA_LIBRARY_PATH due to runner plugin parameter: %s", + self.runner_params.drmaa_library_path, + ) + os.environ["DRMAA_LIBRARY_PATH"] = self.runner_params.drmaa_library_path # Import is delayed until runner initialization to allow for the # drmaa_library_path plugin param to override $DRMAA_LIBRARY_PATH try: drmaa = __import__("drmaa") except (ImportError, RuntimeError) as exc: - raise exc.__class__('The Python drmaa package is required to use this ' - 'feature, please install it or correct the ' - 'following error:\n%s: %s' % - (exc.__class__.__name__, str(exc))) + raise exc.__class__( + "The Python drmaa package is required to use this " + "feature, please install it or correct the " + "following error:\n%s: %s" % (exc.__class__.__name__, str(exc)) + ) from pulsar.managers.util.drmaa import DrmaaSessionFactory # make the drmaa library also available to subclasses @@ -78,16 +84,16 @@ class DRMAAJobRunner(AsynchronousJobRunner): # Descriptive state strings pulled from the drmaa lib itself self.drmaa_job_state_strings = { - drmaa.JobState.UNDETERMINED: 'process status cannot be determined', - drmaa.JobState.QUEUED_ACTIVE: 'job is queued and active', - drmaa.JobState.SYSTEM_ON_HOLD: 'job is queued and in system hold', - drmaa.JobState.USER_ON_HOLD: 'job is queued and in user hold', - drmaa.JobState.USER_SYSTEM_ON_HOLD: 'job is queued and in user and system hold', - drmaa.JobState.RUNNING: 'job is running', - drmaa.JobState.SYSTEM_SUSPENDED: 'job is system suspended', - drmaa.JobState.USER_SUSPENDED: 'job is user suspended', - drmaa.JobState.DONE: 'job finished normally', - drmaa.JobState.FAILED: 'job finished, but failed', + drmaa.JobState.UNDETERMINED: "process status cannot be determined", + drmaa.JobState.QUEUED_ACTIVE: "job is queued and active", + drmaa.JobState.SYSTEM_ON_HOLD: "job is queued and in system hold", + drmaa.JobState.USER_ON_HOLD: "job is queued and in user hold", + drmaa.JobState.USER_SYSTEM_ON_HOLD: "job is queued and in user and system hold", + drmaa.JobState.RUNNING: "job is running", + drmaa.JobState.SYSTEM_SUSPENDED: "job is system suspended", + drmaa.JobState.USER_SUSPENDED: "job is user suspended", + drmaa.JobState.DONE: "job finished normally", + drmaa.JobState.FAILED: "job finished, but failed", } # Ensure a DRMAA session exists and is initialized @@ -101,19 +107,19 @@ class DRMAAJobRunner(AsynchronousJobRunner): """Convert a legacy URL to a job destination""" if not url: return - native_spec = url.split('/')[2] + native_spec = url.split("/")[2] if native_spec: params = dict(nativeSpecification=native_spec) log.debug(f"Converted URL '{url}' to destination runner=drmaa, params={params}") - return JobDestination(runner='drmaa', params=params) + return JobDestination(runner="drmaa", params=params) else: log.debug(f"Converted URL '{url}' to destination runner=drmaa") - return JobDestination(runner='drmaa') + return JobDestination(runner="drmaa") def get_native_spec(self, url): """Get any native DRM arguments specified by the site configuration""" try: - return url.split('/')[2] or None + return url.split("/")[2] or None except Exception: return None @@ -143,15 +149,15 @@ class DRMAAJobRunner(AsynchronousJobRunner): jobName=ajs.job_name, workingDirectory=job_wrapper.working_directory, outputPath=f":{ajs.output_file}", - errorPath=f":{ajs.error_file}" + errorPath=f":{ajs.error_file}", ) # Avoid a jt.exitCodePath for now - it's only used when finishing. - native_spec = job_destination.params.get('nativeSpecification', None) + native_spec = job_destination.params.get("nativeSpecification", None) if native_spec is None: - native_spec = job_destination.params.get('native_specification', None) + native_spec = job_destination.params.get("native_specification", None) if native_spec is not None: - jt['nativeSpecification'] = native_spec + jt["nativeSpecification"] = native_spec # fill in the DRM's job run template script = self.get_job_file(job_wrapper, exit_code_path=ajs.exit_code_file, shell=job_wrapper.shell) @@ -186,11 +192,11 @@ class DRMAAJobRunner(AsynchronousJobRunner): break except (drmaa.InternalException, drmaa.DeniedByDrmException) as e: trynum += 1 - log.warning('(%s) drmaa.Session.runJob() failed, will retry: %s', galaxy_id_tag, e) + log.warning("(%s) drmaa.Session.runJob() failed, will retry: %s", galaxy_id_tag, e) fail_msg = "Unable to run this job due to a cluster error, please retry it later" time.sleep(5) except Exception: - log.exception('(%s) drmaa.Session.runJob() failed unconditionally', galaxy_id_tag) + log.exception("(%s) drmaa.Session.runJob() failed unconditionally", galaxy_id_tag) trynum = 5 else: log.error(f"({galaxy_id_tag}) All attempts to submit job failed") @@ -205,11 +211,13 @@ class DRMAAJobRunner(AsynchronousJobRunner): pwent = job_wrapper.user_system_pwent if pwent is None: if not allow_guests: - fail_msg = f"User {job_wrapper.user} is not mapped to any real user, and not permitted to start jobs." + fail_msg = ( + f"User {job_wrapper.user} is not mapped to any real user, and not permitted to start jobs." + ) job_wrapper.fail(fail_msg) return pwent = job_wrapper.galaxy_system_pwent - log.debug(f'({galaxy_id_tag}) submitting with credentials: {pwent[0]} [uid: {pwent[2]}]') + log.debug(f"({galaxy_id_tag}) submitting with credentials: {pwent[0]} [uid: {pwent[2]}]") filename = self.store_jobtemplate(job_wrapper, jt) self.userid = pwent[2] external_job_id = self.external_runjob(external_runjob_script, filename, pwent[2]) @@ -223,7 +231,7 @@ class DRMAAJobRunner(AsynchronousJobRunner): # Store DRM related state information for job ajs.job_id = external_job_id - ajs.old_state = 'new' + ajs.old_state = "new" ajs.job_destination = job_destination # Add to our 'queue' of jobs to monitor @@ -277,7 +285,7 @@ class DRMAAJobRunner(AsynchronousJobRunner): galaxy_id_tag = ajs.job_wrapper.get_id_tag() state = None try: - assert external_job_id not in (None, 'None'), f'({galaxy_id_tag}/{external_job_id}) Invalid job id' + assert external_job_id not in (None, "None"), f"({galaxy_id_tag}/{external_job_id}) Invalid job id" state = self.ds.job_status(external_job_id) # Reset exception retries for retry_exception in RETRY_EXCEPTIONS_LOWER: @@ -287,7 +295,14 @@ class DRMAAJobRunner(AsynchronousJobRunner): retry_param = f"{ecn.lower()}_retries" state_param = f"{ecn.lower()}_state" retries = getattr(ajs, retry_param, 0) - log.warning("(%s/%s) unable to check job status because of %s exception for %d consecutive tries: %s", galaxy_id_tag, external_job_id, ecn, retries + 1, e) + log.warning( + "(%s/%s) unable to check job status because of %s exception for %d consecutive tries: %s", + galaxy_id_tag, + external_job_id, + ecn, + retries + 1, + e, + ) if self.runner_params[retry_param] > 0: if retries < self.runner_params[retry_param]: # will retry check on next iteration @@ -301,7 +316,11 @@ class DRMAAJobRunner(AsynchronousJobRunner): log.warning("(%s/%s) job will now be errored", galaxy_id_tag, external_job_id) self.work_queue.put((self.fail_job, ajs)) else: - raise Exception("%s is set to an invalid value (%s), this should not be possible. See galaxy.jobs.drmaa.__init__()", state_param, self.runner_params[state_param]) + raise Exception( + "%s is set to an invalid value (%s), this should not be possible. See galaxy.jobs.drmaa.__init__()", + state_param, + self.runner_params[state_param], + ) return None except drmaa.DrmCommunicationException as e: log.warning("(%s/%s) unable to communicate with DRM: %s", galaxy_id_tag, external_job_id, e) @@ -356,7 +375,7 @@ class DRMAAJobRunner(AsynchronousJobRunner): job = job_wrapper.get_job() try: ext_id = job.get_job_runner_external_id() - assert ext_id not in (None, 'None'), 'External job id is None' + assert ext_id not in (None, "None"), "External job id is None" kill_script = job_wrapper.get_destination_configuration("drmaa_external_killjob_script") if kill_script is None: self.ds.kill(ext_id) @@ -384,28 +403,32 @@ class DRMAAJobRunner(AsynchronousJobRunner): ajs.job_wrapper = job_wrapper ajs.job_destination = job_wrapper.job_destination if job.state in (model.Job.states.RUNNING, model.Job.states.STOPPED): - log.debug(f"({job.id}/{job.get_job_runner_external_id()}) is still in {job.state} state, adding to the DRM queue") + log.debug( + f"({job.id}/{job.get_job_runner_external_id()}) is still in {job.state} state, adding to the DRM queue" + ) ajs.old_state = drmaa.JobState.RUNNING ajs.running = True self.monitor_queue.put(ajs) elif job.get_state() == model.Job.states.QUEUED: - log.debug(f"({job.id}/{job.get_job_runner_external_id()}) is still in DRM queued state, adding to the DRM queue") + log.debug( + f"({job.id}/{job.get_job_runner_external_id()}) is still in DRM queued state, adding to the DRM queue" + ) ajs.old_state = drmaa.JobState.QUEUED_ACTIVE ajs.running = False self.monitor_queue.put(ajs) def store_jobtemplate(self, job_wrapper, jt): - """ Stores the content of a DRMAA JobTemplate object in a file as a JSON string. + """Stores the content of a DRMAA JobTemplate object in a file as a JSON string. Path is hard-coded, but it's no worse than other path in this module. Uses Galaxy's JobID, so file is expected to be unique.""" filename = f"{self.app.config.cluster_files_directory}/{job_wrapper.get_id_tag()}.jt_json" - with open(filename, 'w+') as fp: + with open(filename, "w+") as fp: json.dump(jt, fp) - log.debug(f'({job_wrapper.job_id}) Job script for external submission is: {filename}') + log.debug(f"({job_wrapper.job_id}) Job script for external submission is: {filename}") return filename def external_runjob(self, external_runjob_script, jobtemplate_filename, username): - """ runs an external script that will QSUB a new job. + """runs an external script that will QSUB a new job. The external script needs to be run with sudo, and will setuid() to the specified user. Effectively, will QSUB as a different user (than the one used by Galaxy). """ @@ -429,12 +452,12 @@ class DRMAAJobRunner(AsynchronousJobRunner): galaxy_id_tag = job_wrapper.get_id_tag() # define job attributes - job_name = f'g{galaxy_id_tag}' + job_name = f"g{galaxy_id_tag}" if job_wrapper.tool.old_id: - job_name += f'_{job_wrapper.tool.old_id}' + job_name += f"_{job_wrapper.tool.old_id}" if not self.redact_email_in_job_name and external_runjob_script is None: - job_name += f'_{job_wrapper.user}' - job_name = ''.join(x if x in (f"{string.ascii_letters + string.digits}_") else '_' for x in job_name) + job_name += f"_{job_wrapper.user}" + job_name = "".join(x if x in (f"{string.ascii_letters + string.digits}_") else "_" for x in job_name) if self.restrict_job_name_length: - job_name = job_name[:self.restrict_job_name_length] + job_name = job_name[: self.restrict_job_name_length] return job_name diff --git a/lib/galaxy/jobs/runners/godocker.py b/lib/galaxy/jobs/runners/godocker.py index db499c9da30..69ef5ec61bc 100644 --- a/lib/galaxy/jobs/runners/godocker.py +++ b/lib/galaxy/jobs/runners/godocker.py @@ -8,17 +8,16 @@ import requests from galaxy import model from galaxy.jobs.runners import ( AsynchronousJobRunner, - AsynchronousJobState + AsynchronousJobState, ) from galaxy.util import ( DEFAULT_SOCKET_TIMEOUT, unicodify, ) - log = logging.getLogger(__name__) -__all__ = ('GodockerJobRunner', ) +__all__ = ("GodockerJobRunner",) class Godocker: @@ -37,7 +36,7 @@ class Godocker: self.token = token def http_post_request(self, query, data, header): - """ post request with query """ + """post request with query""" verify_ssl = not self.noCert try: @@ -45,13 +44,13 @@ class Godocker: res = requests.post(url, data, headers=header, verify=verify_ssl, timeout=DEFAULT_SOCKET_TIMEOUT) except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as e: - log.error('A transport error occurred in the GoDocker job runner:', e) + log.error("A transport error occurred in the GoDocker job runner:", e) return False return self.test_status_code(res) def http_get_request(self, query, header): - """ get request with query, server and header required """ + """get request with query, server and header required""" # remove warnings if using --no-certificate requests.packages.urllib3.disable_warnings() @@ -61,13 +60,13 @@ class Godocker: res = requests.get(url, headers=header, verify=verify_ssl, timeout=DEFAULT_SOCKET_TIMEOUT) except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as e: - log.error('A communication error occurred in the GoDocker job runner:', e) + log.error("A communication error occurred in the GoDocker job runner:", e) return False return self.test_status_code(res) def http_delete_request(self, query, header): - """ delete request with query, server and header required """ + """delete request with query, server and header required""" # remove warnings if using --no-certificate requests.packages.urllib3.disable_warnings() @@ -77,13 +76,13 @@ class Godocker: res = requests.delete(url, headers=header, verify=verify_ssl, timeout=DEFAULT_SOCKET_TIMEOUT) except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as e: - log.error('A communication error occurred in the GoDocker job runner:', e) + log.error("A communication error occurred in the GoDocker job runner:", e) return False return self.test_status_code(res) def http_put_request(self, query, data, header): - """ put request with query """ + """put request with query""" # remove warnings if using --no-certificate requests.packages.urllib3.disable_warnings() @@ -93,21 +92,23 @@ class Godocker: res = requests.put(url, data, headers=header, verify=verify_ssl, timeout=DEFAULT_SOCKET_TIMEOUT) except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as e: - log.error('A communication error occurred in the GoDocker job runner:', e) + log.error("A communication error occurred in the GoDocker job runner:", e) return False return self.test_status_code(res) def test_status_code(self, httpresult): - """ exit if status code is 401 or 403 or 404 or 200""" + """exit if status code is 401 or 403 or 404 or 200""" if httpresult.status_code == 401: - log.debug('Unauthorized : this server could not verify that you are authorized to access the document you requested.') + log.debug( + "Unauthorized : this server could not verify that you are authorized to access the document you requested." + ) elif httpresult.status_code == 403: - log.debug('Forbidden : Access was denied to this resource. Not authorized to access this resource.') + log.debug("Forbidden : Access was denied to this resource. Not authorized to access this resource.") elif httpresult.status_code == 404: - log.debug('Not Found : The resource could not be found.') + log.debug("Not Found : The resource could not be found.") elif httpresult.status_code == 200: return httpresult @@ -119,28 +120,35 @@ class GodockerJobRunner(AsynchronousJobRunner): """ Job runner backed by a finite pool of worker threads. FIFO scheduling """ + runner_name = "GodockerJobRunner" def __init__(self, app, nworkers, **kwargs): - """ 1: Get runner_param_specs from job_conf.xml - 2: Initialise job runner parent object - 3: Login to godocker and store the token - 4: Start the worker and monitor threads + """1: Get runner_param_specs from job_conf.xml + 2: Initialise job runner parent object + 3: Login to godocker and store the token + 4: Start the worker and monitor threads """ - runner_param_specs = dict(godocker_master=dict(map=str), user=dict(map=str), key=dict(map=str), godocker_project=dict(map=str)) - if 'runner_param_specs' not in kwargs: - kwargs['runner_param_specs'] = dict() - kwargs['runner_param_specs'].update(runner_param_specs) + runner_param_specs = dict( + godocker_master=dict(map=str), user=dict(map=str), key=dict(map=str), godocker_project=dict(map=str) + ) + if "runner_param_specs" not in kwargs: + kwargs["runner_param_specs"] = dict() + kwargs["runner_param_specs"].update(runner_param_specs) # Start the job runner parent object super().__init__(app, nworkers, **kwargs) # godocker API login call - self.auth = self.login(self.runner_params["key"], self.runner_params["user"], self.runner_params["godocker_master"]) + self.auth = self.login( + self.runner_params["key"], self.runner_params["user"], self.runner_params["godocker_master"] + ) def queue_job(self, job_wrapper): - """ Create job script and submit it to godocker """ - if not self.prepare_job(job_wrapper, include_metadata=False, include_work_dir_outputs=True, modify_command_for_container=False): + """Create job script and submit it to godocker""" + if not self.prepare_job( + job_wrapper, include_metadata=False, include_work_dir_outputs=True, modify_command_for_container=False + ): return job_destination = job_wrapper.job_destination @@ -152,18 +160,23 @@ class GodockerJobRunner(AsynchronousJobRunner): else: log.debug(f"Starting queue_job for job {job_id}") # Create an object of AsynchronousJobState and add it to the monitor queue. - ajs = AsynchronousJobState(files_dir=job_wrapper.working_directory, job_wrapper=job_wrapper, job_id=job_id, job_destination=job_destination) + ajs = AsynchronousJobState( + files_dir=job_wrapper.working_directory, + job_wrapper=job_wrapper, + job_id=job_id, + job_destination=job_destination, + ) self.monitor_queue.put(ajs) def check_watched_item(self, job_state): - """ Get the job current status from GoDocker - using job_id and update the status in galaxy. - If the job execution is successful, call - mark_as_finished() and return 'None' to galaxy. - else if the job failed, call mark_as_failed() - and return 'None' to galaxy. - else if the job is running or in pending state, simply - return the 'AsynchronousJobState object' (job_state). + """Get the job current status from GoDocker + using job_id and update the status in galaxy. + If the job execution is successful, call + mark_as_finished() and return 'None' to galaxy. + else if the job failed, call mark_as_failed() + and return 'None' to galaxy. + else if the job is running or in pending state, simply + return the 'AsynchronousJobState object' (job_state). """ # This function is called by check_watched_items() where param job_state # is an object of AsynchronousJobState. @@ -175,14 +188,14 @@ class GodockerJobRunner(AsynchronousJobRunner): job_status_god = self.get_task(job_state.job_id) log.debug(f"Job ID: {str(job_state.job_id)} Job Status: {str(job_status_god['status']['primary'])}") - if job_status_god['status']['primary'] == "over" or job_persisted_state == model.Job.states.STOPPED: + if job_status_god["status"]["primary"] == "over" or job_persisted_state == model.Job.states.STOPPED: job_state.running = False job_state.job_wrapper.change_state(model.Job.states.OK) if self.create_log_file(job_state, job_status_god): self.mark_as_finished(job_state) else: self.mark_as_failed(job_state) - '''The function mark_as_finished() executes: + """The function mark_as_finished() executes: self.work_queue.put((self.finish_job, job_state)) *self.finish_job -> job_state.job_wrapper.finish( stdout, stderr, exit_code ) @@ -196,18 +209,18 @@ class GodockerJobRunner(AsynchronousJobRunner): job_state.job_wrapper.finish( stdout, stderr, exit_code ) job_state.job_wrapper.fail( "Unable to finish job", exception=True) *Similar workflow is done for mark_as_failed() method. - ''' + """ return None - elif job_status_god['status']['primary'] == "running": + elif job_status_god["status"]["primary"] == "running": job_state.running = True job_state.job_wrapper.change_state(model.Job.states.RUNNING) return job_state - elif job_status_god['status']['primary'] == "pending": + elif job_status_god["status"]["primary"] == "pending": return job_state - elif job_status_god['status']['exitcode'] not in [None, 0] and job_persisted_state != model.Job.states.STOPPED: + elif job_status_god["status"]["exitcode"] not in [None, 0] and job_persisted_state != model.Job.states.STOPPED: job_state.running = False job_state.job_wrapper.change_state(model.Job.states.ERROR) self.create_log_file(job_state, job_status_god) @@ -221,7 +234,7 @@ class GodockerJobRunner(AsynchronousJobRunner): return None def stop_job(self, job_wrapper): - """ Attempts to delete a dispatched executing Job in GoDocker """ + """Attempts to delete a dispatched executing Job in GoDocker""" # This function is called by fail_job() where # param job = self.sa_session.query(self.app.model.Job).get(job_state.job_wrapper.job_id) # No Return data expected @@ -229,13 +242,13 @@ class GodockerJobRunner(AsynchronousJobRunner): log.debug(f"STOP JOB EXECUTION OF JOB ID: {str(job_id)}") # Get task status from GoDocker. job_status_god = self.get_task_status(job_id) - if job_status_god['status']['primary'] != "over": + if job_status_god["status"]["primary"] != "over": # Initiate a delete call,if the job is running in GoDocker. self.delete_task(job_id) return None def recover(self, job, job_wrapper): - """ Recovers jobs stuck in the queued/running state when Galaxy started """ + """Recovers jobs stuck in the queued/running state when Galaxy started""" # This method is called by Galaxy at startup time. # Jobs in Running & Queued state in galaxy are put in the monitor_queue # by creating an AsynchronousJobState object @@ -246,27 +259,31 @@ class GodockerJobRunner(AsynchronousJobRunner): job_wrapper.command_line = job.command_line ajs.job_wrapper = job_wrapper if job.state in (model.Job.states.RUNNING, model.Job.states.STOPPED): - log.debug(f"({job.id}/{job.get_job_runner_external_id()}) is still in {job.state} state, adding to the god queue") - ajs.old_state = 'R' + log.debug( + f"({job.id}/{job.get_job_runner_external_id()}) is still in {job.state} state, adding to the god queue" + ) + ajs.old_state = "R" ajs.running = True self.monitor_queue.put(ajs) elif job.state == model.Job.states.QUEUED: - log.debug(f"({job.id}/{job.get_job_runner_external_id()}) is still in god queued state, adding to the god queue") - ajs.old_state = 'Q' + log.debug( + f"({job.id}/{job.get_job_runner_external_id()}) is still in god queued state, adding to the god queue" + ) + ajs.old_state = "Q" ajs.running = False self.monitor_queue.put(ajs) # Helper functions def create_log_file(self, job_state, job_status_god): - """ Create log files in galaxy, namely error_file, output_file, exit_code_file - Return true, if all the file creations are successful + """Create log files in galaxy, namely error_file, output_file, exit_code_file + Return true, if all the file creations are successful """ path = None - for vol in job_status_god['container']['volumes']: - if vol['name'] == "go-docker": - path = str(vol['path']) + for vol in job_status_god["container"]["volumes"]: + if vol["name"] == "go-docker": + path = str(vol["path"]) if path: god_output_file = f"{path}/god.log" god_error_file = f"{path}/god.err" @@ -286,7 +303,7 @@ class GodockerJobRunner(AsynchronousJobRunner): log_file.close() f.close() # Read from GoDocker exit_code and write it into galaxy exit_code_file. - out_log = str(job_status_god['status']['exitcode']) + out_log = str(job_status_god["status"]["exitcode"]) log_file = open(job_state.exit_code_file, "w") log_file.write(out_log) log_file.close() @@ -295,7 +312,7 @@ class GodockerJobRunner(AsynchronousJobRunner): log.debug(f"CREATE ERROR FILE: {job_state.error_file}") log.debug(f"CREATE EXIT CODE FILE: {job_state.exit_code_file}") except OSError as e: - log.error('Could not access task log file: %s', unicodify(e)) + log.error("Could not access task log file: %s", unicodify(e)) log.debug("IO Error occurred when accessing the files.") return False return True @@ -303,27 +320,29 @@ class GodockerJobRunner(AsynchronousJobRunner): # GoDocker API helper functions def login(self, apikey, login, server, noCert=False): - """ Login to GoDocker and return the token - Create Login model schema of GoDocker and call the http_post_request method. + """Login to GoDocker and return the token + Create Login model schema of GoDocker and call the http_post_request method. """ log.debug("LOGIN TASK TO BE EXECUTED \n") log.debug(f"GODOCKER LOGIN: {str(login)}") - data = json.dumps({'user': login, 'apikey': apikey}) + data = json.dumps({"user": login, "apikey": apikey}) # Create object of Godocker class g_auth = Godocker(server, login, apikey, noCert) - auth = g_auth.http_post_request("/api/1.0/authenticate", data, {'Content-type': 'application/json', 'Accept': 'application/json'}) + auth = g_auth.http_post_request( + "/api/1.0/authenticate", data, {"Content-type": "application/json", "Accept": "application/json"} + ) if not auth: raise Exception("Authentication failure, GoDocker runner cannot be started") else: log.debug("GoDocker authentication successful.") - token = auth.json()['token'] + token = auth.json()["token"] g_auth.setToken(token) # Return the object of Godocker class return g_auth def post_task(self, job_wrapper): - """ Sumbit job to GoDocker and return jobid - Create Job model schema of GoDocker and call the http_post_request method. + """Sumbit job to GoDocker and return jobid + Create Job model schema of GoDocker and call the http_post_request method. """ # Get the params from tag in job_conf by using job_destination.params[param] if self.auth.token: @@ -345,7 +364,7 @@ class GodockerJobRunner(AsynchronousJobRunner): volumes = [] labels = [] - tags_tab = ['galaxy', job_wrapper.tool.id] + tags_tab = ["galaxy", job_wrapper.tool.id] tasks_depends = [] name = job_wrapper.tool.name description = "galaxy job" @@ -367,10 +386,12 @@ class GodockerJobRunner(AsynchronousJobRunner): dt = datetime.now() # Enable galaxy venv in the docker containers try: - if(job_destination.params["virtualenv"] == "true"): + if job_destination.params["virtualenv"] == "true": GALAXY_VENV_TEMPLATE = """GALAXY_VIRTUAL_ENV="%s"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi;""" venv = GALAXY_VENV_TEMPLATE % job_wrapper.galaxy_virtual_env - command = f"#!/bin/bash\ncd {job_wrapper.working_directory}\n{venv}\n{job_wrapper.runner_command_line}" + command = ( + f"#!/bin/bash\ncd {job_wrapper.working_directory}\n{venv}\n{job_wrapper.runner_command_line}" + ) else: command = f"#!/bin/bash\ncd {job_wrapper.working_directory}\n{job_wrapper.runner_command_line}" except Exception: @@ -378,89 +399,95 @@ class GodockerJobRunner(AsynchronousJobRunner): # GoDocker Job model schema job = { - 'date': time.mktime(dt.timetuple()), - 'meta': { - 'name': name, - 'description': description, - 'tags': tags_tab + "date": time.mktime(dt.timetuple()), + "meta": {"name": name, "description": description, "tags": tags_tab}, + "requirements": { + "cpu": docker_cpu, + "ram": docker_ram, + "array": {"values": array}, + "label": labels, + "tasks": tasks_depends, + "tmpstorage": None, }, - 'requirements': { - 'cpu': docker_cpu, - 'ram': docker_ram, - 'array': {'values': array}, - 'label': labels, - 'tasks': tasks_depends, - 'tmpstorage': None + "container": { + "image": str(docker_image), + "volumes": volumes, + "network": True, + "id": None, + "meta": None, + "stats": None, + "ports": [], + "root": False, }, - 'container': { - 'image': str(docker_image), - 'volumes': volumes, - 'network': True, - 'id': None, - 'meta': None, - 'stats': None, - 'ports': [], - 'root': False + "command": { + "interactive": False, + "cmd": command, }, - 'command': { - 'interactive': False, - 'cmd': command, - }, - 'status': { - 'primary': None, - 'secondary': None - } + "status": {"primary": None, "secondary": None}, } if project is not None: - job['user'] = {"project": project} + job["user"] = {"project": project} result = self.auth.http_post_request( - "/api/1.0/task", json.dumps(job), - {'Authorization': f"Bearer {self.auth.token}", 'Content-type': 'application/json', 'Accept': 'application/json'} + "/api/1.0/task", + json.dumps(job), + { + "Authorization": f"Bearer {self.auth.token}", + "Content-type": "application/json", + "Accept": "application/json", + }, ) # Return job_id - return str(result.json()['id']) + return str(result.json()["id"]) def get_task(self, job_id): - """ Get job details from GoDocker and return the job. - Pass job_id to the http_get_request method. + """Get job details from GoDocker and return the job. + Pass job_id to the http_get_request method. """ job = False if self.auth.token: - result = self.auth.http_get_request(f"/api/1.0/task/{str(job_id)}", {'Authorization': f"Bearer {self.auth.token}"}) + result = self.auth.http_get_request( + f"/api/1.0/task/{str(job_id)}", {"Authorization": f"Bearer {self.auth.token}"} + ) job = result.json() # Return the job return job def task_suspend(self, job_id): - """ Suspend actively running job in galaxy. - Pass job_id to the http_get_request method. + """Suspend actively running job in galaxy. + Pass job_id to the http_get_request method. """ job = False if self.auth.token: - result = self.auth.http_get_request(f"/api/1.0/task/{str(job_id)}/suspend", {'Authorization': f"Bearer {self.auth.token}"}) + result = self.auth.http_get_request( + f"/api/1.0/task/{str(job_id)}/suspend", {"Authorization": f"Bearer {self.auth.token}"} + ) job = result.json() # Return the job return job def get_task_status(self, job_id): - """ Get job status from GoDocker and return the status of job. - Pass job_id to http_get_request method. + """Get job status from GoDocker and return the status of job. + Pass job_id to http_get_request method. """ job = False if self.auth.token: - result = self.auth.http_get_request(f"/api/1.0/task/{str(job_id)}/status", {'Authorization': f"Bearer {self.auth.token}"}) + result = self.auth.http_get_request( + f"/api/1.0/task/{str(job_id)}/status", {"Authorization": f"Bearer {self.auth.token}"} + ) job = result.json() # Return task status return job def delete_task(self, job_id): - """ Delete a suspended task in GoDocker. - Pass job_id to http_delete_request method. + """Delete a suspended task in GoDocker. + Pass job_id to http_delete_request method. """ job = False if self.auth.token: - result = self.auth.http_delete_request(f"/api/1.0/task/{str(job_id)}", {'Authorization': f"Bearer {self.auth.token}"}) + result = self.auth.http_delete_request( + f"/api/1.0/task/{str(job_id)}", {"Authorization": f"Bearer {self.auth.token}"} + ) job = result.json() # Return the job return job diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index 505e1e0b7bd..5c8886a3e63 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -14,7 +14,7 @@ from galaxy import model from galaxy.jobs.runners import ( AsynchronousJobRunner, AsynchronousJobState, - JobState + JobState, ) from galaxy.jobs.runners.util.pykube_util import ( deduplicate_entries, @@ -41,19 +41,20 @@ from galaxy.jobs.runners.util.pykube_util import ( pull_policy, pykube_client_from_dict, Service, - service_object_dict + service_object_dict, ) from galaxy.util.bytesize import ByteSize log = logging.getLogger(__name__) -__all__ = ('KubernetesJobRunner', ) +__all__ = ("KubernetesJobRunner",) class KubernetesJobRunner(AsynchronousJobRunner): """ Job runner backed by a finite pool of worker threads. FIFO scheduling """ + runner_name = "KubernetesRunner" LABEL_START = re.compile("^[A-Za-z0-9]") @@ -80,21 +81,30 @@ class KubernetesJobRunner(AsynchronousJobRunner): k8s_job_api_version=dict(map=str, default=DEFAULT_JOB_API_VERSION), k8s_job_ttl_secs_after_finished=dict(map=int, valid=lambda x: x is None or int(x) >= 0, default=None), k8s_job_metadata=dict(map=str, default=None), - k8s_supplemental_group_id=dict(map=str, valid=lambda s: s == "$gid" or isinstance(s, int) or not s or s.isdigit(), default=None), + k8s_supplemental_group_id=dict( + map=str, valid=lambda s: s == "$gid" or isinstance(s, int) or not s or s.isdigit(), default=None + ), k8s_pull_policy=dict(map=str, default="Default"), - k8s_run_as_user_id=dict(map=str, valid=lambda s: s == "$uid" or isinstance(s, int) or not s or s.isdigit(), default=None), - k8s_run_as_group_id=dict(map=str, valid=lambda s: s == "$gid" or isinstance(s, int) or not s or s.isdigit(), default=None), - k8s_fs_group_id=dict(map=str, valid=lambda s: s == "$gid" or isinstance(s, int) or not s or s.isdigit(), default=None), + k8s_run_as_user_id=dict( + map=str, valid=lambda s: s == "$uid" or isinstance(s, int) or not s or s.isdigit(), default=None + ), + k8s_run_as_group_id=dict( + map=str, valid=lambda s: s == "$gid" or isinstance(s, int) or not s or s.isdigit(), default=None + ), + k8s_fs_group_id=dict( + map=str, valid=lambda s: s == "$gid" or isinstance(s, int) or not s or s.isdigit(), default=None + ), k8s_cleanup_job=dict(map=str, valid=lambda s: s in {"onsuccess", "always", "never"}, default="always"), k8s_pod_retries=dict(map=int, valid=lambda x: int(x) >= 0, default=3), k8s_walltime_limit=dict(map=int, valid=lambda x: int(x) >= 0, default=172800), k8s_unschedulable_walltime_limit=dict(map=int, valid=lambda x: not x or int(x) >= 0, default=None), k8s_interactivetools_use_ssl=dict(map=bool, default=False), - k8s_interactivetools_ingress_annotations=dict(map=str),) + k8s_interactivetools_ingress_annotations=dict(map=str), + ) - if 'runner_param_specs' not in kwargs: - kwargs['runner_param_specs'] = dict() - kwargs['runner_param_specs'].update(runner_param_specs) + if "runner_param_specs" not in kwargs: + kwargs["runner_param_specs"] = dict() + kwargs["runner_param_specs"].update(runner_param_specs) # Start the job runner parent object super().__init__(app, nworkers, **kwargs) @@ -111,23 +121,23 @@ class KubernetesJobRunner(AsynchronousJobRunner): self.setup_base_volumes() def setup_base_volumes(self): - def generate_volumes(pvc_list): - return [{'name': pvc["name"], 'persistentVolumeClaim': {'claimName': pvc["name"]}} for pvc in pvc_list] + return [{"name": pvc["name"], "persistentVolumeClaim": {"claimName": pvc["name"]}} for pvc in pvc_list] def get_volume_mounts_for(claim): if self.runner_params.get(claim): - volume_mounts = [parse_pvc_param_line(pvc) for pvc in self.runner_params[claim].split(',')] + volume_mounts = [parse_pvc_param_line(pvc) for pvc in self.runner_params[claim].split(",")] # generate default list of volumes for all jobs volumes = generate_volumes(volume_mounts) return volumes, volume_mounts return [], [] - self.runner_params['k8s_volumes'], self.runner_params['k8s_volume_mounts'] = \ - get_volume_mounts_for('k8s_persistent_volume_claims') + self.runner_params["k8s_volumes"], self.runner_params["k8s_volume_mounts"] = get_volume_mounts_for( + "k8s_persistent_volume_claims" + ) # ignore volume mounts for the following two, as they are generated per job - self.runner_params['k8s_volumes'].extend(get_volume_mounts_for('k8s_data_volume_claim')[0]) - self.runner_params['k8s_volumes'].extend(get_volume_mounts_for('k8s_working_volume_claim')[0]) + self.runner_params["k8s_volumes"].extend(get_volume_mounts_for("k8s_data_volume_claim")[0]) + self.runner_params["k8s_volumes"].extend(get_volume_mounts_for("k8s_working_volume_claim")[0]) def queue_job(self, job_wrapper): """Create job script and submit it to Kubernetes cluster""" @@ -136,18 +146,24 @@ class KubernetesJobRunner(AsynchronousJobRunner): # where galaxy will expect results. log.debug(f"Starting queue_job for job {job_wrapper.get_id_tag()}") - ajs = AsynchronousJobState(files_dir=job_wrapper.working_directory, - job_wrapper=job_wrapper, - job_destination=job_wrapper.job_destination) + ajs = AsynchronousJobState( + files_dir=job_wrapper.working_directory, + job_wrapper=job_wrapper, + job_destination=job_wrapper.job_destination, + ) - if not self.prepare_job(job_wrapper, - include_metadata=False, - modify_command_for_container=False, - stdout_file=ajs.output_file, - stderr_file=ajs.error_file): + if not self.prepare_job( + job_wrapper, + include_metadata=False, + modify_command_for_container=False, + stdout_file=ajs.output_file, + stderr_file=ajs.error_file, + ): return - script = self.get_job_file(job_wrapper, exit_code_path=ajs.exit_code_file, shell=job_wrapper.shell, galaxy_virtual_env=None) + script = self.get_job_file( + job_wrapper, exit_code_path=ajs.exit_code_file, shell=job_wrapper.shell, galaxy_virtual_env=None + ) try: self.write_executable_script(ajs.job_file, script, job_io=job_wrapper.job_io) except Exception: @@ -167,11 +183,7 @@ class KubernetesJobRunner(AsynchronousJobRunner): return k8s_job_prefix = self.__produce_k8s_job_prefix() - k8s_job_obj = job_object_dict( - self.runner_params, - k8s_job_prefix, - self.__get_k8s_job_spec(ajs) - ) + k8s_job_obj = job_object_dict(self.runner_params, k8s_job_prefix, self.__get_k8s_job_spec(ajs)) job = Job(self._pykube_api, k8s_job_obj) try: @@ -204,24 +216,16 @@ class KubernetesJobRunner(AsynchronousJobRunner): guest_ports = ajs.job_wrapper.guest_ports ports_dict = {} for guest_port in guest_ports: - ports_dict[str(guest_port)] = dict(host='manual', port=guest_port, protocol="https") + ports_dict[str(guest_port)] = dict(host="manual", port=guest_port, protocol="https") self.app.interactivetool_manager.configure_entry_points(ajs.job_wrapper.get_job(), ports_dict) # Configure additional k8s service and ingress for tools with guest ports k8s_job_prefix = self.__produce_k8s_job_prefix() k8s_job_name = self.__get_k8s_job_name(k8s_job_prefix, ajs.job_wrapper) - log.debug(f'Configuring entry points and deploying service/ingress for job with ID {ajs.job_id}') - k8s_service_obj = service_object_dict( - self.runner_params, - k8s_job_name, - self.__get_k8s_service_spec(ajs) - ) + log.debug(f"Configuring entry points and deploying service/ingress for job with ID {ajs.job_id}") + k8s_service_obj = service_object_dict(self.runner_params, k8s_job_name, self.__get_k8s_service_spec(ajs)) - k8s_ingress_obj = ingress_object_dict( - self.runner_params, - k8s_job_name, - self.__get_k8s_ingress_spec(ajs) - ) + k8s_ingress_obj = ingress_object_dict(self.runner_params, k8s_job_name, self.__get_k8s_ingress_spec(ajs)) service = Service(self._pykube_api, k8s_service_obj) service.create() ingress = Ingress(self._pykube_api, k8s_ingress_obj) @@ -243,8 +247,11 @@ class KubernetesJobRunner(AsynchronousJobRunner): try: return int(self.runner_params["k8s_run_as_user_id"]) except Exception: - log.warning("User ID passed for Kubernetes runner needs to be an integer or \"$uid\", value " - + self.runner_params["k8s_run_as_user_id"] + " passed is invalid") + log.warning( + 'User ID passed for Kubernetes runner needs to be an integer or "$uid", value ' + + self.runner_params["k8s_run_as_user_id"] + + " passed is invalid" + ) return None return None @@ -257,17 +264,26 @@ class KubernetesJobRunner(AsynchronousJobRunner): try: return int(self.runner_params["k8s_run_as_group_id"]) except Exception: - log.warning("Group ID passed for Kubernetes runner needs to be an integer or \"$gid\", value " - + self.runner_params["k8s_run_as_group_id"] + " passed is invalid") + log.warning( + 'Group ID passed for Kubernetes runner needs to be an integer or "$gid", value ' + + self.runner_params["k8s_run_as_group_id"] + + " passed is invalid" + ) return None def __get_supplemental_group(self): - if self.runner_params.get("k8s_supplemental_group_id") or self.runner_params.get("k8s_supplemental_group_id") == 0: + if ( + self.runner_params.get("k8s_supplemental_group_id") + or self.runner_params.get("k8s_supplemental_group_id") == 0 + ): try: return int(self.runner_params["k8s_supplemental_group_id"]) except Exception: - log.warning("Supplemental group passed for Kubernetes runner needs to be an integer or \"$gid\", value " - + self.runner_params["k8s_supplemental_group_id"] + " passed is invalid") + log.warning( + 'Supplemental group passed for Kubernetes runner needs to be an integer or "$gid", value ' + + self.runner_params["k8s_supplemental_group_id"] + + " passed is invalid" + ) return None return None @@ -276,8 +292,11 @@ class KubernetesJobRunner(AsynchronousJobRunner): try: return int(self.runner_params["k8s_fs_group_id"]) except Exception: - log.warning("FS group passed for Kubernetes runner needs to be an integer or \"$gid\", value " - + self.runner_params["k8s_fs_group_id"] + " passed is invalid") + log.warning( + 'FS group passed for Kubernetes runner needs to be an integer or "$gid", value ' + + self.runner_params["k8s_fs_group_id"] + + " passed is invalid" + ) return None return None @@ -286,14 +305,16 @@ class KubernetesJobRunner(AsynchronousJobRunner): return galaxy_instance_id(self.runner_params) def __produce_k8s_job_prefix(self): - instance_id = self._galaxy_instance_id or '' - return produce_k8s_job_prefix(app_prefix='gxy', instance_id=instance_id) + instance_id = self._galaxy_instance_id or "" + return produce_k8s_job_prefix(app_prefix="gxy", instance_id=instance_id) def __get_k8s_job_spec(self, ajs): """Creates the k8s Job spec. For a Job spec, the only requirement is to have a .spec.template. If the job hangs around unlimited it will be ended after k8s wall time limit, which sets activeDeadlineSeconds""" - k8s_job_spec = {"template": self.__get_k8s_job_spec_template(ajs), - "activeDeadlineSeconds": int(self.runner_params['k8s_walltime_limit'])} + k8s_job_spec = { + "template": self.__get_k8s_job_spec_template(ajs), + "activeDeadlineSeconds": int(self.runner_params["k8s_walltime_limit"]), + } job_ttl = self.runner_params["k8s_job_ttl_secs_after_finished"] if self.runner_params["k8s_cleanup_job"] != "never" and job_ttl is not None: k8s_job_spec["ttlSecondsAfterFinished"] = job_ttl @@ -310,7 +331,7 @@ class KubernetesJobRunner(AsynchronousJobRunner): if not self.LABEL_START.search(label_val): label_val = f"x{label_val}" if not self.LABEL_END.search(label_val): - label_val += 'x' + label_val += "x" return label_val def __get_k8s_job_spec_template(self, ajs): @@ -329,32 +350,31 @@ class KubernetesJobRunner(AsynchronousJobRunner): "app.galaxyproject.org/job_id": self.__force_label_conformity(ajs.job_wrapper.get_id_tag()), "app.galaxyproject.org/handler": self.__force_label_conformity(self.app.config.server_name), "app.galaxyproject.org/destination": self.__force_label_conformity( - str(ajs.job_wrapper.job_destination.id)) + str(ajs.job_wrapper.job_destination.id) + ), }, - "annotations": { - "app.galaxyproject.org/tool_id": ajs.job_wrapper.tool.id - } + "annotations": {"app.galaxyproject.org/tool_id": ajs.job_wrapper.tool.id}, }, "spec": { - "volumes": deduplicate_entries(self.runner_params['k8s_volumes']), + "volumes": deduplicate_entries(self.runner_params["k8s_volumes"]), "restartPolicy": self.__get_k8s_restart_policy(ajs.job_wrapper), "containers": self.__get_k8s_containers(ajs), - "priorityClassName": self.runner_params['k8s_pod_priority_class'], - "tolerations": yaml.safe_load(self.runner_params['k8s_tolerations'] or "[]"), - "affinity": yaml.safe_load(self.__get_overridable_params(ajs.job_wrapper, - 'k8s_affinity') or "{}"), - "nodeSelector": yaml.safe_load(self.__get_overridable_params(ajs.job_wrapper, - 'k8s_node_selector') or "{}") - } + "priorityClassName": self.runner_params["k8s_pod_priority_class"], + "tolerations": yaml.safe_load(self.runner_params["k8s_tolerations"] or "[]"), + "affinity": yaml.safe_load(self.__get_overridable_params(ajs.job_wrapper, "k8s_affinity") or "{}"), + "nodeSelector": yaml.safe_load( + self.__get_overridable_params(ajs.job_wrapper, "k8s_node_selector") or "{}" + ), + }, } # TODO include other relevant elements that people might want to use from # TODO http://kubernetes.io/docs/api-reference/v1/definitions/#_v1_podspec k8s_spec_template["spec"]["securityContext"] = self.__get_k8s_security_context() - extra_metadata = self.runner_params['k8s_job_metadata'] or '{}' + extra_metadata = self.runner_params["k8s_job_metadata"] or "{}" if isinstance(extra_metadata, str): extra_metadata = yaml.safe_load(extra_metadata) - k8s_spec_template["metadata"]["labels"].update(extra_metadata.get('labels', {})) - k8s_spec_template["metadata"]["annotations"].update(extra_metadata.get('annotations', {})) + k8s_spec_template["metadata"]["labels"].update(extra_metadata.get("labels", {})) + k8s_spec_template["metadata"]["annotations"].update(extra_metadata.get("annotations", {})) return k8s_spec_template def __get_k8s_service_spec(self, ajs): @@ -366,24 +386,28 @@ class KubernetesJobRunner(AsynchronousJobRunner): "labels": { "app.galaxyproject.org/handler": self.__force_label_conformity(self.app.config.server_name), "app.galaxyproject.org/destination": self.__force_label_conformity( - str(ajs.job_wrapper.job_destination.id)) + str(ajs.job_wrapper.job_destination.id) + ), }, - "annotations": { - "app.galaxyproject.org/tool_id": ajs.job_wrapper.tool.id - } + "annotations": {"app.galaxyproject.org/tool_id": ajs.job_wrapper.tool.id}, }, "spec": { - "ports": [{"name": f"job-{self.__force_label_conformity(ajs.job_wrapper.get_id_tag())}-{p}", - "port": int(p), - "protocol": "TCP", - "targetPort": int(p)} for p in guest_ports], + "ports": [ + { + "name": f"job-{self.__force_label_conformity(ajs.job_wrapper.get_id_tag())}-{p}", + "port": int(p), + "protocol": "TCP", + "targetPort": int(p), + } + for p in guest_ports + ], "selector": { "app.kubernetes.io/name": self.__force_label_conformity(ajs.job_wrapper.tool.old_id), "app.kubernetes.io/component": "tool", - "app.galaxyproject.org/job_id": self.__force_label_conformity(ajs.job_wrapper.get_id_tag()) + "app.galaxyproject.org/job_id": self.__force_label_conformity(ajs.job_wrapper.get_id_tag()), }, - "type": "ClusterIP" - } + "type": "ClusterIP", + }, } return k8s_spec_template @@ -397,45 +421,61 @@ class KubernetesJobRunner(AsynchronousJobRunner): for entry_point in configured_eps: # sending in self.app as `trans` since it's only used for `.security` so seems to work entry_point_path = self.app.interactivetool_manager.get_entry_point_path(self.app, entry_point) - if '?' in entry_point_path: + if "?" in entry_point_path: # Removing all the parameters from the ingress path, but they will still be in the database # so the link that the user clicks on will still have them - log.warning("IT urls including parameters (eg: /myit?mykey=myvalue) are only experimentally supported on K8S") - entry_point_path = entry_point_path.split('?')[0] - entry_point_domain = f'{self.app.config.interactivetools_proxy_host}' + log.warning( + "IT urls including parameters (eg: /myit?mykey=myvalue) are only experimentally supported on K8S" + ) + entry_point_path = entry_point_path.split("?")[0] + entry_point_domain = f"{self.app.config.interactivetools_proxy_host}" if entry_point.requires_domain: - entry_point_subdomain = self.app.interactivetool_manager.get_entry_point_subdomain(self.app, entry_point) - entry_point_domain = f'{entry_point_subdomain}.{entry_point_domain}' - entry_point_path = '/' - entry_points.append({"tool_port": entry_point.tool_port, "domain": entry_point_domain, "entry_path": entry_point_path}) + entry_point_subdomain = self.app.interactivetool_manager.get_entry_point_subdomain( + self.app, entry_point + ) + entry_point_domain = f"{entry_point_subdomain}.{entry_point_domain}" + entry_point_path = "/" + entry_points.append( + {"tool_port": entry_point.tool_port, "domain": entry_point_domain, "entry_path": entry_point_path} + ) k8s_spec_template = { "metadata": { "labels": { "app.galaxyproject.org/handler": self.__force_label_conformity(self.app.config.server_name), "app.galaxyproject.org/destination": self.__force_label_conformity( - str(ajs.job_wrapper.job_destination.id)) + str(ajs.job_wrapper.job_destination.id) + ), }, - "annotations": { - "app.galaxyproject.org/tool_id": ajs.job_wrapper.tool.id - } + "annotations": {"app.galaxyproject.org/tool_id": ajs.job_wrapper.tool.id}, }, "spec": { - "rules": [{"host": ep["domain"], - "http": { - "paths": [{ - "backend": { - "serviceName": self.__get_k8s_job_name(self.__produce_k8s_job_prefix(), ajs.job_wrapper), - "servicePort": int(ep["tool_port"]) - }, - "path": ep.get("entry_path", '/'), - "pathType": "Prefix" - }]}} for ep in entry_points] - } + "rules": [ + { + "host": ep["domain"], + "http": { + "paths": [ + { + "backend": { + "serviceName": self.__get_k8s_job_name( + self.__produce_k8s_job_prefix(), ajs.job_wrapper + ), + "servicePort": int(ep["tool_port"]), + }, + "path": ep.get("entry_path", "/"), + "pathType": "Prefix", + } + ] + }, + } + for ep in entry_points + ] + }, } if self.runner_params.get("k8s_interactivetools_use_ssl"): domains = list({e["domain"] for e in entry_points}) - k8s_spec_template["spec"]["tls"] = [{"hosts": [domain], - "secretName": re.sub("[^a-z0-9-]", "-", domain)} for domain in domains] + k8s_spec_template["spec"]["tls"] = [ + {"hosts": [domain], "secretName": re.sub("[^a-z0-9-]", "-", domain)} for domain in domains + ] if self.runner_params.get("k8s_interactivetools_ingress_annotations"): new_ann = yaml.safe_load(self.runner_params.get("k8s_interactivetools_ingress_annotations")) k8s_spec_template["metadata"]["annotations"].update(new_ann) @@ -459,16 +499,20 @@ class KubernetesJobRunner(AsynchronousJobRunner): def __get_k8s_containers(self, ajs): """Fills in all required for setting up the docker containers to be used, including setting a pull policy if - this has been set. - $GALAXY_VIRTUAL_ENV is set to None to avoid the galaxy virtualenv inside the tool container. - $GALAXY_LIB is set to None to avoid changing the python path inside the container. - Setting these variables changes the described behaviour in the job file shell script - used to execute the tool inside the container. + this has been set. + $GALAXY_VIRTUAL_ENV is set to None to avoid the galaxy virtualenv inside the tool container. + $GALAXY_LIB is set to None to avoid changing the python path inside the container. + Setting these variables changes the described behaviour in the job file shell script + used to execute the tool inside the container. """ container = self._find_container(ajs.job_wrapper) - mounts = get_volume_mounts_for_job(ajs.job_wrapper, self.runner_params.get('k8s_data_volume_claim'), self.runner_params.get('k8s_working_volume_claim')) - mounts.extend(self.runner_params['k8s_volume_mounts']) + mounts = get_volume_mounts_for_job( + ajs.job_wrapper, + self.runner_params.get("k8s_data_volume_claim"), + self.runner_params.get("k8s_working_volume_claim"), + ) + mounts.extend(self.runner_params["k8s_volume_mounts"]) k8s_container = { "name": self.__get_k8s_container_name(ajs.job_wrapper), @@ -479,56 +523,60 @@ class KubernetesJobRunner(AsynchronousJobRunner): "command": [ajs.job_wrapper.shell], "args": ["-c", ajs.job_file], "workingDir": ajs.job_wrapper.working_directory, - "volumeMounts": deduplicate_entries(mounts) + "volumeMounts": deduplicate_entries(mounts), } resources = self.__get_resources(ajs.job_wrapper) if resources: envs = [] cpu_val = None - if 'requests' in resources: - requests = resources['requests'] - if 'cpu' in requests: - cpu_val = int(math.ceil(float(requests['cpu']))) - envs.append({'name': 'GALAXY_SLOTS', 'value': str(cpu_val)}) - if 'memory' in requests: - mem_val = ByteSize(requests['memory']).to_unit('M', as_string=False) - envs.append({'name': 'GALAXY_MEMORY_MB', 'value': str(mem_val)}) + if "requests" in resources: + requests = resources["requests"] + if "cpu" in requests: + cpu_val = int(math.ceil(float(requests["cpu"]))) + envs.append({"name": "GALAXY_SLOTS", "value": str(cpu_val)}) + if "memory" in requests: + mem_val = ByteSize(requests["memory"]).to_unit("M", as_string=False) + envs.append({"name": "GALAXY_MEMORY_MB", "value": str(mem_val)}) if cpu_val: - envs.append({'name': 'GALAXY_MEMORY_MB_PER_SLOT', 'value': str(math.floor(mem_val / cpu_val))}) - elif 'limits' in resources: - limits = resources['limits'] - if 'cpu' in limits: - cpu_val = int(math.floor(float(limits['cpu']))) + envs.append({"name": "GALAXY_MEMORY_MB_PER_SLOT", "value": str(math.floor(mem_val / cpu_val))}) + elif "limits" in resources: + limits = resources["limits"] + if "cpu" in limits: + cpu_val = int(math.floor(float(limits["cpu"]))) cpu_val = cpu_val or 1 - envs.append({'name': 'GALAXY_SLOTS', 'value': str(cpu_val)}) - if 'memory' in limits: - mem_val = ByteSize(limits['memory']).to_unit('M', as_string=False) - envs.append({'name': 'GALAXY_MEMORY_MB', 'value': str(mem_val)}) + envs.append({"name": "GALAXY_SLOTS", "value": str(cpu_val)}) + if "memory" in limits: + mem_val = ByteSize(limits["memory"]).to_unit("M", as_string=False) + envs.append({"name": "GALAXY_MEMORY_MB", "value": str(mem_val)}) if cpu_val: - envs.append({'name': 'GALAXY_MEMORY_MB_PER_SLOT', 'value': str(math.floor(mem_val / cpu_val))}) - extra_envs = yaml.safe_load(self.__get_overridable_params(ajs.job_wrapper, 'k8s_extra_job_envs') or "{}") + envs.append({"name": "GALAXY_MEMORY_MB_PER_SLOT", "value": str(math.floor(mem_val / cpu_val))}) + extra_envs = yaml.safe_load(self.__get_overridable_params(ajs.job_wrapper, "k8s_extra_job_envs") or "{}") for key in extra_envs: - envs.append({'name': key, 'value': extra_envs[key]}) + envs.append({"name": key, "value": extra_envs[key]}) if self.__has_guest_ports(ajs.job_wrapper): configured_eps = [ep for ep in ajs.job_wrapper.get_job().interactivetool_entry_points if ep.configured] for entry_point in configured_eps: # sending in self.app as `trans` since it's only used for `.security` so seems to work entry_point_path = self.app.interactivetool_manager.get_entry_point_path(self.app, entry_point) - if '?' in entry_point_path: + if "?" in entry_point_path: # Removing all the parameters from the ingress path, but they will still be in the database # so the link that the user clicks on will still have them - log.warning("IT urls including parameters (eg: /myit?mykey=myvalue) are only experimentally supported on K8S") - entry_point_path = entry_point_path.split('?')[0] - entry_point_domain = f'{self.app.config.interactivetools_proxy_host}' + log.warning( + "IT urls including parameters (eg: /myit?mykey=myvalue) are only experimentally supported on K8S" + ) + entry_point_path = entry_point_path.split("?")[0] + entry_point_domain = f"{self.app.config.interactivetools_proxy_host}" if entry_point.requires_domain: - entry_point_subdomain = self.app.interactivetool_manager.get_entry_point_subdomain(self.app, entry_point) - entry_point_domain = f'{entry_point_subdomain}.{entry_point_domain}' - envs.append({'name': 'INTERACTIVETOOL_PORT', 'value': str(entry_point.tool_port)}) - envs.append({'name': 'INTERACTIVETOOL_DOMAIN', 'value': str(entry_point_domain)}) - envs.append({'name': 'INTERACTIVETOOL_PATH', 'value': str(entry_point_path)}) - k8s_container['resources'] = resources - k8s_container['env'] = envs + entry_point_subdomain = self.app.interactivetool_manager.get_entry_point_subdomain( + self.app, entry_point + ) + entry_point_domain = f"{entry_point_subdomain}.{entry_point_domain}" + envs.append({"name": "INTERACTIVETOOL_PORT", "value": str(entry_point.tool_port)}) + envs.append({"name": "INTERACTIVETOOL_DOMAIN", "value": str(entry_point_domain)}) + envs.append({"name": "INTERACTIVETOOL_PATH", "value": str(entry_point_path)}) + k8s_container["resources"] = resources + k8s_container["env"] = envs if self._default_pull_policy: k8s_container["imagePullPolicy"] = self._default_pull_policy @@ -546,20 +594,20 @@ class KubernetesJobRunner(AsynchronousJobRunner): limits = {} if mem_request: - requests['memory'] = mem_request + requests["memory"] = mem_request if cpu_request: - requests['cpu'] = cpu_request + requests["cpu"] = cpu_request if mem_limit: - limits['memory'] = mem_limit + limits["memory"] = mem_limit if cpu_limit: - limits['cpu'] = cpu_limit + limits["cpu"] = cpu_limit resources = {} if requests: - resources['requests'] = requests + resources["requests"] = requests if limits: - resources['limits'] = limits + resources["limits"] = limits return resources @@ -567,32 +615,32 @@ class KubernetesJobRunner(AsynchronousJobRunner): """Obtains memory requests for job, checking if available on the destination, otherwise using the default""" job_destination = job_wrapper.job_destination - if 'requests_memory' in job_destination.params: - return self.__transform_memory_value(job_destination.params['requests_memory']) + if "requests_memory" in job_destination.params: + return self.__transform_memory_value(job_destination.params["requests_memory"]) return None def __get_memory_limit(self, job_wrapper): """Obtains memory limits for job, checking if available on the destination, otherwise using the default""" job_destination = job_wrapper.job_destination - if 'limits_memory' in job_destination.params: - return self.__transform_memory_value(job_destination.params['limits_memory']) + if "limits_memory" in job_destination.params: + return self.__transform_memory_value(job_destination.params["limits_memory"]) return None def __get_cpu_request(self, job_wrapper): """Obtains cpu requests for job, checking if available on the destination, otherwise using the default""" job_destination = job_wrapper.job_destination - if 'requests_cpu' in job_destination.params: - return job_destination.params['requests_cpu'] + if "requests_cpu" in job_destination.params: + return job_destination.params["requests_cpu"] return None def __get_cpu_limit(self, job_wrapper): """Obtains cpu requests for job, checking if available on the destination, otherwise using the default""" job_destination = job_wrapper.job_destination - if 'limits_cpu' in job_destination.params: - return job_destination.params['limits_cpu'] + if "limits_cpu" in job_destination.params: + return job_destination.params["limits_cpu"] return None def __transform_memory_value(self, mem_value): @@ -609,14 +657,14 @@ class KubernetesJobRunner(AsynchronousJobRunner): # definition repo = "" owner = "" - if 'repo' in job_destination.params: + if "repo" in job_destination.params: repo = f"{job_destination.params['repo']}/" - if 'owner' in job_destination.params: + if "owner" in job_destination.params: owner = f"{job_destination.params['owner']}/" - k8s_cont_image = repo + owner + job_destination.params['image'] + k8s_cont_image = repo + owner + job_destination.params["image"] - if 'tag' in job_destination.params: + if "tag" in job_destination.params: k8s_cont_image += f":{job_destination.params['tag']}" return k8s_cont_image @@ -646,19 +694,19 @@ class KubernetesJobRunner(AsynchronousJobRunner): def check_watched_item(self, job_state): """Checks the state of a job already submitted on k8s. Job state is an AsynchronousJobState""" - jobs = find_job_object_by_name(self._pykube_api, job_state.job_id, self.runner_params['k8s_namespace']) + jobs = find_job_object_by_name(self._pykube_api, job_state.job_id, self.runner_params["k8s_namespace"]) - if len(jobs.response['items']) == 1: - job = Job(self._pykube_api, jobs.response['items'][0]) + if len(jobs.response["items"]) == 1: + job = Job(self._pykube_api, jobs.response["items"][0]) job_destination = job_state.job_wrapper.job_destination succeeded = 0 active = 0 failed = 0 - if 'max_pod_retries' in job_destination.params: - max_pod_retries = int(job_destination.params['max_pod_retries']) - elif 'k8s_pod_retries' in self.runner_params: - max_pod_retries = int(self.runner_params['k8s_pod_retries']) + if "max_pod_retries" in job_destination.params: + max_pod_retries = int(job_destination.params["max_pod_retries"]) + elif "k8s_pod_retries" in self.runner_params: + max_pod_retries = int(self.runner_params["k8s_pod_retries"]) else: max_pod_retries = 1 @@ -667,14 +715,14 @@ class KubernetesJobRunner(AsynchronousJobRunner): # as probably this means that the k8s API server hasn't # had time to fill in the object status since the # job was created only too recently. - if len(job.obj['status']) == 0: + if len(job.obj["status"]) == 0: return job_state - if 'succeeded' in job.obj['status']: - succeeded = job.obj['status']['succeeded'] - if 'active' in job.obj['status']: - active = job.obj['status']['active'] - if 'failed' in job.obj['status']: - failed = job.obj['status']['failed'] + if "succeeded" in job.obj["status"]: + succeeded = job.obj["status"]["succeeded"] + if "active" in job.obj["status"]: + active = job.obj["status"]["active"] + if "failed" in job.obj["status"]: + failed = job.obj["status"]["failed"] job_persisted_state = job_state.job_wrapper.get_state() @@ -686,11 +734,11 @@ class KubernetesJobRunner(AsynchronousJobRunner): elif active > 0 and failed <= max_pod_retries: if not job_state.running: if self.__job_pending_due_to_unschedulable_pod(job_state): - if self.runner_params.get('k8s_unschedulable_walltime_limit'): - creation_time_str = job.obj['metadata'].get('creationTimestamp') - creation_time = datetime.strptime(creation_time_str, '%Y-%m-%dT%H:%M:%SZ') + if self.runner_params.get("k8s_unschedulable_walltime_limit"): + creation_time_str = job.obj["metadata"].get("creationTimestamp") + creation_time = datetime.strptime(creation_time_str, "%Y-%m-%dT%H:%M:%SZ") elapsed_seconds = (datetime.utcnow() - creation_time).total_seconds() - if elapsed_seconds > self.runner_params['k8s_unschedulable_walltime_limit']: + if elapsed_seconds > self.runner_params["k8s_unschedulable_walltime_limit"]: return self._handle_unschedulable_job(job, job_state) else: pass @@ -709,7 +757,7 @@ class KubernetesJobRunner(AsynchronousJobRunner): else: return self._handle_job_failure(job, job_state) - elif len(jobs.response['items']) == 0: + elif len(jobs.response["items"]) == 0: if job_state.job_wrapper.get_job().state == model.Job.states.DELETED: # Job has been deleted via stop_job and job has been deleted, # cleanup and remove from watched_jobs by returning `None` @@ -744,7 +792,7 @@ class KubernetesJobRunner(AsynchronousJobRunner): def _handle_job_failure(self, job, job_state): # Figure out why job has failed - with open(job_state.error_file, 'a') as error_file: + with open(job_state.error_file, "a") as error_file: if self.__job_failed_due_to_low_memory(job_state): error_file.write("Job killed after running out of memory. Try with more memory.\n") job_state.fail_message = "Tool failed due to insufficient memory. Try with more memory." @@ -767,28 +815,29 @@ class KubernetesJobRunner(AsynchronousJobRunner): return None def __cleanup_k8s_job(self, job): - k8s_cleanup_job = self.runner_params['k8s_cleanup_job'] + k8s_cleanup_job = self.runner_params["k8s_cleanup_job"] delete_job(job, k8s_cleanup_job) def __cleanup_k8s_ingress(self, ingress, job_failed): - k8s_cleanup_job = self.runner_params['k8s_cleanup_job'] + k8s_cleanup_job = self.runner_params["k8s_cleanup_job"] delete_ingress(ingress, k8s_cleanup_job, job_failed) def __cleanup_k8s_service(self, service, job_failed): - k8s_cleanup_job = self.runner_params['k8s_cleanup_job'] + k8s_cleanup_job = self.runner_params["k8s_cleanup_job"] delete_service(service, k8s_cleanup_job, job_failed) def __job_failed_due_to_walltime_limit(self, job): - conditions = job.obj['status'].get('conditions') or [] - return any(True for c in conditions if c['type'] == 'Failed' and c['reason'] == 'DeadlineExceeded') + conditions = job.obj["status"].get("conditions") or [] + return any(True for c in conditions if c["type"] == "Failed" and c["reason"] == "DeadlineExceeded") def _get_pod_for_job(self, job_state): - pods = Pod.objects(self._pykube_api).filter(selector=f"app={job_state.job_id}", - namespace=self.runner_params['k8s_namespace']) - if not pods.response['items']: + pods = Pod.objects(self._pykube_api).filter( + selector=f"app={job_state.job_id}", namespace=self.runner_params["k8s_namespace"] + ) + if not pods.response["items"]: return None - pod = Pod(self._pykube_api, pods.response['items'][0]) + pod = Pod(self._pykube_api, pods.response["items"][0]) return pod def __job_failed_due_to_low_memory(self, job_state): @@ -797,13 +846,16 @@ class KubernetesJobRunner(AsynchronousJobRunner): for being out of memory (pod status OOMKilled). If that is the case marks the job for resubmission (resubmit logic is part of destinations). """ - pods = find_pod_object_by_name(self._pykube_api, job_state.job_id, self.runner_params['k8s_namespace']) - if not pods.response['items']: + pods = find_pod_object_by_name(self._pykube_api, job_state.job_id, self.runner_params["k8s_namespace"]) + if not pods.response["items"]: return False pod = self._get_pod_for_job(job_state) - if pod and pod.obj['status']['phase'] == "Failed" and \ - pod.obj['status']['containerStatuses'][0]['state']['terminated']['reason'] == "OOMKilled": + if ( + pod + and pod.obj["status"]["phase"] == "Failed" + and pod.obj["status"]["containerStatuses"][0]["state"]["terminated"]["reason"] == "OOMKilled" + ): return True return False @@ -812,35 +864,40 @@ class KubernetesJobRunner(AsynchronousJobRunner): """ checks the state of the pod to see if it is unschedulable. """ - pods = find_pod_object_by_name(self._pykube_api, job_state.job_id, self.runner_params['k8s_namespace']) - if not pods.response['items']: + pods = find_pod_object_by_name(self._pykube_api, job_state.job_id, self.runner_params["k8s_namespace"]) + if not pods.response["items"]: return False - pod = Pod(self._pykube_api, pods.response['items'][0]) - return is_pod_unschedulable(self._pykube_api, pod, self.runner_params['k8s_namespace']) + pod = Pod(self._pykube_api, pods.response["items"][0]) + return is_pod_unschedulable(self._pykube_api, pod, self.runner_params["k8s_namespace"]) def __cleanup_k8s_guest_ports(self, job_wrapper, k8s_job): k8s_job_prefix = self.__produce_k8s_job_prefix() k8s_job_name = f"{k8s_job_prefix}-{self.__force_label_conformity(job_wrapper.get_id_tag())}" - log.debug(f'Deleting service/ingress for job with ID {job_wrapper.get_id_tag()}') - job_failed = (k8s_job.obj['status']['failed'] > 0 - if 'failed' in k8s_job.obj['status'] else False) - ingress_to_delete = find_ingress_object_by_name(self._pykube_api, k8s_job_name, self.runner_params['k8s_namespace']) - if ingress_to_delete and len(ingress_to_delete.response['items']) > 0: - k8s_ingress = Ingress(self._pykube_api, ingress_to_delete.response['items'][0]) + log.debug(f"Deleting service/ingress for job with ID {job_wrapper.get_id_tag()}") + job_failed = k8s_job.obj["status"]["failed"] > 0 if "failed" in k8s_job.obj["status"] else False + ingress_to_delete = find_ingress_object_by_name( + self._pykube_api, k8s_job_name, self.runner_params["k8s_namespace"] + ) + if ingress_to_delete and len(ingress_to_delete.response["items"]) > 0: + k8s_ingress = Ingress(self._pykube_api, ingress_to_delete.response["items"][0]) self.__cleanup_k8s_ingress(k8s_ingress, job_failed) - service_to_delete = find_service_object_by_name(self._pykube_api, k8s_job_name, self.runner_params['k8s_namespace']) - if service_to_delete and len(service_to_delete.response['items']) > 0: - k8s_service = Service(self._pykube_api, service_to_delete.response['items'][0]) + service_to_delete = find_service_object_by_name( + self._pykube_api, k8s_job_name, self.runner_params["k8s_namespace"] + ) + if service_to_delete and len(service_to_delete.response["items"]) > 0: + k8s_service = Service(self._pykube_api, service_to_delete.response["items"][0]) self.__cleanup_k8s_service(k8s_service, job_failed) def stop_job(self, job_wrapper): """Attempts to delete a dispatched job to the k8s cluster""" job = job_wrapper.get_job() try: - job_to_delete = find_job_object_by_name(self._pykube_api, job.get_job_runner_external_id(), self.runner_params['k8s_namespace']) - if job_to_delete and len(job_to_delete.response['items']) > 0: - k8s_job = Job(self._pykube_api, job_to_delete.response['items'][0]) + job_to_delete = find_job_object_by_name( + self._pykube_api, job.get_job_runner_external_id(), self.runner_params["k8s_namespace"] + ) + if job_to_delete and len(job_to_delete.response["items"]) > 0: + k8s_job = Job(self._pykube_api, job_to_delete.response["items"][0]) if self.__has_guest_ports(job_wrapper): self.__cleanup_k8s_guest_ports(job_wrapper, k8s_job) self.__cleanup_k8s_job(k8s_job) @@ -849,8 +906,11 @@ class KubernetesJobRunner(AsynchronousJobRunner): log.debug(f"({job.id}/{job.job_runner_external_id}) Terminated at user's request") except Exception as e: - log.exception("({}/{}) User killed running job, but error encountered during termination: {}".format( - job.id, job.get_job_runner_external_id(), e)) + log.exception( + "({}/{}) User killed running job, but error encountered during termination: {}".format( + job.id, job.get_job_runner_external_id(), e + ) + ) def recover(self, job, job_wrapper): """Recovers jobs stuck in the queued/running state when Galaxy started""" @@ -865,14 +925,20 @@ class KubernetesJobRunner(AsynchronousJobRunner): ajs.job_wrapper = job_wrapper ajs.job_destination = job_wrapper.job_destination if job.state in (model.Job.states.RUNNING, model.Job.states.STOPPED): - log.debug("({}/{}) is still in {} state, adding to the runner monitor queue".format( - job.id, job.job_runner_external_id, job.state)) + log.debug( + "({}/{}) is still in {} state, adding to the runner monitor queue".format( + job.id, job.job_runner_external_id, job.state + ) + ) ajs.old_state = model.Job.states.RUNNING ajs.running = True self.monitor_queue.put(ajs) elif job.state == model.Job.states.QUEUED: - log.debug("({}/{}) is still in queued state, adding to the runner monitor queue".format( - job.id, job.job_runner_external_id)) + log.debug( + "({}/{}) is still in queued state, adding to the runner monitor queue".format( + job.id, job.job_runner_external_id + ) + ) ajs.old_state = model.Job.states.QUEUED ajs.running = False self.monitor_queue.put(ajs) @@ -880,14 +946,16 @@ class KubernetesJobRunner(AsynchronousJobRunner): def finish_job(self, job_state): self._handle_metadata_externally(job_state.job_wrapper, resolve_requirements=True) super().finish_job(job_state) - jobs = find_job_object_by_name(self._pykube_api, job_state.job_id, self.runner_params['k8s_namespace']) - if len(jobs.response['items']) > 1: - log.warning("More than one job matches selector: %s. Possible configuration error" - " in job id '%s'" % (jobs.response['items'], job_state.job_id)) - elif len(jobs.response['items']) == 0: + jobs = find_job_object_by_name(self._pykube_api, job_state.job_id, self.runner_params["k8s_namespace"]) + if len(jobs.response["items"]) > 1: + log.warning( + "More than one job matches selector: %s. Possible configuration error" + " in job id '%s'" % (jobs.response["items"], job_state.job_id) + ) + elif len(jobs.response["items"]) == 0: log.warning("No k8s job found which matches job id '%s'. Ignoring...", job_state.job_id) else: - job = Job(self._pykube_api, jobs.response['items'][0]) + job = Job(self._pykube_api, jobs.response["items"][0]) if self.__has_guest_ports(job_state.job_wrapper): self.__cleanup_k8s_guest_ports(job_state.job_wrapper, job) self.__cleanup_k8s_job(job) diff --git a/lib/galaxy/jobs/runners/local.py b/lib/galaxy/jobs/runners/local.py index 395ab3ceeef..c0e9cb4fdca 100644 --- a/lib/galaxy/jobs/runners/local.py +++ b/lib/galaxy/jobs/runners/local.py @@ -11,22 +11,20 @@ from time import sleep from galaxy import model from galaxy.job_execution.output_collect import default_exit_code_file -from galaxy.util import ( - asbool, -) +from galaxy.util import asbool from galaxy.util.commands import new_clean_env from . import ( BaseJobRunner, - JobState + JobState, ) from .util.process_groups import ( check_pg, - kill_pg + kill_pg, ) log = logging.getLogger(__name__) -__all__ = ('LocalJobRunner', ) +__all__ = ("LocalJobRunner",) DEFAULT_POOL_SLEEP_TIME = 1 # TODO: Set to false and just get rid of this option. It would simplify this @@ -38,10 +36,11 @@ class LocalJobRunner(BaseJobRunner): """ Job runner backed by a finite pool of worker threads. FIFO scheduling """ + runner_name = "LocalRunner" def __init__(self, app, nworkers): - """Start the job runner """ + """Start the job runner""" self._proc_lock = threading.Lock() self._procs = [] @@ -51,8 +50,7 @@ class LocalJobRunner(BaseJobRunner): super().__init__(app, nworkers) def __command_line(self, job_wrapper): - """ - """ + """ """ command_line = job_wrapper.runner_command_line # slots would be cleaner name, but don't want deployers to see examples and think it @@ -67,11 +65,11 @@ class LocalJobRunner(BaseJobRunner): job_file = JobState.default_job_file(job_wrapper.working_directory, job_id) exit_code_path = default_exit_code_file(job_wrapper.working_directory, job_id) job_script_props = { - 'slots_statement': slots_statement, - 'command': command_line, - 'exit_code_path': exit_code_path, - 'working_directory': job_wrapper.working_directory, - 'shell': job_wrapper.shell, + "slots_statement": slots_statement, + "command": command_line, + "exit_code_path": exit_code_path, + "working_directory": job_wrapper.working_directory, + "shell": job_wrapper.shell, } job_file_contents = self.get_job_file(job_wrapper, **job_script_props) self.write_executable_script(job_file, job_file_contents, job_io=job_wrapper.job_io) @@ -81,26 +79,28 @@ class LocalJobRunner(BaseJobRunner): if not self._prepare_job_local(job_wrapper): return - stderr = stdout = '' + stderr = stdout = "" # command line has been added to the wrapper by prepare_job() job_file, exit_code_path = self.__command_line(job_wrapper) job_id = job_wrapper.get_id_tag() try: - stdout_file = tempfile.NamedTemporaryFile(mode='wb+', suffix='_stdout', dir=job_wrapper.working_directory) - stderr_file = tempfile.NamedTemporaryFile(mode='wb+', suffix='_stderr', dir=job_wrapper.working_directory) - log.debug(f'({job_id}) executing job script: {job_file}') + stdout_file = tempfile.NamedTemporaryFile(mode="wb+", suffix="_stdout", dir=job_wrapper.working_directory) + stderr_file = tempfile.NamedTemporaryFile(mode="wb+", suffix="_stderr", dir=job_wrapper.working_directory) + log.debug(f"({job_id}) executing job script: {job_file}") # The preexec_fn argument of Popen() is used to call os.setpgrp() in # the child process just before the child is executed. This will set # the PGID of the child process to its PID (i.e. ensures that it is # the root of its own process group instead of Galaxy's one). - proc = subprocess.Popen(args=[job_file], - cwd=job_wrapper.working_directory, - stdout=stdout_file, - stderr=stderr_file, - env=self._environ, - preexec_fn=os.setpgrp) + proc = subprocess.Popen( + args=[job_file], + cwd=job_wrapper.working_directory, + stdout=stdout_file, + stderr=stderr_file, + env=self._environ, + preexec_fn=os.setpgrp, + ) proc.terminated_by_shutdown = False with self._proc_lock: @@ -133,7 +133,7 @@ class LocalJobRunner(BaseJobRunner): stderr = self._job_io_for_db(stderr_file) stdout_file.close() stderr_file.close() - log.debug(f'execution finished: {job_file}') + log.debug(f"execution finished: {job_file}") except Exception: log.exception("failure running job %d", job_wrapper.job_id) self._fail_job_local(job_wrapper, "failure running job") @@ -152,24 +152,28 @@ class LocalJobRunner(BaseJobRunner): job = job_wrapper.get_job() job_ext_output_metadata = job.get_external_output_metadata() try: - pid = job_ext_output_metadata[0].job_runner_external_pid # every JobExternalOutputMetadata has a pid set, we just need to take from one of them - assert pid not in [None, ''] + pid = job_ext_output_metadata[ + 0 + ].job_runner_external_pid # every JobExternalOutputMetadata has a pid set, we just need to take from one of them + assert pid not in [None, ""] except Exception: # metadata internal or job not complete yet pid = job.get_job_runner_external_id() - if pid in [None, '']: + if pid in [None, ""]: log.warning(f"stop_job(): {job.id}: no PID in database for job, unable to stop") return pid = int(pid) if not check_pg(pid): log.warning("stop_job(): %s: Process group %d was already dead or can't be signaled" % (job.id, pid)) return - log.debug('stop_job(): %s: Terminating process group %d', job.id, pid) + log.debug("stop_job(): %s: Terminating process group %d", job.id, pid) kill_pg(pid) def recover(self, job, job_wrapper): # local jobs can't be recovered - job_wrapper.change_state(model.Job.states.ERROR, info="This job was killed when Galaxy was restarted. Please retry the job.") + job_wrapper.change_state( + model.Job.states.ERROR, info="This job was killed when Galaxy was restarted. Please retry the job." + ) def shutdown(self): super().shutdown() @@ -223,7 +227,7 @@ class LocalJobRunner(BaseJobRunner): limit_state = job_wrapper.check_limits(runtime=datetime.datetime.now() - job_start) if limit_state is not None: job_wrapper.fail(limit_state[1]) - log.debug('(%s) Terminating process group %d', job_id, pgid) + log.debug("(%s) Terminating process group %d", job_id, pgid) kill_pg(pgid) return True else: diff --git a/lib/galaxy/jobs/runners/pbs.py b/lib/galaxy/jobs/runners/pbs.py index 5cf1871bda8..72e0876d39e 100644 --- a/lib/galaxy/jobs/runners/pbs.py +++ b/lib/galaxy/jobs/runners/pbs.py @@ -6,27 +6,30 @@ from datetime import timedelta try: import pbs + PBS_IMPORT_MESSAGE = None except ImportError as exc: pbs = None - PBS_IMPORT_MESSAGE = ('The Python pbs-python package is required to use ' - 'this feature, please install it or correct the ' - 'following error:\nImportError %s' % str(exc)) + PBS_IMPORT_MESSAGE = ( + "The Python pbs-python package is required to use " + "this feature, please install it or correct the " + "following error:\nImportError %s" % str(exc) + ) from galaxy import ( model, - util + util, ) from galaxy.jobs import JobDestination from galaxy.jobs.runners import ( AsynchronousJobRunner, - AsynchronousJobState + AsynchronousJobState, ) from galaxy.util.bunch import Bunch log = logging.getLogger(__name__) -__all__ = ('PBSJobRunner', ) +__all__ = ("PBSJobRunner",) CLUSTER_ERROR_MESSAGE = "Job cannot be completed due to a cluster error, please retry it later: %s" @@ -43,26 +46,26 @@ mkdir -p %s """ PBS_ARGMAP = { - 'destination': '-q', - 'Execution_Time': '-a', - 'Account_Name': '-A', - 'Checkpoint': '-c', - 'Error_Path': '-e', - 'Group_List': '-g', - 'Hold_Types': '-h', - 'Join_Paths': '-j', - 'Keep_Files': '-k', - 'Resource_List': '-l', - 'Mail_Points': '-m', - 'Mail_Users': '-M', - 'Job_Name': '-N', - 'Output_Path': '-o', - 'Priority': '-p', - 'Rerunable': '-r', - 'Shell_Path_List': '-S', - 'job_array_request': '-t', - 'User_List': '-u', - 'Variable_List': '-v', + "destination": "-q", + "Execution_Time": "-a", + "Account_Name": "-A", + "Checkpoint": "-c", + "Error_Path": "-e", + "Group_List": "-g", + "Hold_Types": "-h", + "Join_Paths": "-j", + "Keep_Files": "-k", + "Resource_List": "-l", + "Mail_Points": "-m", + "Mail_Users": "-M", + "Job_Name": "-N", + "Output_Path": "-o", + "Priority": "-p", + "Rerunable": "-r", + "Shell_Path_List": "-S", + "job_array_request": "-t", + "User_List": "-u", + "Variable_List": "-v", } # From pbs' pbs_job.h @@ -87,18 +90,21 @@ class PBSJobRunner(AsynchronousJobRunner): """ Job runner backed by a finite pool of worker threads. FIFO scheduling """ + runner_name = "PBSRunner" def __init__(self, app, nworkers): - """Start the job runner """ + """Start the job runner""" # Check if PBS was importable, fail if not assert pbs is not None, PBS_IMPORT_MESSAGE if app.config.pbs_application_server and app.config.outputs_to_working_directory: - raise Exception("pbs_application_server (file staging) and outputs_to_working_directory options are mutually exclusive") + raise Exception( + "pbs_application_server (file staging) and outputs_to_working_directory options are mutually exclusive" + ) # Set the default server during startup self.__default_pbs_server = None - self.default_pbs_server # this is a method with a property decorator, so this causes the default server to be set + self.default_pbs_server # this is a method with a property decorator, so this causes the default server to be set # Proceed with general initialization super().__init__(app, nworkers) @@ -119,23 +125,23 @@ class PBSJobRunner(AsynchronousJobRunner): # Determine the the PBS server url_split = url.split("/") server = url_split[2] - if server == '': + if server == "": server = self.default_pbs_server if server is None: raise Exception("Could not find TORQUE server") # Determine the queue, set the PBS destination (not the same thing as a Galaxy job destination) - pbs_destination = f'@{server}' + pbs_destination = f"@{server}" pbs_queue = url_split[3] or None if pbs_queue is not None: - pbs_destination = f'{pbs_queue}{pbs_destination}' + pbs_destination = f"{pbs_queue}{pbs_destination}" params = dict(destination=pbs_destination) # Determine the args (long-format args were never supported in URLs so they are not supported here) try: - opts = url.split('/')[4].strip().lstrip('-').split(' -') - assert opts != [''] + opts = url.split("/")[4].strip().lstrip("-").split(" -") + assert opts != [""] # stripping the - comes later (in parse_destination_params) for i, opt in enumerate(opts): opts[i] = f"-{opt}" @@ -148,7 +154,7 @@ class PBSJobRunner(AsynchronousJobRunner): log.debug(f"Converted URL '{url}' to destination runner=pbs, params={params}") # Create a dynamic JobDestination - return JobDestination(runner='pbs', params=params) + return JobDestination(runner="pbs", params=params) def parse_destination_params(self, params): """A wrapper method around __args_to_attrs() that allow administrators to define PBS @@ -160,12 +166,12 @@ class PBSJobRunner(AsynchronousJobRunner): args = {} for arg, value in params.items(): try: - if not arg.startswith('-'): + if not arg.startswith("-"): arg = PBS_ARGMAP[arg] - arg = arg.lstrip('-') + arg = arg.lstrip("-") args[arg] = value except Exception: - log.warning(f'Unrecognized long argument in destination params: {arg}') + log.warning(f"Unrecognized long argument in destination params: {arg}") return self.__args_to_attrs(args) # Internal stuff @@ -176,9 +182,9 @@ class PBSJobRunner(AsynchronousJobRunner): """ rval = [] for arg, value in args.items(): - if arg == 'l': - resource_attrs = value.split(',') - for res, val in [a.split('=', 1) for a in resource_attrs]: + if arg == "l": + resource_attrs = value.split(",") + for res, val in [a.split("=", 1) for a in resource_attrs]: rval.append(dict(name=pbs.ATTR_l, value=val, resource=res)) else: try: @@ -190,12 +196,12 @@ class PBSJobRunner(AsynchronousJobRunner): def __get_pbs_server(self, job_destination_params): if job_destination_params is None: return None - return job_destination_params['destination'].split('@')[-1] + return job_destination_params["destination"].split("@")[-1] def queue_job(self, job_wrapper): """Create PBS script for a job and submit it to the PBS queue""" # prepare the job - if not self.prepare_job(job_wrapper, include_metadata=not(self.app.config.pbs_stage_path)): + if not self.prepare_job(job_wrapper, include_metadata=not (self.app.config.pbs_stage_path)): return job_destination = job_wrapper.job_destination @@ -204,25 +210,25 @@ class PBSJobRunner(AsynchronousJobRunner): pbs_queue_name = None pbs_server_name = self.default_pbs_server pbs_options = [] - if '-q' in job_destination.params and 'destination' not in job_destination.params: - job_destination.params['destination'] = job_destination.params.pop('-q') - if 'destination' in job_destination.params: - if '@' in job_destination.params['destination']: + if "-q" in job_destination.params and "destination" not in job_destination.params: + job_destination.params["destination"] = job_destination.params.pop("-q") + if "destination" in job_destination.params: + if "@" in job_destination.params["destination"]: # Destination includes a server - pbs_queue_name, pbs_server_name = job_destination.params['destination'].split('@') - if pbs_queue_name == '': + pbs_queue_name, pbs_server_name = job_destination.params["destination"].split("@") + if pbs_queue_name == "": # e.g. `qsub -q @server` pbs_queue_name = None else: # Destination is just a queue - pbs_queue_name = job_destination.params['destination'] - job_destination.params.pop('destination') + pbs_queue_name = job_destination.params["destination"] + job_destination.params.pop("destination") # Parse PBS params pbs_options = self.parse_destination_params(job_destination.params) # Explicitly set the determined PBS destination in the persisted job destination for recovery - job_destination.params['destination'] = f"{pbs_queue_name or ''}@{pbs_server_name}" + job_destination.params["destination"] = f"{pbs_queue_name or ''}@{pbs_server_name}" c = pbs.pbs_connect(util.smart_str(pbs_server_name)) if c <= 0: @@ -263,16 +269,16 @@ class PBSJobRunner(AsynchronousJobRunner): attrs.append(dict(name=pbs.ATTR_N, value=str(f"{job_wrapper.job_id}_{job_wrapper.tool.id}_{job_wrapper.user}"))) job_attrs = pbs.new_attropl(len(attrs) + len(pbs_options)) for i, attr in enumerate(attrs + pbs_options): - job_attrs[i].name = attr['name'] - job_attrs[i].value = attr['value'] - if 'resource' in attr: - job_attrs[i].resource = attr['resource'] + job_attrs[i].name = attr["name"] + job_attrs[i].value = attr["value"] + if "resource" in attr: + job_attrs[i].resource = attr["resource"] exec_dir = os.path.abspath(job_wrapper.working_directory) # write the job script - if self.app.config.pbs_stage_path != '': + if self.app.config.pbs_stage_path != "": # touch the ecfile so that it gets staged - with open(ecfile, 'a'): + with open(ecfile, "a"): os.utime(ecfile, None) stage_commands = pbs_symlink_template % ( @@ -281,10 +287,12 @@ class PBSJobRunner(AsynchronousJobRunner): exec_dir, ) else: - stage_commands = '' + stage_commands = "" env_setup_commands = [stage_commands] - script = self.get_job_file(job_wrapper, exit_code_path=ecfile, env_setup_commands=env_setup_commands, shell=job_wrapper.shell) + script = self.get_job_file( + job_wrapper, exit_code_path=ecfile, env_setup_commands=env_setup_commands, shell=job_wrapper.shell + ) job_file = f"{self.app.config.cluster_files_directory}/{job_wrapper.job_id}.sh" self.write_executable_script(job_file, script, job_io=job_wrapper.job_io) # job was deleted while we were preparing it @@ -333,7 +341,7 @@ class PBSJobRunner(AsynchronousJobRunner): job_state.output_file = ofile job_state.error_file = efile job_state.exit_code_file = ecfile - job_state.old_state = 'N' + job_state.old_state = "N" job_state.running = False job_state.job_destination = job_destination @@ -365,7 +373,9 @@ class PBSJobRunner(AsynchronousJobRunner): try: # Recheck to make sure it wasn't a communication problem self.check_single_job(pbs_server_name, job_id) - log.warning(f"({galaxy_job_id}/{job_id}) PBS job was not in state check list, but was found with individual state check") + log.warning( + f"({galaxy_job_id}/{job_id}) PBS job was not in state check list, but was found with individual state check" + ) new_watched.append(pbs_job_state) except Exception: errno, text = pbs.error() @@ -375,7 +385,9 @@ class PBSJobRunner(AsynchronousJobRunner): self.work_queue.put((self.finish_job, pbs_job_state)) else: # Unhandled error, continue to monitor - log.info("(%s/%s) PBS state check resulted in error (%d): %s" % (galaxy_job_id, job_id, errno, text)) + log.info( + "(%s/%s) PBS state check resulted in error (%d): %s" % (galaxy_job_id, job_id, errno, text) + ) new_watched.append(pbs_job_state) continue if status.job_state != old_state: @@ -383,9 +395,9 @@ class PBSJobRunner(AsynchronousJobRunner): if status.job_state == "R" and not pbs_job_state.running: pbs_job_state.running = True pbs_job_state.job_wrapper.change_state(model.Job.states.RUNNING) - if status.job_state == "R" and status.get('resources_used', False): + if status.job_state == "R" and status.get("resources_used", False): # resources_used may not be in the status for new jobs - h, m, s = (int(i) for i in status.resources_used.walltime.split(':')) + h, m, s = (int(i) for i in status.resources_used.walltime.split(":")) runtime = timedelta(0, s, 0, 0, m, h) if pbs_job_state.check_limits(runtime=runtime): self.work_queue.put((self.fail_job, pbs_job_state)) @@ -393,13 +405,16 @@ class PBSJobRunner(AsynchronousJobRunner): elif status.job_state == "C": # "keep_completed" is enabled in PBS, so try to check exit status try: - assert int(status.exit_status) == 0 or pbs_job_state.job_wrapper.get_state() == model.Job.states.STOPPED + assert ( + int(status.exit_status) == 0 + or pbs_job_state.job_wrapper.get_state() == model.Job.states.STOPPED + ) log.debug(f"({galaxy_job_id}/{job_id}) PBS job has completed successfully") except AssertionError: exit_status = int(status.exit_status) - error_message = JOB_EXIT_STATUS.get(exit_status, f'Unknown error: {status.exit_status}') + error_message = JOB_EXIT_STATUS.get(exit_status, f"Unknown error: {status.exit_status}") pbs_job_state.fail_message = CLUSTER_ERROR_MESSAGE % error_message - log.error(f'({galaxy_job_id}/{job_id}) PBS job failed: {error_message}') + log.error(f"({galaxy_job_id}/{job_id}) PBS job failed: {error_message}") pbs_job_state.stop_job = False self.work_queue.put((self.fail_job, pbs_job_state)) continue @@ -440,7 +455,7 @@ class PBSJobRunner(AsynchronousJobRunner): jobs = pbs.pbs_statjob(c, None, stat_attrl, None) pbs.pbs_disconnect(c) statuses.update(self.convert_statjob_to_bunches(jobs)) - return((failures, statuses)) + return (failures, statuses) def convert_statjob_to_bunches(self, statjob_out): statuses = {} @@ -481,15 +496,22 @@ class PBSJobRunner(AsynchronousJobRunner): self.stop_job(pbs_job_state.job_wrapper) pbs_job_state.job_wrapper.fail(pbs_job_state.fail_message) if pbs_job_state.job_wrapper.cleanup_job == "always": - self.cleanup((pbs_job_state.output_file, pbs_job_state.error_file, pbs_job_state.exit_code_file, pbs_job_state.job_file)) + self.cleanup( + ( + pbs_job_state.output_file, + pbs_job_state.error_file, + pbs_job_state.exit_code_file, + pbs_job_state.job_file, + ) + ) def get_stage_in_out(self, fnames, symlink=False): """Convenience function to create a stagein/stageout list""" - stage = '' + stage = "" for fname in fnames: if os.access(fname, os.R_OK): if stage: - stage += ',' + stage += "," # pathnames are now absolute if symlink and self.app.config.pbs_stage_path: stage_name = os.path.join(self.app.config.pbs_stage_path, os.path.split(fname)[1]) @@ -501,7 +523,7 @@ class PBSJobRunner(AsynchronousJobRunner): def stop_job(self, job_wrapper): """Attempts to delete a job from the PBS queue""" job = job_wrapper.get_job() - job_id = job.get_job_runner_external_id().encode('utf-8') + job_id = job.get_job_runner_external_id().encode("utf-8") job_tag = f"({job.get_id_tag()}/{job_id})" log.debug(f"{job_tag} Stopping PBS job") @@ -511,21 +533,20 @@ class PBSJobRunner(AsynchronousJobRunner): try: pbs_server_name = self.__get_pbs_server(job.destination_params) if pbs_server_name is None: - log.debug("(%s) Job queued but no destination stored in job params, cannot delete" - % job_tag) + log.debug("(%s) Job queued but no destination stored in job params, cannot delete" % job_tag) return c = pbs.pbs_connect(util.smart_str(pbs_server_name)) if c <= 0: log.debug(f"({job_tag}) Connection to PBS server for job delete failed") return - pbs.pbs_deljob(c, job_id, '') + pbs.pbs_deljob(c, job_id, "") log.debug(f"{job_tag} Removed from PBS queue before job completion") except Exception: e = traceback.format_exc() log.debug(f"{job_tag} Unable to stop job: {e}") finally: # Cleanup: disconnect from the server. - if (None is not c): + if None is not c: pbs.pbs_disconnect(c) def recover(self, job, job_wrapper): @@ -542,12 +563,16 @@ class PBSJobRunner(AsynchronousJobRunner): job_wrapper.command_line = job.command_line pbs_job_state.job_wrapper = job_wrapper if job.state in (model.Job.states.RUNNING, model.Job.states.STOPPED): - log.debug(f"({job.id}/{job.get_job_runner_external_id()}) is still in {job.state} state, adding to the PBS queue") - pbs_job_state.old_state = 'R' + log.debug( + f"({job.id}/{job.get_job_runner_external_id()}) is still in {job.state} state, adding to the PBS queue" + ) + pbs_job_state.old_state = "R" pbs_job_state.running = True self.monitor_queue.put(pbs_job_state) elif job.state == model.Job.states.QUEUED: - log.debug(f"({job.id}/{job.get_job_runner_external_id()}) is still in PBS queued state, adding to the PBS queue") - pbs_job_state.old_state = 'Q' + log.debug( + f"({job.id}/{job.get_job_runner_external_id()}) is still in PBS queued state, adding to the PBS queue" + ) + pbs_job_state.old_state = "Q" pbs_job_state.running = False self.monitor_queue.put(pbs_job_state) diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index 9132ef9ea12..22b72282110 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -21,13 +21,16 @@ from pulsar.client import ( ClientJobDescription, ClientOutputs, EXTENDED_METADATA_DYNAMIC_COLLECTION_PATTERN, - finish_job as pulsar_finish_job, +) +from pulsar.client import finish_job as pulsar_finish_job +from pulsar.client import ( PathMapper, PulsarClientTransportError, PulsarOutputs, - submit_job as pulsar_submit_job, - url_to_destination_params ) +from pulsar.client import submit_job as pulsar_submit_job +from pulsar.client import url_to_destination_params + # TODO: Perform pulsar release with this included in the client package from pulsar.client.staging import DEFAULT_DYNAMIC_COLLECTION_PATTERN @@ -37,30 +40,30 @@ from galaxy.jobs import JobDestination from galaxy.jobs.command_factory import build_command from galaxy.jobs.runners import ( AsynchronousJobRunner, - AsynchronousJobState + AsynchronousJobState, ) from galaxy.tool_util.deps import dependencies from galaxy.util import ( galaxy_directory, specs, - string_as_bool_or_none + string_as_bool_or_none, ) from galaxy.util.bunch import Bunch log = logging.getLogger(__name__) __all__ = ( - 'PulsarLegacyJobRunner', - 'PulsarRESTJobRunner', - 'PulsarMQJobRunner', - 'PulsarEmbeddedJobRunner', - 'PulsarEmbeddedMQJobRunner', + "PulsarLegacyJobRunner", + "PulsarRESTJobRunner", + "PulsarMQJobRunner", + "PulsarEmbeddedJobRunner", + "PulsarEmbeddedMQJobRunner", ) MINIMUM_PULSAR_VERSIONS = { - '_default_': packaging.version.parse("0.7.0.dev3"), - 'remote_metadata': packaging.version.parse("0.8.0"), - 'remote_container_handling': packaging.version.parse("0.9.1.dev0") # probably 0.10 ultimately? + "_default_": packaging.version.parse("0.7.0.dev3"), + "remote_metadata": packaging.version.parse("0.8.0"), + "remote_container_handling": packaging.version.parse("0.9.1.dev0"), # probably 0.10 ultimately? } NO_REMOTE_GALAXY_FOR_METADATA_MESSAGE = "Pulsar misconfiguration - Pulsar client configured to set metadata remotely, but remote Pulsar isn't properly configured with a galaxy_home directory." @@ -76,11 +79,7 @@ UPGRADE_PULSAR_ERROR = "Galaxy is misconfigured, please contact administrator. T DEFAULT_GALAXY_URL = "http://localhost:8080" PULSAR_PARAM_SPECS = dict( - transport=dict( - map=specs.to_str_or_none, - valid=specs.is_in("urllib", "curl", None), - default=None - ), + transport=dict(map=specs.to_str_or_none, valid=specs.is_in("urllib", "curl", None), default=None), transport_timeout=dict( map=lambda val: None if val == "None" else int(val), default=None, @@ -116,10 +115,7 @@ PULSAR_PARAM_SPECS = dict( map=specs.to_str_or_none, default=None, ), - amqp_acknowledge=dict( - map=specs.to_bool_or_none, - default=None - ), + amqp_acknowledge=dict(map=specs.to_bool_or_none, default=None), amqp_ack_republish_time=dict( map=lambda val: None if val == "None" else int(val), default=None, @@ -186,7 +182,7 @@ PARAMETER_SPECIFICATION_IGNORED = object() class PulsarJobRunner(AsynchronousJobRunner): """Base class for pulsar job runners.""" - start_methods = ['_init_worker_threads', '_init_client_manager', '_monitor'] + start_methods = ["_init_worker_threads", "_init_client_manager", "_monitor"] runner_name = "PulsarJobRunner" default_build_pulsar_app = False use_mq = False @@ -215,14 +211,14 @@ class PulsarJobRunner(AsynchronousJobRunner): self._init_noop_monitor() def _init_client_manager(self): - pulsar_conf = self.runner_params.get('pulsar_app_config', None) + pulsar_conf = self.runner_params.get("pulsar_app_config", None) pulsar_conf_file = None if pulsar_conf is None: - pulsar_conf_file = self.runner_params.get('pulsar_config', None) + pulsar_conf_file = self.runner_params.get("pulsar_config", None) self.__init_pulsar_app(pulsar_conf, pulsar_conf_file) client_manager_kwargs = {} - for kwd in 'manager', 'cache', 'transport', 'persistence_directory': + for kwd in "manager", "cache", "transport", "persistence_directory": client_manager_kwargs[kwd] = self.runner_params[kwd] if self.pulsar_app is not None: client_manager_kwargs["pulsar_app"] = self.pulsar_app @@ -231,7 +227,7 @@ class PulsarJobRunner(AsynchronousJobRunner): client_manager_kwargs["file_cache"] = None for kwd in self.runner_params.keys(): - if kwd.startswith('amqp_') or kwd.startswith('transport_'): + if kwd.startswith("amqp_") or kwd.startswith("transport_"): client_manager_kwargs[kwd] = self.runner_params[kwd] self.client_manager = build_client_manager(**client_manager_kwargs) @@ -269,7 +265,11 @@ class PulsarJobRunner(AsynchronousJobRunner): if len(guest_ports) > 0: persisted_state = job_wrapper.get_state() if persisted_state in model.Job.terminal_states + [model.Job.states.DELETED_NEW]: - log.debug("(%s) Watched job in terminal state, will stop monitoring: %s", job_state.job_id, persisted_state) + log.debug( + "(%s) Watched job in terminal state, will stop monitoring: %s", + job_state.job_id, + persisted_state, + ) job_state = None elif persisted_state == model.Job.states.RUNNING: client = self.get_client_from_state(job_state) @@ -304,7 +304,7 @@ class PulsarJobRunner(AsynchronousJobRunner): return job_state def _update_job_state_for_status(self, job_state, pulsar_status, full_status=None): - log.debug('(%s) Received status update: %s %s', job_state.job_id, type(pulsar_status), pulsar_status) + log.debug("(%s) Received status update: %s %s", job_state.job_id, type(pulsar_status), pulsar_status) if pulsar_status in ["complete", "cancelled"] or job_state.job_wrapper.get_state() == model.Job.states.STOPPED: self.mark_as_finished(job_state) return None @@ -325,7 +325,9 @@ class PulsarJobRunner(AsynchronousJobRunner): job_destination = job_wrapper.job_destination self._populate_parameter_defaults(job_destination) - command_line, client, remote_job_config, compute_environment, remote_container = self.__prepare_job(job_wrapper, job_destination) + command_line, client, remote_job_config, compute_environment, remote_container = self.__prepare_job( + job_wrapper, job_destination + ) if not command_line: return @@ -348,15 +350,21 @@ class PulsarJobRunner(AsynchronousJobRunner): "dataset_uuid": str(input_dataset_wrapper.dataset_uuid), "object_store_id": input_dataset_wrapper.object_store_id, } - client_inputs_list.append(ClientInput(path, CLIENT_INPUT_PATH_TYPES.INPUT_PATH, object_store_ref=object_store_ref)) + client_inputs_list.append( + ClientInput(path, CLIENT_INPUT_PATH_TYPES.INPUT_PATH, object_store_ref=object_store_ref) + ) for input_extra_path in compute_environment.path_rewrites_input_extra.keys(): # TODO: track dataset for object_Store_ref... - client_inputs_list.append(ClientInput(input_extra_path, CLIENT_INPUT_PATH_TYPES.INPUT_EXTRA_FILES_PATH)) + client_inputs_list.append( + ClientInput(input_extra_path, CLIENT_INPUT_PATH_TYPES.INPUT_EXTRA_FILES_PATH) + ) for input_metadata_path in compute_environment.path_rewrites_input_metadata.keys(): # TODO: track dataset for object_Store_ref... - client_inputs_list.append(ClientInput(input_metadata_path, CLIENT_INPUT_PATH_TYPES.INPUT_METADATA_PATH)) + client_inputs_list.append( + ClientInput(input_metadata_path, CLIENT_INPUT_PATH_TYPES.INPUT_METADATA_PATH) + ) input_files = None client_inputs = ClientInputs(client_inputs_list) @@ -436,8 +444,8 @@ class PulsarJobRunner(AsynchronousJobRunner): def __needed_features(self, client): return { - 'remote_metadata': PulsarJobRunner.__remote_metadata(client), - 'remote_container_handling': PulsarJobRunner.__remote_container_handling(client), + "remote_metadata": PulsarJobRunner.__remote_metadata(client), + "remote_container_handling": PulsarJobRunner.__remote_container_handling(client), } def __prepare_job(self, job_wrapper, job_destination): @@ -466,17 +474,17 @@ class PulsarJobRunner(AsynchronousJobRunner): prepare_kwds = {} if rewrite_parameters: compute_environment = PulsarComputeEnvironment(client, job_wrapper, remote_job_config) - prepare_kwds['compute_environment'] = compute_environment + prepare_kwds["compute_environment"] = compute_environment job_wrapper.prepare(**prepare_kwds) self.__prepare_input_files_locally(job_wrapper) remote_metadata = PulsarJobRunner.__remote_metadata(client) dependency_resolution = PulsarJobRunner.__dependency_resolution(client) metadata_kwds = self.__build_metadata_configuration(client, job_wrapper, remote_metadata, remote_job_config) - remote_working_directory = remote_job_config['working_directory'] + remote_working_directory = remote_job_config["working_directory"] remote_job_directory = os.path.abspath(os.path.join(remote_working_directory, os.path.pardir)) remote_tool_directory = os.path.abspath(os.path.join(remote_job_directory, "tool_files")) remote_command_params = dict( - working_directory=remote_job_config['metadata_directory'], + working_directory=remote_job_config["metadata_directory"], script_directory=remote_job_directory, metadata_kwds=metadata_kwds, dependency_resolution=dependency_resolution, @@ -525,7 +533,7 @@ class PulsarJobRunner(AsynchronousJobRunner): def __prepare_input_files_locally(self, job_wrapper): """Run task splitting commands locally.""" - prepare_input_files_cmds = getattr(job_wrapper, 'prepare_input_files_cmds', None) + prepare_input_files_cmds = getattr(job_wrapper, "prepare_input_files_cmds", None) if prepare_input_files_cmds is not None: for cmd in prepare_input_files_cmds: # run the commands to stage the input files subprocess.check_call(cmd, shell=True) @@ -558,7 +566,7 @@ class PulsarJobRunner(AsynchronousJobRunner): def get_output_files(self, job_wrapper): output_paths = job_wrapper.job_io.get_output_fnames() - return [str(o) for o in output_paths] # Force job_path from DatasetPath objects. + return [str(o) for o in output_paths] # Force job_path from DatasetPath objects. def get_input_files(self, job_wrapper): input_paths = job_wrapper.job_io.get_input_paths() @@ -566,7 +574,7 @@ class PulsarJobRunner(AsynchronousJobRunner): def get_client_from_wrapper(self, job_wrapper): job_id = job_wrapper.job_id - if hasattr(job_wrapper, 'task_id'): + if hasattr(job_wrapper, "task_id"): job_id = f"{job_id}_{job_wrapper.task_id}" params = job_wrapper.job_destination.params.copy() user = job_wrapper.get_job().user @@ -592,33 +600,23 @@ class PulsarJobRunner(AsynchronousJobRunner): job_key = self.app.security.encode_id(job_id, kind="jobs_files") endpoint_base = "%s/api/jobs/%s/files?job_key=%s" if self.app.config.nginx_upload_job_files_path: - endpoint_base = "%s" + \ - self.app.config.nginx_upload_job_files_path + \ - "?job_id=%s&job_key=%s" - files_endpoint = endpoint_base % ( - self.galaxy_url, - encoded_job_id, - job_key - ) - get_client_kwds = dict( - job_id=str(job_id), - files_endpoint=files_endpoint, - env=env - ) + endpoint_base = "%s" + self.app.config.nginx_upload_job_files_path + "?job_id=%s&job_key=%s" + files_endpoint = endpoint_base % (self.galaxy_url, encoded_job_id, job_key) + get_client_kwds = dict(job_id=str(job_id), files_endpoint=files_endpoint, env=env) # Turn MutableDict into standard dict for pulsar consumption job_destination_params = dict(job_destination_params.items()) return self.client_manager.get_client(job_destination_params, **get_client_kwds) def finish_job(self, job_state): - stderr = stdout = '' + stderr = stdout = "" job_wrapper = job_state.job_wrapper try: client = self.get_client_from_state(job_state) run_results = client.full_status() remote_metadata_directory = run_results.get("metadata_directory", None) - stdout = run_results.get('stdout', '') - stderr = run_results.get('stderr', '') - exit_code = run_results.get('returncode', None) + stdout = run_results.get("stdout", "") + stderr = run_results.get("stderr", "") + exit_code = run_results.get("returncode", None) pulsar_outputs = PulsarOutputs.from_status_response(run_results) job_state = job_wrapper.get_state() # Use Pulsar client code to transfer/copy files back @@ -630,14 +628,18 @@ class PulsarJobRunner(AsynchronousJobRunner): exit_code = 0 cleanup_job = job_wrapper.cleanup_job client_outputs = self.__client_outputs(client, job_wrapper) - finish_args = dict(client=client, - job_completed_normally=completed_normally, - cleanup_job=cleanup_job, - client_outputs=client_outputs, - pulsar_outputs=pulsar_outputs) + finish_args = dict( + client=client, + job_completed_normally=completed_normally, + cleanup_job=cleanup_job, + client_outputs=client_outputs, + pulsar_outputs=pulsar_outputs, + ) failed = pulsar_finish_job(**finish_args) if failed: - job_wrapper.fail("Failed to find or download one or more job outputs from remote server.", exception=True) + job_wrapper.fail( + "Failed to find or download one or more job outputs from remote server.", exception=True + ) except Exception: self.fail_job(job_state, message=GENERIC_REMOTE_ERROR, exception=True) log.exception("failure finishing job %d", job_wrapper.job_id) @@ -650,7 +652,9 @@ class PulsarJobRunner(AsynchronousJobRunner): # Following check is a hack for jobs started during 19.01 or earlier release # and finishing with a 19.05 code base. Eliminate the hack in 19.09 or later # along with hacks for legacy metadata compute strategy. - if not os.path.exists(job_metrics_directory) or not any("__instrument" in f for f in os.listdir(job_metrics_directory)): + if not os.path.exists(job_metrics_directory) or not any( + "__instrument" in f for f in os.listdir(job_metrics_directory) + ): job_metrics_directory = job_wrapper.working_directory job_wrapper.finish( stdout, @@ -671,10 +675,11 @@ class PulsarJobRunner(AsynchronousJobRunner): if full_status: stdout = full_status.get("stdout", "") stderr = full_status.get("stderr", "") - self._handle_runner_state('failure', job_state) + self._handle_runner_state("failure", job_state) if not job_state.runner_state_handled: - job_state.job_wrapper.fail(getattr(job_state, "fail_message", message), - tool_stdout=stdout, tool_stderr=stderr, exception=exception) + job_state.job_wrapper.fail( + getattr(job_state, "fail_message", message), tool_stdout=stdout, tool_stderr=stderr, exception=exception + ) def check_pid(self, pid): try: @@ -684,7 +689,10 @@ class PulsarJobRunner(AsynchronousJobRunner): if e.errno == errno.ESRCH: log.debug("check_pid(): PID %d is dead" % pid) else: - log.warning("check_pid(): Got errno %s when attempting to check PID %d: %s" % (errno.errorcode[e.errno], pid, e.strerror)) + log.warning( + "check_pid(): Got errno %s when attempting to check PID %d: %s" + % (errno.errorcode[e.errno], pid, e.strerror) + ) return False def stop_job(self, job_wrapper): @@ -695,8 +703,10 @@ class PulsarJobRunner(AsynchronousJobRunner): client = self.get_client(job.destination_params, job.job_runner_external_id) job_ext_output_metadata = job.get_external_output_metadata() if not PulsarJobRunner.__remote_metadata(client) and job_ext_output_metadata: - pid = job_ext_output_metadata[0].job_runner_external_pid # every JobExternalOutputMetadata has a pid set, we just need to take from one of them - if pid in [None, '']: + pid = job_ext_output_metadata[ + 0 + ].job_runner_external_pid # every JobExternalOutputMetadata has a pid set, we just need to take from one of them + if pid in [None, ""]: log.warning(f"stop_job(): {job.id}: no PID in database for job, unable to stop") return pid = int(pid) @@ -707,7 +717,10 @@ class PulsarJobRunner(AsynchronousJobRunner): try: os.killpg(pid, sig) except OSError as e: - log.warning("stop_job(): %s: Got errno %s when attempting to signal %d to PID %d: %s" % (job.id, errno.errorcode[e.errno], sig, pid, e.strerror)) + log.warning( + "stop_job(): %s: Got errno %s when attempting to signal %d to PID %d: %s" + % (job.id, errno.errorcode[e.errno], sig, pid, e.strerror) + ) return # give up sleep(2) if not self.check_pid(pid): @@ -753,7 +766,7 @@ class PulsarJobRunner(AsynchronousJobRunner): def __client_outputs(self, client, job_wrapper): metadata_directory = os.path.join(job_wrapper.working_directory, "metadata") - metadata_strategy = job_wrapper.get_destination_configuration('metadata_strategy', None) + metadata_strategy = job_wrapper.get_destination_configuration("metadata_strategy", None) tool = job_wrapper.tool tool_provided_metadata_file_path = tool.provided_metadata_file tool_provided_metadata_style = tool.provided_metadata_style @@ -775,7 +788,10 @@ class PulsarJobRunner(AsynchronousJobRunner): output_files = self.get_output_files(job_wrapper) work_dir_outputs = self.get_work_dir_outputs(job_wrapper) dynamic_file_sources = [ - {"path": tool_provided_metadata_file_path, "type": "galaxy" if tool_provided_metadata_style == "default" else "legacy_galaxy"} + { + "path": tool_provided_metadata_file_path, + "type": "galaxy" if tool_provided_metadata_style == "default" else "legacy_galaxy", + } ] client_outputs = ClientOutputs( working_directory=job_wrapper.tool_working_directory, @@ -792,10 +808,10 @@ class PulsarJobRunner(AsynchronousJobRunner): def check_job_config(remote_job_config, check_features=None): check_features = check_features or {} # 0.6.0 was newest Pulsar version that did not report it's version. - pulsar_version = packaging.version.parse(remote_job_config.get('pulsar_version', "0.6.0")) + pulsar_version = packaging.version.parse(remote_job_config.get("pulsar_version", "0.6.0")) needed_version = packaging.version.parse("0.0.0") log.info(f"pulsar_version is {pulsar_version}") - for feature, needed in list(check_features.items()) + [('_default_', True)]: + for feature, needed in list(check_features.items()) + [("_default_", True)]: if not needed: continue if pulsar_version < MINIMUM_PULSAR_VERSIONS[feature]: @@ -830,7 +846,9 @@ class PulsarJobRunner(AsynchronousJobRunner): @staticmethod def __remote_container_handling(pulsar_client): - remote_container_handling = string_as_bool_or_none(pulsar_client.destination_params.get("remote_container_handling", False)) + remote_container_handling = string_as_bool_or_none( + pulsar_client.destination_params.get("remote_container_handling", False) + ) return remote_container_handling @staticmethod @@ -848,7 +866,9 @@ class PulsarJobRunner(AsynchronousJobRunner): there is no guarentee that it will contain all the datatypes available to this Galaxy. """ - use_remote_datatypes = string_as_bool_or_none(pulsar_client.destination_params.get("use_remote_datatypes", False)) + use_remote_datatypes = string_as_bool_or_none( + pulsar_client.destination_params.get("use_remote_datatypes", False) + ) return use_remote_datatypes @staticmethod @@ -862,10 +882,10 @@ class PulsarJobRunner(AsynchronousJobRunner): remote_galaxy_home = remote_system_properties.get("galaxy_home", None) if not remote_galaxy_home: raise Exception(NO_REMOTE_GALAXY_FOR_METADATA_MESSAGE) - metadata_kwds['exec_dir'] = remote_galaxy_home - outputs_directory = remote_job_config['outputs_directory'] - working_directory = remote_job_config['working_directory'] - metadata_directory = remote_job_config['metadata_directory'] + metadata_kwds["exec_dir"] = remote_galaxy_home + outputs_directory = remote_job_config["outputs_directory"] + working_directory = remote_job_config["working_directory"] + metadata_directory = remote_job_config["metadata_directory"] # For metadata calculation, we need to build a list of of output # file objects with real path indicating location on Galaxy server # and false path indicating location on compute server. Since the @@ -876,29 +896,32 @@ class PulsarJobRunner(AsynchronousJobRunner): # server relative to the remote working directory as the # false_path to send the metadata command generation module. work_dir_outputs = self.get_work_dir_outputs(job_wrapper, tool_working_directory=working_directory) - outputs = [Bunch(false_path=os.path.join(outputs_directory, os.path.basename(path)), real_path=path) for path in self.get_output_files(job_wrapper)] + outputs = [ + Bunch(false_path=os.path.join(outputs_directory, os.path.basename(path)), real_path=path) + for path in self.get_output_files(job_wrapper) + ] for output in outputs: for pulsar_workdir_path, real_path in work_dir_outputs: if real_path == output.real_path: output.false_path = pulsar_workdir_path - metadata_kwds['output_fnames'] = outputs - metadata_kwds['compute_tmp_dir'] = metadata_directory - metadata_kwds['config_root'] = remote_galaxy_home - default_config_file = os.path.join(remote_galaxy_home, 'config/galaxy.ini') - metadata_kwds['config_file'] = remote_system_properties.get('galaxy_config_file', default_config_file) - metadata_kwds['dataset_files_path'] = remote_system_properties.get('galaxy_dataset_files_path', None) + metadata_kwds["output_fnames"] = outputs + metadata_kwds["compute_tmp_dir"] = metadata_directory + metadata_kwds["config_root"] = remote_galaxy_home + default_config_file = os.path.join(remote_galaxy_home, "config/galaxy.ini") + metadata_kwds["config_file"] = remote_system_properties.get("galaxy_config_file", default_config_file) + metadata_kwds["dataset_files_path"] = remote_system_properties.get("galaxy_dataset_files_path", None) if PulsarJobRunner.__use_remote_datatypes_conf(client): - remote_datatypes_config = remote_system_properties.get('galaxy_datatypes_config_file', None) + remote_datatypes_config = remote_system_properties.get("galaxy_datatypes_config_file", None) if not remote_datatypes_config: log.warning(NO_REMOTE_DATATYPES_CONFIG) - remote_datatypes_config = os.path.join(remote_galaxy_home, 'datatypes_conf.xml') - metadata_kwds['datatypes_config'] = remote_datatypes_config + remote_datatypes_config = os.path.join(remote_galaxy_home, "datatypes_conf.xml") + metadata_kwds["datatypes_config"] = remote_datatypes_config else: - datatypes_config = os.path.join(job_wrapper.working_directory, 'registry.xml') + datatypes_config = os.path.join(job_wrapper.working_directory, "registry.xml") self.app.datatypes_registry.to_xml_file(path=datatypes_config) # Ensure this file gets pushed out to the remote config dir. job_wrapper.extra_filenames.append(datatypes_config) - metadata_kwds['datatypes_config'] = datatypes_config + metadata_kwds["datatypes_config"] = datatypes_config return metadata_kwds def __async_update(self, full_status): @@ -908,7 +931,9 @@ class PulsarJobRunner(AsynchronousJobRunner): if len(remote_job_id) == 32: # It is a UUID - assign_ids = uuid in destination params... sa_session = self.app.model.session - galaxy_job_id = sa_session.query(model.Job).filter(model.Job.job_runner_external_id == remote_job_id).one().id + galaxy_job_id = ( + sa_session.query(model.Job).filter(model.Job.job_runner_external_id == remote_job_id).one().id + ) else: galaxy_job_id = remote_job_id job, job_wrapper = self.app.job_manager.job_handler.job_queue.job_pair_for_id(galaxy_job_id) @@ -931,6 +956,7 @@ class PulsarLegacyJobRunner(PulsarJobRunner): class PulsarMQJobRunner(PulsarJobRunner): """Flavor of Pulsar job runner with sensible defaults for message queue communication.""" + use_mq = True poll = False @@ -940,7 +966,7 @@ class PulsarMQJobRunner(PulsarJobRunner): dependency_resolution="remote", jobs_directory=PARAMETER_SPECIFICATION_REQUIRED, url=PARAMETER_SPECIFICATION_IGNORED, - private_token=PARAMETER_SPECIFICATION_IGNORED + private_token=PARAMETER_SPECIFICATION_IGNORED, ) @@ -1003,7 +1029,6 @@ class PulsarEmbeddedMQJobRunner(PulsarMQJobRunner): class PulsarComputeEnvironment(ComputeEnvironment): - def __init__(self, pulsar_client, job_wrapper, remote_job_config): self.pulsar_client = pulsar_client self.job_wrapper = job_wrapper @@ -1020,8 +1045,8 @@ class PulsarComputeEnvironment(ComputeEnvironment): self._working_directory = remote_job_config["working_directory"] self._sep = remote_job_config["system_properties"]["separator"] self._tool_dir = remote_job_config["tools_directory"] - self._tmp_dir = remote_job_config.get('tmp_dir') - self._shared_home_dir = remote_job_config.get('shared_home_dir') + self._tmp_dir = remote_job_config.get("tmp_dir") + self._shared_home_dir = remote_job_config.get("shared_home_dir") version_path = self.local_path_config.version_path() new_version_path = self.path_mapper.remote_version_path_rewrite(version_path) if new_version_path: @@ -1052,21 +1077,23 @@ class PulsarComputeEnvironment(ComputeEnvironment): def input_extra_files_rewrite(self, dataset): input_path_rewrite = self.input_path_rewrite(dataset) - base_input_path = input_path_rewrite[0:-len(".dat")] + base_input_path = input_path_rewrite[0 : -len(".dat")] remote_extra_files_path_rewrite = f"{base_input_path}_files" self.path_rewrites_input_extra[dataset.extra_files_path] = remote_extra_files_path_rewrite return remote_extra_files_path_rewrite def output_extra_files_rewrite(self, dataset): output_path_rewrite = self.output_path_rewrite(dataset) - base_output_path = output_path_rewrite[0:-len(".dat")] + base_output_path = output_path_rewrite[0 : -len(".dat")] remote_extra_files_path_rewrite = f"{base_output_path}_files" return remote_extra_files_path_rewrite def input_metadata_rewrite(self, dataset, metadata_val): # May technically be incorrect to not pass through local_path_config.input_metadata_rewrite # first but that adds untested logic that wouln't ever be used. - remote_input_path = self.path_mapper.remote_input_path_rewrite(metadata_val, client_input_path_type=CLIENT_INPUT_PATH_TYPES.INPUT_METADATA_PATH) + remote_input_path = self.path_mapper.remote_input_path_rewrite( + metadata_val, client_input_path_type=CLIENT_INPUT_PATH_TYPES.INPUT_METADATA_PATH + ) if remote_input_path: log.info(f"input_metadata_rewrite is {remote_input_path} from {metadata_val}") self.path_rewrites_input_metadata[metadata_val] = remote_input_path @@ -1137,6 +1164,5 @@ class PulsarComputeEnvironment(ComputeEnvironment): class UnsupportedPulsarException(Exception): - def __init__(self, needed): super().__init__(UPGRADE_PULSAR_ERROR % needed) diff --git a/lib/galaxy/jobs/runners/slurm.py b/lib/galaxy/jobs/runners/slurm.py index 794d9bde7ae..92573c8e679 100644 --- a/lib/galaxy/jobs/runners/slurm.py +++ b/lib/galaxy/jobs/runners/slurm.py @@ -11,23 +11,25 @@ from galaxy.util.custom_logging import get_logger log = get_logger(__name__) -__all__ = ('SlurmJobRunner', ) +__all__ = ("SlurmJobRunner",) # Error message printed to job stderr when SLURM itself kills a job. # See src/common/slurm_jobacct_gather.c and src/slurmd/slurmd/req.c in # https://github.com/SchedMD/slurm/ -SLURM_MEMORY_LIMIT_EXCEEDED_MSG = 'slurmstepd: error: Exceeded job memory limit' +SLURM_MEMORY_LIMIT_EXCEEDED_MSG = "slurmstepd: error: Exceeded job memory limit" # Warning messages which may be printed to job stderr by SLURM after termination # of a job step when using the cgroup task plugin. The exceeded memory is not # always the cause of the step termination, which can be successful. # See src/plugins/task/cgroup/task_cgroup_memory.c in # https://github.com/SchedMD/slurm/ -SLURM_MEMORY_LIMIT_EXCEEDED_PARTIAL_WARNINGS = [': Exceeded job memory limit at some point.', - ': Exceeded step memory limit at some point.'] +SLURM_MEMORY_LIMIT_EXCEEDED_PARTIAL_WARNINGS = [ + ": Exceeded job memory limit at some point.", + ": Exceeded step memory limit at some point.", +] # These messages are returned to the user -OUT_OF_MEMORY_MSG = 'This job was terminated because it used more memory than it was allocated.' -PROBABLY_OUT_OF_MEMORY_MSG = 'This job was cancelled probably because it used more memory than it was allocated.' +OUT_OF_MEMORY_MSG = "This job was terminated because it used more memory than it was allocated." +PROBABLY_OUT_OF_MEMORY_MSG = "This job was cancelled probably because it used more memory than it was allocated." class SlurmJobRunner(DRMAAJobRunner): @@ -36,15 +38,15 @@ class SlurmJobRunner(DRMAAJobRunner): def _complete_terminal_job(self, ajs, drmaa_state, **kwargs): def _get_slurm_state_with_sacct(job_id, cluster): - cmd = ['sacct', '-n', '-o', 'state%-32'] + cmd = ["sacct", "-n", "-o", "state%-32"] if cluster: - cmd.extend(['-M', cluster]) - cmd.extend(['-j', job_id]) + cmd.extend(["-M", cluster]) + cmd.extend(["-j", job_id]) try: stdout = commands.execute(cmd) except commands.CommandLineException as e: - if e.stderr.strip() == 'SLURM accounting storage is disabled': - log.warning('SLURM accounting storage is not properly configured, unable to run sacct') + if e.stderr.strip() == "SLURM accounting storage is disabled": + log.warning("SLURM accounting storage is not properly configured, unable to run sacct") return raise e # First line is for 'job_id' @@ -52,27 +54,27 @@ class SlurmJobRunner(DRMAAJobRunner): # Following lines are for the steps 'job_id.0', 'job_id.1', ... (but Galaxy does not use steps) first_line = stdout.splitlines()[0] # Strip whitespaces and the final '+' (if present), only return the first word - return first_line.strip().rstrip('+').split()[0] + return first_line.strip().rstrip("+").split()[0] def _get_slurm_state(): - cmd = ['scontrol', '-o'] - if '.' in ajs.job_id: + cmd = ["scontrol", "-o"] + if "." in ajs.job_id: # custom slurm-drmaa-with-cluster-support job id syntax - job_id, cluster = ajs.job_id.split('.', 1) - cmd.extend(['-M', cluster]) + job_id, cluster = ajs.job_id.split(".", 1) + cmd.extend(["-M", cluster]) else: job_id = ajs.job_id cluster = None - cmd.extend(['show', 'job', job_id]) + cmd.extend(["show", "job", job_id]) try: stdout = commands.execute(cmd).strip() except commands.CommandLineException as e: - if e.stderr == 'slurm_load_jobs error: Invalid job id specified\n': + if e.stderr == "slurm_load_jobs error: Invalid job id specified\n": # The job may be old, try to get its state with sacct job_state = _get_slurm_state_with_sacct(job_id, cluster) if job_state: return job_state - return 'NOT_FOUND' + return "NOT_FOUND" raise e # stdout is a single line in format "key1=value1 key2=value2 ..." job_info_keys = [] @@ -80,72 +82,122 @@ class SlurmJobRunner(DRMAAJobRunner): for job_info in stdout.split(): try: # Some value may contain `=` (e.g. `StdIn=StdIn=/dev/null`) - k, v = job_info.split('=', 1) + k, v = job_info.split("=", 1) job_info_keys.append(k) job_info_values.append(v) except ValueError: # Some value may contain spaces (e.g. `Comment=** time_limit (60m) min_nodes (1) **`) job_info_values[-1] += f" {job_info}" job_info_dict = dict(zip(job_info_keys, job_info_values)) - return job_info_dict['JobState'] + return job_info_dict["JobState"] try: if drmaa_state == self.drmaa_job_states.FAILED: slurm_state = _get_slurm_state() sleep = 1 - while slurm_state == 'COMPLETING': - log.debug('(%s/%s) Waiting %s seconds for failed job to exit COMPLETING state for post-mortem', ajs.job_wrapper.get_id_tag(), ajs.job_id, sleep) + while slurm_state == "COMPLETING": + log.debug( + "(%s/%s) Waiting %s seconds for failed job to exit COMPLETING state for post-mortem", + ajs.job_wrapper.get_id_tag(), + ajs.job_id, + sleep, + ) time.sleep(sleep) sleep *= 2 if sleep > 64: ajs.fail_message = "This job failed and the system timed out while trying to determine the cause of the failure." break slurm_state = _get_slurm_state() - if slurm_state == 'NOT_FOUND': - log.warning('(%s/%s) Job not found, assuming job check exceeded MinJobAge and completing as successful', ajs.job_wrapper.get_id_tag(), ajs.job_id) + if slurm_state == "NOT_FOUND": + log.warning( + "(%s/%s) Job not found, assuming job check exceeded MinJobAge and completing as successful", + ajs.job_wrapper.get_id_tag(), + ajs.job_id, + ) drmaa_state = self.drmaa_job_states.DONE - elif slurm_state == 'COMPLETED': - log.debug("(%s/%s) SLURM reported job success, assuming job check exceeded MinJobAge and completing as successful", ajs.job_wrapper.get_id_tag(), ajs.job_id) + elif slurm_state == "COMPLETED": + log.debug( + "(%s/%s) SLURM reported job success, assuming job check exceeded MinJobAge and completing as successful", + ajs.job_wrapper.get_id_tag(), + ajs.job_id, + ) drmaa_state = self.drmaa_job_states.DONE - elif slurm_state == 'TIMEOUT': - log.info('(%s/%s) Job hit walltime', ajs.job_wrapper.get_id_tag(), ajs.job_id) - ajs.fail_message = "This job was terminated because it ran longer than the maximum allowed job run time." + elif slurm_state == "TIMEOUT": + log.info("(%s/%s) Job hit walltime", ajs.job_wrapper.get_id_tag(), ajs.job_id) + ajs.fail_message = ( + "This job was terminated because it ran longer than the maximum allowed job run time." + ) ajs.runner_state = ajs.runner_states.WALLTIME_REACHED - elif slurm_state == 'NODE_FAIL': - log.warning('(%s/%s) Job failed due to node failure, attempting resubmission', ajs.job_wrapper.get_id_tag(), ajs.job_id) - ajs.job_wrapper.change_state(model.Job.states.QUEUED, info='Job was resubmitted due to node failure') + elif slurm_state == "NODE_FAIL": + log.warning( + "(%s/%s) Job failed due to node failure, attempting resubmission", + ajs.job_wrapper.get_id_tag(), + ajs.job_id, + ) + ajs.job_wrapper.change_state( + model.Job.states.QUEUED, info="Job was resubmitted due to node failure" + ) try: self.queue_job(ajs.job_wrapper) return except Exception: - ajs.fail_message = "This job failed due to a cluster node failure, and an attempt to resubmit the job failed." - elif slurm_state == 'OUT_OF_MEMORY': - log.info('(%s/%s) Job hit memory limit (SLURM state: OUT_OF_MEMORY)', ajs.job_wrapper.get_id_tag(), ajs.job_id) + ajs.fail_message = ( + "This job failed due to a cluster node failure, and an attempt to resubmit the job failed." + ) + elif slurm_state == "OUT_OF_MEMORY": + log.info( + "(%s/%s) Job hit memory limit (SLURM state: OUT_OF_MEMORY)", + ajs.job_wrapper.get_id_tag(), + ajs.job_id, + ) ajs.fail_message = OUT_OF_MEMORY_MSG ajs.runner_state = ajs.runner_states.MEMORY_LIMIT_REACHED - elif slurm_state == 'CANCELLED': + elif slurm_state == "CANCELLED": # Check to see if the job was killed for exceeding memory consumption check_memory_limit_msg = self.__check_memory_limit(ajs.error_file) if check_memory_limit_msg: - log.info('(%s/%s) Job hit memory limit (SLURM state: CANCELLED)', ajs.job_wrapper.get_id_tag(), ajs.job_id) + log.info( + "(%s/%s) Job hit memory limit (SLURM state: CANCELLED)", + ajs.job_wrapper.get_id_tag(), + ajs.job_id, + ) ajs.fail_message = check_memory_limit_msg ajs.runner_state = ajs.runner_states.MEMORY_LIMIT_REACHED else: - log.info('(%s/%s) Job was cancelled via SLURM (e.g. with scancel(1))', ajs.job_wrapper.get_id_tag(), ajs.job_id) + log.info( + "(%s/%s) Job was cancelled via SLURM (e.g. with scancel(1))", + ajs.job_wrapper.get_id_tag(), + ajs.job_id, + ) ajs.fail_message = "This job failed because it was cancelled by an administrator." - elif slurm_state in ('PENDING', 'RUNNING'): - log.warning('(%s/%s) Job was reported by drmaa as terminal but job state in SLURM is: %s, returning to monitor queue', ajs.job_wrapper.get_id_tag(), ajs.job_id, slurm_state) + elif slurm_state in ("PENDING", "RUNNING"): + log.warning( + "(%s/%s) Job was reported by drmaa as terminal but job state in SLURM is: %s, returning to monitor queue", + ajs.job_wrapper.get_id_tag(), + ajs.job_id, + slurm_state, + ) return True else: - log.warning('(%s/%s) Job failed due to unknown reasons, job state in SLURM was: %s', ajs.job_wrapper.get_id_tag(), ajs.job_id, slurm_state) + log.warning( + "(%s/%s) Job failed due to unknown reasons, job state in SLURM was: %s", + ajs.job_wrapper.get_id_tag(), + ajs.job_id, + slurm_state, + ) ajs.fail_message = "This job failed for reasons that could not be determined." if drmaa_state == self.drmaa_job_states.FAILED: - ajs.fail_message += '\nPlease click the bug icon to report this problem if you need help.' + ajs.fail_message += "\nPlease click the bug icon to report this problem if you need help." ajs.stop_job = False self.work_queue.put((self.fail_job, ajs)) return except Exception: - log.exception('(%s/%s) Failure in SLURM _complete_terminal_job(), job final state will be: %s', ajs.job_wrapper.get_id_tag(), ajs.job_id, drmaa_state) + log.exception( + "(%s/%s) Failure in SLURM _complete_terminal_job(), job final state will be: %s", + ajs.job_wrapper.get_id_tag(), + ajs.job_id, + drmaa_state, + ) # by default, finish the job with the state from drmaa return super()._complete_terminal_job(ajs, drmaa_state=drmaa_state) @@ -155,7 +207,7 @@ class SlurmJobRunner(DRMAAJobRunner): since we are only searching the last 2K """ try: - log.debug('Checking %s for exceeded memory message from SLURM', efile_path) + log.debug("Checking %s for exceeded memory message from SLURM", efile_path) with open(efile_path) as f: if os.path.getsize(efile_path) > 2048: f.seek(-2048, os.SEEK_END) @@ -167,6 +219,6 @@ class SlurmJobRunner(DRMAAJobRunner): elif any(_ in stripped_line for _ in SLURM_MEMORY_LIMIT_EXCEEDED_PARTIAL_WARNINGS): return PROBABLY_OUT_OF_MEMORY_MSG except Exception: - log.exception('Error reading end of %s:', efile_path) + log.exception("Error reading end of %s:", efile_path) return False diff --git a/lib/galaxy/jobs/runners/state_handlers/_safe_eval.py b/lib/galaxy/jobs/runners/state_handlers/_safe_eval.py index e900071cde9..6c04fede96f 100644 --- a/lib/galaxy/jobs/runners/state_handlers/_safe_eval.py +++ b/lib/galaxy/jobs/runners/state_handlers/_safe_eval.py @@ -5,17 +5,54 @@ from ast import ( ) AST_NODE_TYPE_ALLOWLIST = [ - 'Expr', 'Load', 'Str', 'Num', 'BoolOp', 'Compare', 'And', 'Eq', 'NotEq', - 'Or', 'GtE', 'LtE', 'Lt', 'Gt', 'BinOp', 'Add', 'Div', 'Sub', 'Mult', 'Mod', - 'Pow', 'LShift', 'GShift', 'BitAnd', 'BitOr', 'BitXor', 'UnaryOp', 'Invert', - 'Not', 'NotIn', 'In', 'Is', 'IsNot', 'List', 'Index', 'Subscript', 'Constant', + "Expr", + "Load", + "Str", + "Num", + "BoolOp", + "Compare", + "And", + "Eq", + "NotEq", + "Or", + "GtE", + "LtE", + "Lt", + "Gt", + "BinOp", + "Add", + "Div", + "Sub", + "Mult", + "Mod", + "Pow", + "LShift", + "GShift", + "BitAnd", + "BitOr", + "BitXor", + "UnaryOp", + "Invert", + "Not", + "NotIn", + "In", + "Is", + "IsNot", + "List", + "Index", + "Subscript", + "Constant", # Further checks - 'Name', 'Call', 'Attribute', + "Name", + "Call", + "Attribute", ] -BUILTIN_AND_MATH_FUNCTIONS = 'abs|all|any|bin|chr|cmp|complex|divmod|float|hex|int|len|long|max|min|oct|ord|pow|range|reversed|round|sorted|str|sum|type|unichr|unicode|log|exp|sqrt|ceil|floor'.split('|') -STRING_AND_LIST_METHODS = [name for name in dir('') + dir([]) if not name.startswith('_')] +BUILTIN_AND_MATH_FUNCTIONS = "abs|all|any|bin|chr|cmp|complex|divmod|float|hex|int|len|long|max|min|oct|ord|pow|range|reversed|round|sorted|str|sum|type|unichr|unicode|log|exp|sqrt|ceil|floor".split( + "|" +) +STRING_AND_LIST_METHODS = [name for name in dir("") + dir([]) if not name.startswith("_")] VALID_FUNCTIONS = BUILTIN_AND_MATH_FUNCTIONS + STRING_AND_LIST_METHODS @@ -38,10 +75,10 @@ def _check_call(ast_node): # string or list function. ast_func = ast_node.func ast_func_class = ast_func.__class__.__name__ - if ast_func_class == 'Name': + if ast_func_class == "Name": if ast_func.id not in BUILTIN_AND_MATH_FUNCTIONS: return False - elif ast_func_class == 'Attribute': + elif ast_func_class == "Attribute": if not _check_attribute(ast_func): return False else: @@ -96,7 +133,7 @@ def _check_expression(text, allowed_variables=None): if not len(statements) == 1: return False expression = statements[0] - if expression.__class__.__name__ != 'Expr': + if expression.__class__.__name__ != "Expr": return False for ast_node in walk(expression): @@ -108,17 +145,17 @@ def _check_expression(text, allowed_variables=None): return False # White-list more potentially dangerous types AST elements. - if ast_node_class == 'Name': + if ast_node_class == "Name": # In order to prevent loading 'exec', 'eval', etc... # put string restriction on names allowed. if not _check_name(ast_node, allowed_variables): return False # Check only valid, white-listed functions are called. - elif ast_node_class == 'Call': + elif ast_node_class == "Call": if not _check_call(ast_node): return False # Check only valid, white-listed attributes are accessed - elif ast_node_class == 'Attribute': + elif ast_node_class == "Attribute": if not _check_attribute(ast_node): return False diff --git a/lib/galaxy/jobs/runners/state_handlers/resubmit.py b/lib/galaxy/jobs/runners/state_handlers/resubmit.py index 2c09923a2e5..3ad5530aa38 100644 --- a/lib/galaxy/jobs/runners/state_handlers/resubmit.py +++ b/lib/galaxy/jobs/runners/state_handlers/resubmit.py @@ -5,20 +5,20 @@ from galaxy import model from galaxy.jobs.runners import JobState from ._safe_eval import safe_eval -__all__ = ('failure', ) +__all__ = ("failure",) log = logging.getLogger(__name__) MESSAGES = dict( - walltime_reached='it reached the walltime', - memory_limit_reached='it exceeded the amount of allocated memory', - unknown_error='it encountered an unknown error', - tool_detected='it encountered a tool detected error condition', + walltime_reached="it reached the walltime", + memory_limit_reached="it exceeded the amount of allocated memory", + unknown_error="it encountered an unknown error", + tool_detected="it encountered a tool detected error condition", ) def eval_condition(condition, job_state): - runner_state = getattr(job_state, 'runner_state', None) or JobState.runner_states.UNKNOWN_ERROR + runner_state = getattr(job_state, "runner_state", None) or JobState.runner_states.UNKNOWN_ERROR attempt = 1 now = datetime.utcnow() @@ -60,16 +60,18 @@ def eval_condition(condition, job_state): def failure(app, job_runner, job_state): # Leave handler quickly if no resubmit conditions specified or if the runner state doesn't allow resubmission. - resubmit_definitions = job_state.job_destination.get('resubmit') + resubmit_definitions = job_state.job_destination.get("resubmit") if not resubmit_definitions: return - runner_state = getattr(job_state, 'runner_state', None) or JobState.runner_states.UNKNOWN_ERROR - if (runner_state not in (JobState.runner_states.WALLTIME_REACHED, - JobState.runner_states.MEMORY_LIMIT_REACHED, - JobState.runner_states.JOB_OUTPUT_NOT_RETURNED_FROM_CLUSTER, - JobState.runner_states.TOOL_DETECT_ERROR, - JobState.runner_states.UNKNOWN_ERROR)): + runner_state = getattr(job_state, "runner_state", None) or JobState.runner_states.UNKNOWN_ERROR + if runner_state not in ( + JobState.runner_states.WALLTIME_REACHED, + JobState.runner_states.MEMORY_LIMIT_REACHED, + JobState.runner_states.JOB_OUTPUT_NOT_RETURNED_FROM_CLUSTER, + JobState.runner_states.TOOL_DETECT_ERROR, + JobState.runner_states.UNKNOWN_ERROR, + ): # not set or not a handleable runner state return @@ -77,7 +79,7 @@ def failure(app, job_runner, job_state): def _handle_resubmit_definitions(resubmit_definitions, app, job_runner, job_state): - runner_state = getattr(job_state, 'runner_state', None) or JobState.runner_states.UNKNOWN_ERROR + runner_state = getattr(job_state, "runner_state", None) or JobState.runner_states.UNKNOWN_ERROR # Setup environment for evaluating resubmission conditions and related expression. expression_context = _ExpressionContext(job_state) @@ -85,7 +87,7 @@ def _handle_resubmit_definitions(resubmit_definitions, app, job_runner, job_stat # Intercept jobs that hit the walltime and have a walltime or # nonspecific resubmit destination configured for resubmit in resubmit_definitions: - condition = resubmit.get('condition', None) + condition = resubmit.get("condition", None) if condition and not expression_context.safe_eval(condition): # There is a resubmit defined for the destination but # its condition is not for the encountered state @@ -98,13 +100,14 @@ def _handle_resubmit_definitions(resubmit_definitions, app, job_runner, job_stat job_log_prefix = f"({job_state.job_wrapper.job_id})" # Is destination needed here, might these be serialized to the database? - destination = resubmit.get('environment') or resubmit.get('destination') - log.info("%s Job will be resubmitted to '%s' because %s at " - "the '%s' destination", - job_log_prefix, - destination, - MESSAGES[runner_state], - job_state.job_wrapper.job_destination.id) + destination = resubmit.get("environment") or resubmit.get("destination") + log.info( + "%s Job will be resubmitted to '%s' because %s at " "the '%s' destination", + job_log_prefix, + destination, + MESSAGES[runner_state], + job_state.job_wrapper.job_destination.id, + ) # fetch JobDestination for the id or tag if destination: new_destination = app.job_config.get_destination(destination) @@ -112,32 +115,28 @@ def _handle_resubmit_definitions(resubmit_definitions, app, job_runner, job_stat new_destination = job_state.job_destination # Resolve dynamic if necessary - new_destination = (job_state.job_wrapper.job_runner_mapper - .cache_job_destination(new_destination)) + new_destination = job_state.job_wrapper.job_runner_mapper.cache_job_destination(new_destination) # Reset job state job_state.job_wrapper.clear_working_directory() job_state.job_wrapper.invalidate_external_metadata() job = job_state.job_wrapper.get_job() - if resubmit.get('handler', None): - log.debug('%s Job reassigned to handler %s', - job_log_prefix, - resubmit['handler']) - job.set_handler(resubmit['handler']) + if resubmit.get("handler", None): + log.debug("%s Job reassigned to handler %s", job_log_prefix, resubmit["handler"]) + job.set_handler(resubmit["handler"]) job_runner.sa_session.add(job) # Is this safe to do here? job_runner.sa_session.flush() # Cache the destination to prevent rerunning dynamic after # resubmit - job_state.job_wrapper.job_runner_mapper \ - .cached_job_destination = new_destination + job_state.job_wrapper.job_runner_mapper.cached_job_destination = new_destination # Handle delaying before resubmission if needed. - raw_delay = resubmit.get('delay') + raw_delay = resubmit.get("delay") if raw_delay: delay = str(expression_context.safe_eval(str(raw_delay))) try: # ensure result acts like a number when persisted. float(delay) - new_destination.params['__resubmit_delay_seconds'] = str(delay) + new_destination.params["__resubmit_delay_seconds"] = str(delay) except ValueError: log.warning(f"Cannot delay job with delay [{delay}], does not appear to be a number.") job_state.job_wrapper.set_job_destination(new_destination) @@ -147,14 +146,12 @@ def _handle_resubmit_definitions(resubmit_definitions, app, job_runner, job_stat if job.params is None: job.params = {} job_state.runner_state_handled = True - info = "This job was resubmitted to the queue because %s on its " \ - "compute resource." % MESSAGES[runner_state] + info = "This job was resubmitted to the queue because %s on its " "compute resource." % MESSAGES[runner_state] job_runner.mark_as_resubmitted(job_state, info=info) return class _ExpressionContext: - def __init__(self, job_state): self._job_state = job_state self._lazy_context = None @@ -164,7 +161,7 @@ class _ExpressionContext: return int(condition) if self._lazy_context is None: - runner_state = getattr(self._job_state, 'runner_state', None) or JobState.runner_states.UNKNOWN_ERROR + runner_state = getattr(self._job_state, "runner_state", None) or JobState.runner_states.UNKNOWN_ERROR attempt = 1 now = datetime.utcnow() last_running_state = None diff --git a/lib/galaxy/jobs/runners/tasks.py b/lib/galaxy/jobs/runners/tasks.py index d9af589a4b9..72249681d97 100644 --- a/lib/galaxy/jobs/runners/tasks.py +++ b/lib/galaxy/jobs/runners/tasks.py @@ -9,13 +9,14 @@ from galaxy.jobs.runners import BaseJobRunner log = logging.getLogger(__name__) -__all__ = ('TaskedJobRunner', ) +__all__ = ("TaskedJobRunner",) class TaskedJobRunner(BaseJobRunner): """ Job runner backed by a finite pool of worker threads. FIFO scheduling """ + runner_name = "TaskRunner" def __init__(self, app, nworkers): @@ -31,7 +32,7 @@ class TaskedJobRunner(BaseJobRunner): # command line has been added to the wrapper by prepare_job() command_line = job_wrapper.runner_command_line - stderr = stdout = '' + stderr = stdout = "" # Persist the destination job_wrapper.set_job_destination(job_wrapper.job_destination) @@ -51,7 +52,9 @@ class TaskedJobRunner(BaseJobRunner): # Split with the defined method. parallelism = job_wrapper.get_parallelism() try: - splitter = getattr(__import__('galaxy.jobs.splitters', globals(), locals(), [parallelism.method]), parallelism.method) + splitter = getattr( + __import__("galaxy.jobs.splitters", globals(), locals(), [parallelism.method]), parallelism.method + ) except Exception: job_wrapper.change_state(model.Job.states.ERROR) job_wrapper.fail(f"Job Splitting Failed, no match for '{parallelism}'") @@ -75,9 +78,7 @@ class TaskedJobRunner(BaseJobRunner): sleep_time = 1 # sleep/loop until no more progress can be made. That is when # all tasks are one of { OK, ERROR, DELETED }. If a task - completed_states = [model.Task.states.OK, - model.Task.states.ERROR, - model.Task.states.DELETED] + completed_states = [model.Task.states.OK, model.Task.states.ERROR, model.Task.states.DELETED] # TODO: Should we report an error (and not merge outputs) if # one of the subtasks errored out? Should we prevent any that @@ -94,10 +95,9 @@ class TaskedJobRunner(BaseJobRunner): tasks_complete = True for tw in task_wrappers: task_state = tw.get_state() - if (model.Task.states.ERROR == task_state): + if model.Task.states.ERROR == task_state: job_exit_code = tw.get_exit_code() - log.debug("Canceling job %d: Task %s returned an error" - % (tw.job_id, tw.task_id)) + log.debug("Canceling job %d: Task %s returned an error" % (tw.job_id, tw.task_id)) self._cancel_job(job_wrapper, task_wrappers) tasks_complete = True break @@ -110,8 +110,8 @@ class TaskedJobRunner(BaseJobRunner): sleep(sleep_time) if sleep_time < 8: sleep_time *= 2 - job_wrapper.reclaim_ownership() # if running as the actual user, change ownership before merging. - log.debug(f'execution finished - beginning merge: {command_line}') + job_wrapper.reclaim_ownership() # if running as the actual user, change ownership before merging. + log.debug(f"execution finished - beginning merge: {command_line}") stdout, stderr = splitter.do_merge(job_wrapper, task_wrappers) except Exception: job_wrapper.fail("failure running job", exception=True) @@ -136,7 +136,7 @@ class TaskedJobRunner(BaseJobRunner): # to retrieve a job's list of tasks. job = job_wrapper.get_job() tasks = job.get_tasks() - if (len(tasks) > 0): + if len(tasks) > 0: for task in tasks: log.debug(f"Killing task's job {task.id}") self.app.job_manager.job_handler.dispatcher.stop(task) @@ -148,17 +148,21 @@ class TaskedJobRunner(BaseJobRunner): # if our local job has JobExternalOutputMetadata associated, then our primary job has to have already finished job_ext_output_metadata = job.get_external_output_metadata() if job_ext_output_metadata: - pid = job_ext_output_metadata[0].job_runner_external_pid # every JobExternalOutputMetadata has a pid set, we just need to take from one of them + pid = job_ext_output_metadata[ + 0 + ].job_runner_external_pid # every JobExternalOutputMetadata has a pid set, we just need to take from one of them else: pid = job.job_runner_external_id - if pid in [None, '']: + if pid in [None, ""]: log.warning(f"stop_job(): {job.id}: no PID in database for job, unable to stop") return self._stop_pid(pid, job.id) def recover(self, job, job_wrapper): # DBTODO Task Recovery, this should be possible. - job_wrapper.change_state(model.Job.states.ERROR, info="This job was killed when Galaxy was restarted. Please retry the job.") + job_wrapper.change_state( + model.Job.states.ERROR, info="This job was killed when Galaxy was restarted. Please retry the job." + ) def _cancel_job(self, job_wrapper, task_wrappers): """ @@ -188,9 +192,10 @@ class TaskedJobRunner(BaseJobRunner): for task_wrapper in task_wrappers: task = task_wrapper.get_task() task_state = task.get_state() - if (model.Task.states.QUEUED == task_state): - log.debug("_cancel_job for job %d: Task %d is not running; setting state to DELETED" - % (job.id, task.id)) + if model.Task.states.QUEUED == task_state: + log.debug( + "_cancel_job for job %d: Task %d is not running; setting state to DELETED" % (job.id, task.id) + ) task_wrapper.change_state(task.states.DELETED) # If a task failed, then the caller will have waited a few seconds # before recognizing the failure. In that time, a queued task could @@ -199,10 +204,9 @@ class TaskedJobRunner(BaseJobRunner): # are running. sleep(5) for task_wrapper in task_wrappers: - if (model.Task.states.RUNNING == task_wrapper.get_state()): + if model.Task.states.RUNNING == task_wrapper.get_state(): task = task_wrapper.get_task() - log.debug("_cancel_job for job %d: Stopping running task %d" - % (job.id, task.id)) + log.debug("_cancel_job for job %d: Stopping running task %d" % (job.id, task.id)) job_wrapper.app.job_manager.job_handler.dispatcher.stop(task) def _check_pid(self, pid): @@ -227,7 +231,10 @@ class TaskedJobRunner(BaseJobRunner): except OSError as e: # This warning could be bogus; many tasks are stopped with # SIGTERM (signal 15), but ymmv depending on the platform. - log.warning("_stop_pid(): %s: Got errno %s when attempting to signal %d to PID %d: %s" % (job_id, errno.errorcode[e.errno], sig, pid, e.strerror)) + log.warning( + "_stop_pid(): %s: Got errno %s when attempting to signal %d to PID %d: %s" + % (job_id, errno.errorcode[e.errno], sig, pid, e.strerror) + ) return # TODO: If we're stopping lots of tasks, then we will want to put this # avoid a two-second overhead using some other asynchronous method. diff --git a/lib/galaxy/jobs/runners/univa.py b/lib/galaxy/jobs/runners/univa.py index 2fe1a5c76ca..018e99caebb 100644 --- a/lib/galaxy/jobs/runners/univa.py +++ b/lib/galaxy/jobs/runners/univa.py @@ -37,12 +37,12 @@ from galaxy.jobs.runners.drmaa import DRMAAJobRunner from galaxy.util import ( commands, size_to_bytes, - unicodify + unicodify, ) log = logging.getLogger(__name__) -__all__ = ('UnivaJobRunner',) +__all__ = ("UnivaJobRunner",) MEMORY_LIMIT_SCAN_SIZE = 1024 * 1024 # 1MB @@ -80,7 +80,7 @@ class UnivaJobRunner(DRMAAJobRunner): if state in [self.drmaa.JobState.DONE, self.drmaa.JobState.FAILED]: # get configured job destination job_destination = ajs.job_wrapper.job_destination - native_spec = job_destination.params.get('nativeSpecification', None) + native_spec = job_destination.params.get("nativeSpecification", None) # determine time and memory that was granted for the job time_granted, mem_granted = _parse_native_specs(ajs.job_id, native_spec) time_wasted = extinfo["time_wasted"] @@ -91,26 +91,40 @@ class UnivaJobRunner(DRMAAJobRunner): # check job for run time or memory violation if "deleted" in extinfo and extinfo["deleted"]: - log.info('(%s/%s) Job was cancelled (e.g. with qdel)', ajs.job_wrapper.get_id_tag(), ajs.job_id) + log.info("(%s/%s) Job was cancelled (e.g. with qdel)", ajs.job_wrapper.get_id_tag(), ajs.job_id) ajs.fail_message = "This job failed because it was cancelled." drmaa_state = self.drmaa.JobState.FAILED elif ("signal" in extinfo and extinfo["signal"] == "SIGKILL") and time_wasted > time_granted: - log.error(f'({ajs.job_wrapper.get_id_tag()}/{ajs.job_id}) Job hit walltime') - ajs.fail_message = "This job was terminated because it ran longer than the maximum allowed job run time." + log.error(f"({ajs.job_wrapper.get_id_tag()}/{ajs.job_id}) Job hit walltime") + ajs.fail_message = ( + "This job was terminated because it ran longer than the maximum allowed job run time." + ) ajs.runner_state = ajs.runner_states.WALLTIME_REACHED drmaa_state = self.drmaa.JobState.FAILED # test wasted>granted memory only if failed != 0 and exit_status != 0, ie if marked as failed elif state == self.drmaa.JobState.FAILED and mem_wasted > mem_granted * slots: - log.error(f'({ajs.job_wrapper.get_id_tag()}/{ajs.job_id}) Job hit memory limit ({mem_wasted}>{mem_granted})') + log.error( + f"({ajs.job_wrapper.get_id_tag()}/{ajs.job_id}) Job hit memory limit ({mem_wasted}>{mem_granted})" + ) ajs.fail_message = "This job was terminated because it used more than the maximum allowed memory." ajs.runner_state = ajs.runner_states.MEMORY_LIMIT_REACHED drmaa_state = self.drmaa_job_states.FAILED - elif state in [self.drmaa.JobState.QUEUED_ACTIVE, self.drmaa.JobState.SYSTEM_ON_HOLD, self.drmaa.JobState.USER_ON_HOLD, self.drmaa.JobState.USER_SYSTEM_ON_HOLD, self.drmaa.JobState.RUNNING, self.drmaa.JobState.SYSTEM_SUSPENDED, self.drmaa.JobState.USER_SUSPENDED]: - log.warning(f'({ajs.job_wrapper.get_id_tag()}/{ajs.job_id}) Job is {self.drmaa_job_state_strings[state]}, returning to monitor queue') + elif state in [ + self.drmaa.JobState.QUEUED_ACTIVE, + self.drmaa.JobState.SYSTEM_ON_HOLD, + self.drmaa.JobState.USER_ON_HOLD, + self.drmaa.JobState.USER_SYSTEM_ON_HOLD, + self.drmaa.JobState.RUNNING, + self.drmaa.JobState.SYSTEM_SUSPENDED, + self.drmaa.JobState.USER_SUSPENDED, + ]: + log.warning( + f"({ajs.job_wrapper.get_id_tag()}/{ajs.job_id}) Job is {self.drmaa_job_state_strings[state]}, returning to monitor queue" + ) # TODO return True? return True # job was not actually terminal elif state == self.drmaa.JobState.UNDETERMINED: - log.warning(f'({ajs.job_wrapper.get_id_tag()}/{ajs.job_id}) Job state could not be determined') + log.warning(f"({ajs.job_wrapper.get_id_tag()}/{ajs.job_id}) Job state could not be determined") drmaa_state = self.drmaa_job_states.FAILED else: log.error(f"DRMAAUniva: job {ajs.job_id} determined unknown state {state}") @@ -160,14 +174,14 @@ class UnivaJobRunner(DRMAAJobRunner): # exit code in case of error as well as if the jobid is not found (if job is finished). # even if this could be disambiguated by the stderr message the `qstat -u "*"` # way seems more generic - cmd = ['qstat', '-u', '"*"'] + cmd = ["qstat", "-u", '"*"'] try: stdout = commands.execute(cmd).strip() except commands.CommandLineException as e: log.error(unicodify(e)) raise self.drmaa.InternalException() state = self.drmaa.JobState.UNDETERMINED - for line in stdout.split('\n'): + for line in stdout.split("\n"): line = line.split() if len(line) >= 5 and line[0] == str(job_id): state = self._map_qstat_drmaa_states(job_id, line[5], extinfo) @@ -176,7 +190,7 @@ class UnivaJobRunner(DRMAAJobRunner): return state def _get_drmaa_state_qacct(self, job_id, extinfo): - ''' + """ get the job (drmaa) state with qacct. extinfo: dict where signal, exit_status, deleted = True, time_wasted, and memory_wasted can be stored: @@ -194,11 +208,14 @@ class UnivaJobRunner(DRMAAJobRunner): - FAILED if exit state != 0 - RUNNING if failed in 24,25 - FAILED if failed not in [0,24,25,100] - ''' + """ # log.debug("UnivaJobRunner._get_drmaa_state_qacct ({jobid})".format(jobid=job_id)) - signals = {k: v for v, k in reversed(sorted(signal.__dict__.items())) - if v.startswith('SIG') and not v.startswith('SIG_')} - cmd = ['qacct', '-j', job_id] + signals = { + k: v + for v, k in reversed(sorted(signal.__dict__.items())) + if v.startswith("SIG") and not v.startswith("SIG_") + } + cmd = ["qacct", "-j", job_id] slp = 1 # run qacct -j JOBID (since the accounting data for the job might not be # available immediately a simple retry mechanism is implemented .. @@ -277,7 +294,7 @@ class UnivaJobRunner(DRMAAJobRunner): state = self.drmaa.JobState.DONE elif 0 < qacct["exit_status"] < 129: log.error(f"DRMAAUniva: job {job_id} has exit status {qacct['exit_status']}") - extinfo['exit_status'] = qacct["exit_status"] + extinfo["exit_status"] = qacct["exit_status"] state = self.drmaa.JobState.FAILED else: log.error(f"DRMAAUniva: job {job_id} was killed by signal {qacct['exit_status'] - 128}") @@ -306,7 +323,7 @@ class UnivaJobRunner(DRMAAJobRunner): return state def _get_drmaa_state_wait(self, job_id, ds, extinfo): - ''' + """ get the (drmaa) job state with the python-drmaa wait function this function will not work if the job was started as real user since the external runner uses a different drmaa session. @@ -315,7 +332,7 @@ class UnivaJobRunner(DRMAAJobRunner): jobid: the jobid ds: drmaa session extinfo dict where signal, exit_status, deleted = True, time_wasted, and memory_wasted can be stored - ''' + """ # log.debug("UnivaJobRunner._get_drmaa_state_wait ({jobid})".format(jobid=job_id)) # experiments @@ -395,10 +412,10 @@ class UnivaJobRunner(DRMAAJobRunner): # check if job was aborted # get the used time and memory - extinfo["time_wasted"] = float(rv.resourceUsage['wallclock']) - extinfo["memory_wasted"] = float(rv.resourceUsage['maxvmem']) + extinfo["time_wasted"] = float(rv.resourceUsage["wallclock"]) + extinfo["memory_wasted"] = float(rv.resourceUsage["maxvmem"]) # TODO unsure if the resourceUsage key is really slots -> test in submit as galaxy user setting - extinfo["slots"] = float(rv.resourceUsage['slots']) + extinfo["slots"] = float(rv.resourceUsage["slots"]) # log.debug("wait -> \texitStatus {0}\thasCoreDump {1}\thasExited {2}\thasSignal {3}\tjobId {4}\t\tterminatedSignal {5}\twasAborted {6}\tresourceUsage {7}".format(rv.exitStatus, rv.hasCoreDump, rv.hasExited, rv.hasSignal, rv.jobId, rv.terminatedSignal, rv.wasAborted, rv.resourceUsage)) if rv.wasAborted: log.error(f"DRMAAUniva: job {job_id} was aborted according to wait()") @@ -448,7 +465,11 @@ class UnivaJobRunner(DRMAAJobRunner): # if the job is finished (in whatever state) get (additional) infos # drmaa.wait or qacct (oposed to job_status/qstat these methods work # only for finished jobs) - if waitqacct and state in [self.drmaa.JobState.UNDETERMINED, self.drmaa.JobState.DONE, self.drmaa.JobState.FAILED]: + if waitqacct and state in [ + self.drmaa.JobState.UNDETERMINED, + self.drmaa.JobState.DONE, + self.drmaa.JobState.FAILED, + ]: try: # log.debug("UnivaJobRunner trying wait ({jobid})".format(jobid=job_id)) wstate = self._get_drmaa_state_wait(job_id, ds, extinfo) @@ -530,7 +551,7 @@ class UnivaJobRunner(DRMAAJobRunner): return self.drmaa.JobState.FAILED elif "s" in state: return self.drmaa.JobState.USER_SUSPENDED - elif "S" in state or "T" in state or 'N' in state or 'P' in state: + elif "S" in state or "T" in state or "N" in state or "P" in state: return self.drmaa.JobState.SYSTEM_SUSPENDED elif "h" in state: return self.drmaa.JobState.USER_SYSTEM_ON_HOLD @@ -547,7 +568,7 @@ def _parse_time(tstring): tme = None m = re.search("([0-9:.]+)", tstring) if m is not None: - timespl = m.group(1).split(':') + timespl = m.group(1).split(":") tme = float(timespl[-1]) # sec if len(timespl) > 1: # min tme += float(timespl[-2]) * 60 diff --git a/lib/galaxy/jobs/runners/util/__init__.py b/lib/galaxy/jobs/runners/util/__init__.py index 73bac582bbb..4aa4cce543e 100644 --- a/lib/galaxy/jobs/runners/util/__init__.py +++ b/lib/galaxy/jobs/runners/util/__init__.py @@ -6,19 +6,18 @@ functionality shared between Galaxy and the Pulsar. from galaxy.util.bunch import Bunch from .kill import kill_pid - runner_states = Bunch( - WALLTIME_REACHED='walltime_reached', - MEMORY_LIMIT_REACHED='memory_limit_reached', - JOB_OUTPUT_NOT_RETURNED_FROM_CLUSTER='Job output not returned from cluster', - UNKNOWN_ERROR='unknown_error', - GLOBAL_WALLTIME_REACHED='global_walltime_reached', - OUTPUT_SIZE_LIMIT='output_size_limit', - TOOL_DETECT_ERROR='tool_detected', # job runner interaction worked fine but the tool indicated error + WALLTIME_REACHED="walltime_reached", + MEMORY_LIMIT_REACHED="memory_limit_reached", + JOB_OUTPUT_NOT_RETURNED_FROM_CLUSTER="Job output not returned from cluster", + UNKNOWN_ERROR="unknown_error", + GLOBAL_WALLTIME_REACHED="global_walltime_reached", + OUTPUT_SIZE_LIMIT="output_size_limit", + TOOL_DETECT_ERROR="tool_detected", # job runner interaction worked fine but the tool indicated error ) __all__ = ( - 'kill_pid', - 'runner_states', + "kill_pid", + "runner_states", ) diff --git a/lib/galaxy/jobs/runners/util/cli/__init__.py b/lib/galaxy/jobs/runners/util/cli/__init__.py index 5e199fe787b..9b677bea0a8 100644 --- a/lib/galaxy/jobs/runners/util/cli/__init__.py +++ b/lib/galaxy/jobs/runners/util/cli/__init__.py @@ -4,7 +4,7 @@ import json from galaxy.util.plugin_config import plugins_dict -DEFAULT_SHELL_PLUGIN = 'LocalShell' +DEFAULT_SHELL_PLUGIN = "LocalShell" ERROR_MESSAGE_NO_JOB_PLUGIN = "No job plugin parameter found, cannot create CLI job interface" ERROR_MESSAGE_NO_SUCH_JOB_PLUGIN = "Failed to find job_plugin of type %s, available types include %s" @@ -17,8 +17,7 @@ class CliInterface: """ def __init__(self): - """ - """ + """ """ module_prefix = self.__module__ self.cli_shells = plugins_dict(f"{module_prefix}.shell", "__name__") self.cli_job_interfaces = plugins_dict(f"{module_prefix}.job", "__name__") @@ -34,17 +33,19 @@ class CliInterface: return shell, job_interface def get_shell_plugin(self, shell_params): - shell_plugin = shell_params.get('plugin', DEFAULT_SHELL_PLUGIN) + shell_plugin = shell_params.get("plugin", DEFAULT_SHELL_PLUGIN) requested_shell_settings = json.dumps(shell_params, sort_keys=True) if requested_shell_settings not in self.active_cli_shells: shell_plugin_class = self.cli_shells.get(shell_plugin) if not shell_plugin_class: - raise ValueError(f"Unknown shell_plugin [{shell_plugin}], available plugins are {list(self.cli_shells.keys())}") + raise ValueError( + f"Unknown shell_plugin [{shell_plugin}], available plugins are {list(self.cli_shells.keys())}" + ) self.active_cli_shells[requested_shell_settings] = shell_plugin_class(**shell_params) return self.active_cli_shells[requested_shell_settings] def get_job_interface(self, job_params): - job_plugin = job_params.get('plugin') + job_plugin = job_params.get("plugin") if not job_plugin: raise ValueError(ERROR_MESSAGE_NO_JOB_PLUGIN) job_plugin_class = self.cli_job_interfaces.get(job_plugin) @@ -54,6 +55,6 @@ class CliInterface: def split_params(params): - shell_params = {k.replace('shell_', '', 1): v for k, v in params.items() if k.startswith('shell_')} - job_params = {k.replace('job_', '', 1): v for k, v in params.items() if k.startswith('job_')} + shell_params = {k.replace("shell_", "", 1): v for k, v in params.items() if k.startswith("shell_")} + job_params = {k.replace("job_", "", 1): v for k, v in params.items() if k.startswith("job_")} return shell_params, job_params diff --git a/lib/galaxy/jobs/runners/util/cli/factory.py b/lib/galaxy/jobs/runners/util/cli/factory.py index 64f8cb944f9..e7fe72b3c37 100644 --- a/lib/galaxy/jobs/runners/util/cli/factory.py +++ b/lib/galaxy/jobs/runners/util/cli/factory.py @@ -1,12 +1,12 @@ try: from galaxy.jobs.runners.util.cli import ( CliInterface, - split_params + split_params, ) except ImportError: from pulsar.managers.util.cli import ( # type: ignore[no-redef] CliInterface, - split_params + split_params, ) diff --git a/lib/galaxy/jobs/runners/util/cli/job/__init__.py b/lib/galaxy/jobs/runners/util/cli/job/__init__.py index 73d3f0b53e3..829724143ed 100644 --- a/lib/galaxy/jobs/runners/util/cli/job/__init__.py +++ b/lib/galaxy/jobs/runners/util/cli/job/__init__.py @@ -3,25 +3,25 @@ Abstract base class for cli job plugins. """ from abc import ( ABCMeta, - abstractmethod + abstractmethod, ) from enum import Enum try: from galaxy.model import Job + job_states = Job.states except ImportError: # Not in Galaxy, map Galaxy job states to Pulsar ones. class job_states(str, Enum): # type: ignore[no-redef] - RUNNING = 'running' - OK = 'complete' - QUEUED = 'queued' + RUNNING = "running" + OK = "complete" + QUEUED = "queued" ERROR = "failed" class BaseJobExec(metaclass=ABCMeta): - def __init__(self, **params): """ Constructor for CLI job executor. @@ -29,7 +29,7 @@ class BaseJobExec(metaclass=ABCMeta): self.params = params.copy() def job_script_kwargs(self, ofile, efile, job_name): - """ Return extra keyword argument for consumption by job script + """Return extra keyword argument for consumption by job script module. """ return {} @@ -86,6 +86,6 @@ class BaseJobExec(metaclass=ABCMeta): __all__ = ( - 'BaseJobExec', - 'job_states', + "BaseJobExec", + "job_states", ) diff --git a/lib/galaxy/jobs/runners/util/cli/job/lsf.py b/lib/galaxy/jobs/runners/util/cli/job/lsf.py index a1e871b732d..68c96345096 100644 --- a/lib/galaxy/jobs/runners/util/cli/job/lsf.py +++ b/lib/galaxy/jobs/runners/util/cli/job/lsf.py @@ -3,48 +3,48 @@ from logging import getLogger from os import path -from ..job import BaseJobExec, job_states +from ..job import ( + BaseJobExec, + job_states, +) from ... import runner_states log = getLogger(__name__) argmap = { - 'memory': '-M', # There is code in job_script_kwargs relying on this name's setting - 'cores': '-n', - 'queue': '-q', - 'working_dir': '-cwd', - 'project': '-P' + "memory": "-M", # There is code in job_script_kwargs relying on this name's setting + "cores": "-n", + "queue": "-q", + "working_dir": "-cwd", + "project": "-P", } class LSF(BaseJobExec): - def job_script_kwargs(self, ofile, efile, job_name): - scriptargs = {'-o': ofile, - '-e': efile, - '-J': job_name} + scriptargs = {"-o": ofile, "-e": efile, "-J": job_name} # Map arguments using argmap. for k, v in self.params.items(): - if k == 'plugin' or k == 'excluded_hosts': + if k == "plugin" or k == "excluded_hosts": continue try: - if k == 'memory': + if k == "memory": # Memory requires both -m and -R rusage[mem=v] request - scriptargs['-R'] = f"\"rusage[mem={v}]\"" - if not k.startswith('-'): + scriptargs["-R"] = f'"rusage[mem={v}]"' + if not k.startswith("-"): k = argmap[k] scriptargs[k] = v except Exception: - log.warning(f'Unrecognized long argument passed to LSF CLI plugin: {k}') + log.warning(f"Unrecognized long argument passed to LSF CLI plugin: {k}") # Generated template. - template_scriptargs = '' + template_scriptargs = "" for k, v in scriptargs.items(): - template_scriptargs += f'#BSUB {k} {v}\n' + template_scriptargs += f"#BSUB {k} {v}\n" # Excluded hosts use the same -R option already in use for mem, so easier adding here. for host in self._get_excluded_hosts(): - template_scriptargs += f'#BSUB -R \"select[hname!=\'{host}\']\"\n' + template_scriptargs += f"#BSUB -R \"select[hname!='{host}']\"\n" return dict(headers=template_scriptargs) def submit(self, script_file): @@ -55,10 +55,10 @@ class LSF(BaseJobExec): return "bsub <%s | awk '{ print $2}' | sed 's/[<>]//g'" % script_file def delete(self, job_id): - return f'bkill {job_id}' + return f"bkill {job_id}" def get_status(self, job_ids=None): - return "bjobs -a -o \"id stat\" -noheader" # check this + return 'bjobs -a -o "id stat" -noheader' # check this def get_single_status(self, job_id): return f"bjobs -o stat -noheader {job_id}" @@ -103,16 +103,16 @@ class LSF(BaseJobExec): # https://www.ibm.com/support/knowledgecenter/en/SSETD4_9.1.2/lsf_command_ref/bjobs.1.html try: return { - 'EXIT': job_states.ERROR, - 'RUN': job_states.RUNNING, - 'PEND': job_states.QUEUED, - 'DONE': job_states.OK, - 'PSUSP': job_states.ERROR, - 'USUSP': job_states.ERROR, - 'SSUSP': job_states.ERROR, - 'UNKWN': job_states.ERROR, - 'WAIT': job_states.QUEUED, - 'ZOMBI': job_states.ERROR + "EXIT": job_states.ERROR, + "RUN": job_states.RUNNING, + "PEND": job_states.QUEUED, + "DONE": job_states.OK, + "PSUSP": job_states.ERROR, + "USUSP": job_states.ERROR, + "SSUSP": job_states.ERROR, + "UNKWN": job_states.ERROR, + "WAIT": job_states.QUEUED, + "ZOMBI": job_states.ERROR, }.get(state) except KeyError: raise KeyError(f"Failed to map LSF status code [{state}] to job state.") @@ -143,4 +143,4 @@ class LSF(BaseJobExec): return [] -__all__ = ('LSF',) +__all__ = ("LSF",) diff --git a/lib/galaxy/jobs/runners/util/cli/job/pbs.py b/lib/galaxy/jobs/runners/util/cli/job/pbs.py index 19746660ae3..b885a2de118 100644 --- a/lib/galaxy/jobs/runners/util/cli/job/pbs.py +++ b/lib/galaxy/jobs/runners/util/cli/job/pbs.py @@ -8,25 +8,25 @@ log = getLogger(__name__) class OpenPBS(Torque): - ERROR_MESSAGE_UNRECOGNIZED_ARG = 'Unrecognized long argument passed to OpenPBS CLI plugin: %s' + ERROR_MESSAGE_UNRECOGNIZED_ARG = "Unrecognized long argument passed to OpenPBS CLI plugin: %s" def get_status(self, job_ids=None): - return 'qstat -f -F json' + return "qstat -f -F json" def get_single_status(self, job_id): - return f'qstat -f {job_id}' + return f"qstat -f {job_id}" def parse_status(self, status, job_ids): try: data = json.loads(status) except Exception: - log.warning(f'No valid qstat JSON return from `qstat -f -F json`, got the following: {status}') + log.warning(f"No valid qstat JSON return from `qstat -f -F json`, got the following: {status}") rval = {} - for job_id, job in data.get('Jobs', {}).items(): + for job_id, job in data.get("Jobs", {}).items(): if job_id in job_ids: # map PBS job states to Galaxy job states. - rval[id] = self._get_job_state(job['job_state']) + rval[id] = self._get_job_state(job["job_state"]) return rval -__all__ = ('OpenPBS',) +__all__ = ("OpenPBS",) diff --git a/lib/galaxy/jobs/runners/util/cli/job/slurm.py b/lib/galaxy/jobs/runners/util/cli/job/slurm.py index 9ae10295ace..5b694b13253 100644 --- a/lib/galaxy/jobs/runners/util/cli/job/slurm.py +++ b/lib/galaxy/jobs/runners/util/cli/job/slurm.py @@ -2,46 +2,42 @@ # non-submit host and using a Slurm cluster. from logging import getLogger -from ..job import BaseJobExec, job_states +from ..job import ( + BaseJobExec, + job_states, +) log = getLogger(__name__) -argmap = { - 'time': '-t', - 'ncpus': '-c', - 'partition': '-p' -} +argmap = {"time": "-t", "ncpus": "-c", "partition": "-p"} class Slurm(BaseJobExec): - def job_script_kwargs(self, ofile, efile, job_name): - scriptargs = {'-o': ofile, - '-e': efile, - '-J': job_name} + scriptargs = {"-o": ofile, "-e": efile, "-J": job_name} # Map arguments using argmap. for k, v in self.params.items(): - if k == 'plugin': + if k == "plugin": continue try: - if not k.startswith('-'): + if not k.startswith("-"): k = argmap[k] scriptargs[k] = v except Exception: - log.warning(f'Unrecognized long argument passed to Slurm CLI plugin: {k}') + log.warning(f"Unrecognized long argument passed to Slurm CLI plugin: {k}") # Generated template. - template_scriptargs = '' + template_scriptargs = "" for k, v in scriptargs.items(): - template_scriptargs += f'#SBATCH {k} {v}\n' + template_scriptargs += f"#SBATCH {k} {v}\n" return dict(headers=template_scriptargs) def submit(self, script_file): - return f'sbatch {script_file}' + return f"sbatch {script_file}" def delete(self, job_id): - return f'scancel {job_id}' + return f"scancel {job_id}" def get_status(self, job_ids=None): return "squeue -a -o '%A %t'" @@ -71,14 +67,14 @@ class Slurm(BaseJobExec): def _get_job_state(self, state): try: return { - 'F': job_states.ERROR, - 'R': job_states.RUNNING, - 'CG': job_states.RUNNING, - 'PD': job_states.QUEUED, - 'CD': job_states.OK + "F": job_states.ERROR, + "R": job_states.RUNNING, + "CG": job_states.RUNNING, + "PD": job_states.QUEUED, + "CD": job_states.OK, }.get(state) except KeyError: raise KeyError(f"Failed to map slurm status code [{state}] to job state.") -__all__ = ('Slurm',) +__all__ = ("Slurm",) diff --git a/lib/galaxy/jobs/runners/util/cli/job/slurm_torque.py b/lib/galaxy/jobs/runners/util/cli/job/slurm_torque.py index 78f2576ca38..1cc09708767 100644 --- a/lib/galaxy/jobs/runners/util/cli/job/slurm_torque.py +++ b/lib/galaxy/jobs/runners/util/cli/job/slurm_torque.py @@ -2,17 +2,17 @@ import re from .torque import Torque -__all__ = ('SlurmTorque',) +__all__ = ("SlurmTorque",) class SlurmTorque(Torque): - """ A CLI job executor for Slurm's Torque compatibility mode. This differs + """A CLI job executor for Slurm's Torque compatibility mode. This differs from real torque CLI in that -x command line is not available so job status needs to be parsed from qstat table instead of XML. """ def get_status(self, job_ids=None): - return 'qstat' + return "qstat" def parse_status(self, status, job_ids): rval = {} diff --git a/lib/galaxy/jobs/runners/util/cli/job/torque.py b/lib/galaxy/jobs/runners/util/cli/job/torque.py index a1a988d8cf3..cb801ed1c0a 100644 --- a/lib/galaxy/jobs/runners/util/cli/job/torque.py +++ b/lib/galaxy/jobs/runners/util/cli/job/torque.py @@ -1,66 +1,69 @@ from logging import getLogger from galaxy.util import parse_xml_string -from ..job import BaseJobExec, job_states +from ..job import ( + BaseJobExec, + job_states, +) log = getLogger(__name__) -argmap = {'destination': '-q', - 'Execution_Time': '-a', - 'Account_Name': '-A', - 'Checkpoint': '-c', - 'Error_Path': '-e', - 'Group_List': '-g', - 'Hold_Types': '-h', - 'Join_Paths': '-j', - 'Keep_Files': '-k', - 'Resource_List': '-l', - 'Mail_Points': '-m', - 'Mail_Users': '-M', - 'Job_Name': '-N', - 'Output_Path': '-o', - 'Priority': '-p', - 'Rerunable': '-r', - 'Shell_Path_List': '-S', - 'job_array_request': '-t', - 'User_List': '-u', - 'Variable_List': '-v'} +argmap = { + "destination": "-q", + "Execution_Time": "-a", + "Account_Name": "-A", + "Checkpoint": "-c", + "Error_Path": "-e", + "Group_List": "-g", + "Hold_Types": "-h", + "Join_Paths": "-j", + "Keep_Files": "-k", + "Resource_List": "-l", + "Mail_Points": "-m", + "Mail_Users": "-M", + "Job_Name": "-N", + "Output_Path": "-o", + "Priority": "-p", + "Rerunable": "-r", + "Shell_Path_List": "-S", + "job_array_request": "-t", + "User_List": "-u", + "Variable_List": "-v", +} class Torque(BaseJobExec): - ERROR_MESSAGE_UNRECOGNIZED_ARG = 'Unrecognized long argument passed to Torque CLI plugin: %s' + ERROR_MESSAGE_UNRECOGNIZED_ARG = "Unrecognized long argument passed to Torque CLI plugin: %s" def job_script_kwargs(self, ofile, efile, job_name): - pbsargs = {'-o': ofile, - '-e': efile, - '-N': job_name} + pbsargs = {"-o": ofile, "-e": efile, "-N": job_name} for k, v in self.params.items(): - if k == 'plugin': + if k == "plugin": continue try: - if not k.startswith('-'): + if not k.startswith("-"): k = argmap[k] pbsargs[k] = v except KeyError: log.warning(self.ERROR_MESSAGE_UNRECOGNIZED_ARG, k) - template_pbsargs = '' + template_pbsargs = "" for k, v in pbsargs.items(): - template_pbsargs += f'#PBS {k} {v}\n' + template_pbsargs += f"#PBS {k} {v}\n" return dict(headers=template_pbsargs) def submit(self, script_file): - return f'qsub {script_file}' + return f"qsub {script_file}" def delete(self, job_id): - return f'qdel {job_id}' + return f"qdel {job_id}" def get_status(self, job_ids=None): - return 'qstat -x' + return "qstat -x" def get_single_status(self, job_id): - return f'qstat -f {job_id}' + return f"qstat -f {job_id}" def parse_status(self, status, job_ids): # in case there's noise in the output, find the big blob 'o xml @@ -69,40 +72,37 @@ class Torque(BaseJobExec): for line in status.strip().splitlines(): try: tree = parse_xml_string(line.strip()) - assert tree.tag == 'Data' + assert tree.tag == "Data" break except Exception: tree = None if tree is None: - log.warning(f'No valid qstat XML return from `qstat -x`, got the following: {status}') + log.warning(f"No valid qstat XML return from `qstat -x`, got the following: {status}") return None else: - for job in tree.findall('Job'): - id = job.find('Job_Id').text + for job in tree.findall("Job"): + id = job.find("Job_Id").text if id in job_ids: - state = job.find('job_state').text + state = job.find("job_state").text # map PBS job states to Galaxy job states. rval[id] = self._get_job_state(state) return rval def parse_single_status(self, status, job_id): for line in status.splitlines(): - line = line.split(' = ') - if line[0].strip() == 'job_state': + line = line.split(" = ") + if line[0].strip() == "job_state": return self._get_job_state(line[1].strip()) # no state found, job has exited return job_states.OK def _get_job_state(self, state): try: - return { - 'E': job_states.RUNNING, - 'R': job_states.RUNNING, - 'Q': job_states.QUEUED, - 'C': job_states.OK - }.get(state) + return {"E": job_states.RUNNING, "R": job_states.RUNNING, "Q": job_states.QUEUED, "C": job_states.OK}.get( + state + ) except KeyError: raise KeyError(f"Failed to map torque status code [{state}] to job state.") -__all__ = ('Torque',) +__all__ = ("Torque",) diff --git a/lib/galaxy/jobs/runners/util/cli/shell/__init__.py b/lib/galaxy/jobs/runners/util/cli/shell/__init__.py index 54e77e22bd1..5e650fc2835 100644 --- a/lib/galaxy/jobs/runners/util/cli/shell/__init__.py +++ b/lib/galaxy/jobs/runners/util/cli/shell/__init__.py @@ -3,12 +3,11 @@ Abstract base class for runners which execute commands via a shell. """ from abc import ( ABCMeta, - abstractmethod + abstractmethod, ) class BaseShellExec(metaclass=ABCMeta): - @abstractmethod def __init__(self, *args, **kwargs): """ diff --git a/lib/galaxy/jobs/runners/util/cli/shell/local.py b/lib/galaxy/jobs/runners/util/cli/shell/local.py index 440eccda437..933ba622f69 100644 --- a/lib/galaxy/jobs/runners/util/cli/shell/local.py +++ b/lib/galaxy/jobs/runners/util/cli/shell/local.py @@ -2,22 +2,21 @@ import os from logging import getLogger from subprocess import ( PIPE, - Popen + Popen, ) from tempfile import TemporaryFile from time import sleep - from galaxy.util.bunch import Bunch from . import BaseShellExec from ....util.process_groups import ( check_pg, - kill_pg + kill_pg, ) log = getLogger(__name__) -TIMEOUT_ERROR_MESSAGE = 'Execution timed out' +TIMEOUT_ERROR_MESSAGE = "Execution timed out" TIMEOUT_RETURN_CODE = -1 DEFAULT_TIMEOUT = 60 DEFAULT_TIMEOUT_CHECK_INTERVAL = 3 @@ -49,7 +48,9 @@ class LocalShell(BaseShellExec): def __init__(self, **kwds): pass - def execute(self, cmd, persist=False, timeout=DEFAULT_TIMEOUT, timeout_check_interval=DEFAULT_TIMEOUT_CHECK_INTERVAL, **kwds): + def execute( + self, cmd, persist=False, timeout=DEFAULT_TIMEOUT, timeout_check_interval=DEFAULT_TIMEOUT_CHECK_INTERVAL, **kwds + ): is_cmd_string = isinstance(cmd, str) outf = TemporaryFile() p = Popen(cmd, stdin=None, stdout=outf, stderr=PIPE, shell=is_cmd_string, preexec_fn=os.setpgrp) @@ -62,7 +63,7 @@ class LocalShell(BaseShellExec): sleep(timeout_check_interval) else: kill_pg(p.pid) - return Bunch(stdout='', stderr=TIMEOUT_ERROR_MESSAGE, returncode=TIMEOUT_RETURN_CODE) + return Bunch(stdout="", stderr=TIMEOUT_ERROR_MESSAGE, returncode=TIMEOUT_RETURN_CODE) outf.seek(0) # Need to poll once to establish return code p.poll() @@ -71,7 +72,7 @@ class LocalShell(BaseShellExec): def _read_str(stream): contents = stream.read() - return contents.decode('UTF-8') if isinstance(contents, bytes) else contents + return contents.decode("UTF-8") if isinstance(contents, bytes) else contents -__all__ = ('LocalShell',) +__all__ = ("LocalShell",) diff --git a/lib/galaxy/jobs/runners/util/cli/shell/rsh.py b/lib/galaxy/jobs/runners/util/cli/shell/rsh.py index 18c128bde68..9302a3fdc41 100644 --- a/lib/galaxy/jobs/runners/util/cli/shell/rsh.py +++ b/lib/galaxy/jobs/runners/util/cli/shell/rsh.py @@ -15,12 +15,11 @@ from .local import LocalShell log = logging.getLogger(__name__) logging.getLogger("paramiko").setLevel(logging.WARNING) # paramiko logging is very verbose -__all__ = ('RemoteShell', 'SecureShell', 'GlobusSecureShell', 'ParamikoShell') +__all__ = ("RemoteShell", "SecureShell", "GlobusSecureShell", "ParamikoShell") class RemoteShell(LocalShell): - - def __init__(self, rsh='rsh', rcp='rcp', hostname='localhost', username=None, options=None, **kwargs): + def __init__(self, rsh="rsh", rcp="rcp", hostname="localhost", username=None, options=None, **kwargs): super().__init__(**kwargs) self.rsh = rsh self.rcp = rcp @@ -41,22 +40,30 @@ class RemoteShell(LocalShell): class SecureShell(RemoteShell): - - def __init__(self, rsh='ssh', rcp='scp', private_key=None, port=None, strict_host_key_checking=True, **kwargs): + def __init__(self, rsh="ssh", rcp="scp", private_key=None, port=None, strict_host_key_checking=True, **kwargs): options = [] if not string_as_bool(strict_host_key_checking): options.extend(["-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null"]) options.extend(["-o", "ConnectTimeout=60"]) if private_key: - options.extend(['-i', private_key]) + options.extend(["-i", private_key]) if port: - options.extend(['-p', str(port)]) + options.extend(["-p", str(port)]) super().__init__(rsh=rsh, rcp=rcp, options=options, **kwargs) class ParamikoShell: - - def __init__(self, username, hostname, password=None, private_key=None, port=22, timeout=60, strict_host_key_checking=True, **kwargs): + def __init__( + self, + username, + hostname, + password=None, + private_key=None, + port=22, + timeout=60, + strict_host_key_checking=True, + **kwargs, + ): self.username = username self.hostname = hostname self.password = password @@ -71,17 +78,20 @@ class ParamikoShell: def connect(self): log.info("Attempting establishment of new paramiko SSH channel") self.ssh = paramiko.SSHClient() - self.ssh.set_missing_host_key_policy(paramiko.RejectPolicy() if self.strict_host_key_checking else paramiko.WarningPolicy()) + self.ssh.set_missing_host_key_policy( + paramiko.RejectPolicy() if self.strict_host_key_checking else paramiko.WarningPolicy() + ) self.ssh.load_system_host_keys() - self.ssh.connect(hostname=self.hostname, - port=self.port, - username=self.username, - password=self.password, - key_filename=self.private_key, - timeout=self.timeout) + self.ssh.connect( + hostname=self.hostname, + port=self.port, + username=self.username, + password=self.password, + key_filename=self.private_key, + timeout=self.timeout, + ) def execute(self, cmd, timeout=60): - def retry(): try: _, stdout, stderr = self._execute(cmd, timeout) @@ -101,6 +111,5 @@ class ParamikoShell: class GlobusSecureShell(SecureShell): - - def __init__(self, rsh='gsissh', rcp='gsiscp', **kwargs): + def __init__(self, rsh="gsissh", rcp="gsiscp", **kwargs): super().__init__(rsh=rsh, rcp=rcp, **kwargs) diff --git a/lib/galaxy/jobs/runners/util/condor/__init__.py b/lib/galaxy/jobs/runners/util/condor/__init__.py index 93b2c5cdf5b..c608d13e9ca 100644 --- a/lib/galaxy/jobs/runners/util/condor/__init__.py +++ b/lib/galaxy/jobs/runners/util/condor/__init__.py @@ -3,23 +3,22 @@ Condor helper utilities. """ from subprocess import ( CalledProcessError, - check_call + check_call, ) from galaxy.util import ( commands, - unicodify + unicodify, ) from ..external import parse_external_id DEFAULT_QUERY_CLASSAD = dict( - universe='vanilla', - getenv='true', - notification='NEVER', + universe="vanilla", + getenv="true", + notification="NEVER", ) -PROBLEM_PARSING_EXTERNAL_ID = \ - "Failed to find job id from condor_submit" +PROBLEM_PARSING_EXTERNAL_ID = "Failed to find job id from condor_submit" SUBMIT_PARAM_PREFIX = "submit_" @@ -59,13 +58,13 @@ def build_submit_description(executable, output, error, user_log, query_params): submit_description = [] for key, value in all_query_params.items(): - submit_description.append(f'{key} = {value}') + submit_description.append(f"{key} = {value}") submit_description.append(f"executable = {executable}") submit_description.append(f"output = {output}") submit_description.append(f"error = {error}") submit_description.append(f"log = {user_log}") - submit_description.append('queue') - return '\n'.join(submit_description) + submit_description.append("queue") + return "\n".join(submit_description) def condor_submit(submit_file): @@ -76,12 +75,12 @@ def condor_submit(submit_file): external_id = None failure_message = None try: - condor_message = commands.execute(('condor_submit', submit_file)) + condor_message = commands.execute(("condor_submit", submit_file)) except commands.CommandLineException as e: failure_message = unicodify(e) else: try: - external_id = parse_external_id(condor_message, type='condor') + external_id = parse_external_id(condor_message, type="condor") except Exception: failure_message = f"{PROBLEM_PARSING_EXTERNAL_ID}: {condor_message}" return external_id, failure_message @@ -94,7 +93,7 @@ def condor_stop(external_id): """ failure_message = None try: - check_call(('condor_rm', external_id)) + check_call(("condor_rm", external_id)) except CalledProcessError: failure_message = "condor_rm failed" except Exception as e: @@ -103,8 +102,7 @@ def condor_stop(external_id): def summarize_condor_log(log_file, external_id): - """ - """ + """ """ log_job_id = external_id.zfill(3) s1 = s4 = s7 = s5 = s9 = False with open(log_file) as log_handle: diff --git a/lib/galaxy/jobs/runners/util/env.py b/lib/galaxy/jobs/runners/util/env.py index 3de52700078..b7a30d89505 100644 --- a/lib/galaxy/jobs/runners/util/env.py +++ b/lib/galaxy/jobs/runners/util/env.py @@ -2,7 +2,7 @@ RAW_VALUE_BY_DEFAULT = False def env_to_statement(env): - ''' Return the abstraction description of an environment variable definition + """Return the abstraction description of an environment variable definition into a statement for shell script. >>> env_to_statement(dict(name='X', value='Y')) @@ -20,20 +20,20 @@ def env_to_statement(env): '. "S"' >>> env_to_statement(dict(execute="module load java/1.5.1")) 'module load java/1.5.1' - ''' - source_file = env.get('file', None) + """ + source_file = env.get("file", None) if source_file: - return f'. {__escape(source_file, env)}' - execute = env.get('execute', None) + return f". {__escape(source_file, env)}" + execute = env.get("execute", None) if execute: return execute - name = env['name'] - value = __escape(env['value'], env) - return f'{name}={value}; export {name}' + name = env["name"] + value = __escape(env["value"], env) + return f"{name}={value}; export {name}" def __escape(value, env): - raw = env.get('raw', RAW_VALUE_BY_DEFAULT) + raw = env.get("raw", RAW_VALUE_BY_DEFAULT) if not raw: value = '"' + value.replace('"', '\\"') + '"' return value diff --git a/lib/galaxy/jobs/runners/util/external.py b/lib/galaxy/jobs/runners/util/external.py index a02b448fec0..863ee002933 100644 --- a/lib/galaxy/jobs/runners/util/external.py +++ b/lib/galaxy/jobs/runners/util/external.py @@ -3,9 +3,9 @@ from re import search EXTERNAL_ID_TYPE_ANY = None EXTERNAL_ID_PATTERNS = [ - ('condor', r'submitted to cluster (\d+)\.'), - ('slurm', r'Submitted batch job (\w+)'), - ('torque', r'(.+)'), # Default 'pattern' assumed by Galaxy code circa August 2013. + ("condor", r"submitted to cluster (\d+)\."), + ("slurm", r"Submitted batch job (\w+)"), + ("torque", r"(.+)"), # Default 'pattern' assumed by Galaxy code circa August 2013. ] diff --git a/lib/galaxy/jobs/runners/util/job_script/__init__.py b/lib/galaxy/jobs/runners/util/job_script/__init__.py index 61f13eb7b49..f48b2e41063 100644 --- a/lib/galaxy/jobs/runners/util/job_script/__init__.py +++ b/lib/galaxy/jobs/runners/util/job_script/__init__.py @@ -3,7 +3,10 @@ import os import subprocess import time from string import Template -from typing import Any, Dict +from typing import ( + Any, + Dict, +) from pkg_resources import resource_string @@ -14,17 +17,13 @@ from galaxy.util import ( ) log = logging.getLogger(__name__) -DEFAULT_SHELL = '/bin/bash' +DEFAULT_SHELL = "/bin/bash" -DEFAULT_JOB_FILE_TEMPLATE = Template( - unicodify(resource_string(__name__, 'DEFAULT_JOB_FILE_TEMPLATE.sh')) -) +DEFAULT_JOB_FILE_TEMPLATE = Template(unicodify(resource_string(__name__, "DEFAULT_JOB_FILE_TEMPLATE.sh"))) -SLOTS_STATEMENT_CLUSTER_DEFAULT = \ - unicodify(resource_string(__name__, 'CLUSTER_SLOTS_STATEMENT.sh')) +SLOTS_STATEMENT_CLUSTER_DEFAULT = unicodify(resource_string(__name__, "CLUSTER_SLOTS_STATEMENT.sh")) -MEMORY_STATEMENT_DEFAULT = \ - unicodify(resource_string(__name__, 'MEMORY_STATEMENT.sh')) +MEMORY_STATEMENT_DEFAULT = unicodify(resource_string(__name__, "MEMORY_STATEMENT.sh")) SLOTS_STATEMENT_SINGLE = """ GALAXY_SLOTS="1" @@ -42,21 +41,21 @@ fi INTEGRITY_SYNC_COMMAND = "/bin/sync" DEFAULT_INTEGRITY_CHECK = True DEFAULT_INTEGRITY_COUNT = 35 -DEFAULT_INTEGRITY_SLEEP = .25 -REQUIRED_TEMPLATE_PARAMS = ['working_directory', 'command', 'exit_code_path'] +DEFAULT_INTEGRITY_SLEEP = 0.25 +REQUIRED_TEMPLATE_PARAMS = ["working_directory", "command", "exit_code_path"] OPTIONAL_TEMPLATE_PARAMS: Dict[str, Any] = { - 'galaxy_lib': None, - 'galaxy_virtual_env': None, - 'headers': '', - 'env_setup_commands': [], - 'slots_statement': SLOTS_STATEMENT_CLUSTER_DEFAULT, - 'memory_statement': MEMORY_STATEMENT_DEFAULT, - 'instrument_pre_commands': '', - 'instrument_post_commands': '', - 'integrity_injection': INTEGRITY_INJECTION, - 'shell': DEFAULT_SHELL, - 'preserve_python_environment': True, - 'tmp_dir_creation_statement': '""', + "galaxy_lib": None, + "galaxy_virtual_env": None, + "headers": "", + "env_setup_commands": [], + "slots_statement": SLOTS_STATEMENT_CLUSTER_DEFAULT, + "memory_statement": MEMORY_STATEMENT_DEFAULT, + "instrument_pre_commands": "", + "instrument_post_commands": "", + "integrity_injection": INTEGRITY_INJECTION, + "shell": DEFAULT_SHELL, + "preserve_python_environment": True, + "tmp_dir_creation_statement": '""', } @@ -93,8 +92,8 @@ def job_script(template=DEFAULT_JOB_FILE_TEMPLATE, **kwds): if job_instrumenter: del kwds["job_instrumenter"] working_directory = kwds.get("metadata_directory", kwds["working_directory"]) - kwds["instrument_pre_commands"] = job_instrumenter.pre_execute_commands(working_directory) or '' - kwds["instrument_post_commands"] = job_instrumenter.post_execute_commands(working_directory) or '' + kwds["instrument_pre_commands"] = job_instrumenter.pre_execute_commands(working_directory) or "" + kwds["instrument_post_commands"] = job_instrumenter.post_execute_commands(working_directory) or "" template_params = OPTIONAL_TEMPLATE_PARAMS.copy() template_params.update(**kwds) @@ -107,16 +106,12 @@ def job_script(template=DEFAULT_JOB_FILE_TEMPLATE, **kwds): return template.safe_substitute(template_params) -def write_script( - path, - contents, - job_io: JobIO, - mode=RWXR_XR_X): +def write_script(path, contents, job_io: JobIO, mode=RWXR_XR_X): dir = os.path.dirname(path) if not os.path.exists(dir): os.makedirs(dir) - with open(path, 'w', encoding='utf-8') as f: + with open(path, "w", encoding="utf-8") as f: f.write(unicodify(contents)) os.chmod(path, mode) if job_io.check_job_script_integrity: @@ -155,7 +150,7 @@ def _handle_script_integrity(path, check_job_script_integrity_count, check_job_s __all__ = ( - 'job_script', - 'write_script', - 'INTEGRITY_INJECTION', + "job_script", + "write_script", + "INTEGRITY_INJECTION", ) diff --git a/lib/galaxy/jobs/runners/util/kill.py b/lib/galaxy/jobs/runners/util/kill.py index 6b4e3f8854d..f0abd62fede 100644 --- a/lib/galaxy/jobs/runners/util/kill.py +++ b/lib/galaxy/jobs/runners/util/kill.py @@ -5,9 +5,12 @@ from platform import system from time import sleep try: - from psutil import NoSuchProcess, Process + from psutil import ( + NoSuchProcess, + Process, + ) except ImportError: - """ Don't make psutil a strict requirement, but use if available. """ + """Don't make psutil a strict requirement, but use if available.""" Process = None @@ -32,7 +35,7 @@ def _psutil_kill_pid(pid): def _stock_kill_pid(pid): - is_windows = system() == 'Windows' + is_windows = system() == "Windows" if is_windows: __kill_windows(pid) @@ -42,7 +45,7 @@ def _stock_kill_pid(pid): def __kill_windows(pid): try: - subprocess.check_call(['taskkill', '/F', '/T', '/PID', pid]) + subprocess.check_call(["taskkill", "/F", "/T", "/PID", pid]) except subprocess.CalledProcessError: pass diff --git a/lib/galaxy/jobs/runners/util/process_groups.py b/lib/galaxy/jobs/runners/util/process_groups.py index c35d5397f9a..c56042b31c3 100644 --- a/lib/galaxy/jobs/runners/util/process_groups.py +++ b/lib/galaxy/jobs/runners/util/process_groups.py @@ -15,7 +15,12 @@ def check_pg(pgid): if e.errno == errno.ECHILD: log.debug("check_pg(): No process found in process group %d", pgid) else: - log.warning("check_pg(): Got errno %s when checking process group %d: %s", errno.errorcode[e.errno], pgid, e.strerror) + log.warning( + "check_pg(): Got errno %s when checking process group %d: %s", + errno.errorcode[e.errno], + pgid, + e.strerror, + ) return False # Since we are passing os.WNOHANG to os.waitpid(), pid is 0 if no process # status is available immediately. @@ -30,7 +35,13 @@ def kill_pg(pgid): except OSError as e: if e.errno == errno.ESRCH: return - log.warning("Got errno %s when sending signal %d to process group %d: %s", errno.errorcode[e.errno], sig, pgid, e.strerror) + log.warning( + "Got errno %s when sending signal %d to process group %d: %s", + errno.errorcode[e.errno], + sig, + pgid, + e.strerror, + ) sleep(1) if not check_pg(pgid): log.debug("Processes in process group %d successfully killed with signal %d", pgid, sig) diff --git a/lib/galaxy/jobs/runners/util/pykube_util.py b/lib/galaxy/jobs/runners/util/pykube_util.py index 7c500646cf2..a77733c54a5 100644 --- a/lib/galaxy/jobs/runners/util/pykube_util.py +++ b/lib/galaxy/jobs/runners/util/pykube_util.py @@ -6,14 +6,14 @@ from pathlib import PurePath try: from pykube.config import KubeConfig + from pykube.exceptions import HTTPError from pykube.http import HTTPClient from pykube.objects import ( + Ingress, Job, Pod, Service, - Ingress, ) - from pykube.exceptions import HTTPError except ImportError as exc: KubeConfig = None Ingress = None @@ -21,9 +21,11 @@ except ImportError as exc: Pod = None Service = None HTTPError = None - K8S_IMPORT_MESSAGE = ('The Python pykube package is required to use ' - 'this feature, please install it or correct the ' - 'following error:\nImportError %s' % str(exc)) + K8S_IMPORT_MESSAGE = ( + "The Python pykube package is required to use " + "this feature, please install it or correct the " + "following error:\nImportError %s" % str(exc) + ) log = logging.getLogger(__name__) @@ -31,9 +33,11 @@ DEFAULT_JOB_API_VERSION = "batch/v1" DEFAULT_SERVICE_API_VERSION = "v1" DEFAULT_INGRESS_API_VERSION = "extensions/v1beta1" DEFAULT_NAMESPACE = "default" -INSTANCE_ID_INVALID_MESSAGE = ("Galaxy instance [%s] is either too long " - "(>20 characters) or it includes non DNS " - "acceptable characters, ignoring it.") +INSTANCE_ID_INVALID_MESSAGE = ( + "Galaxy instance [%s] is either too long " + "(>20 characters) or it includes non DNS " + "acceptable characters, ignoring it." +) def ensure_pykube(): @@ -47,23 +51,23 @@ def pykube_client_from_dict(params): else: config_path = params.get("k8s_config_path") if config_path is None: - config_path = os.environ.get('KUBECONFIG', None) + config_path = os.environ.get("KUBECONFIG", None) if config_path is None: - config_path = '~/.kube/config' + config_path = "~/.kube/config" pykube_client = HTTPClient(KubeConfig.from_file(config_path)) return pykube_client def produce_k8s_job_prefix(app_prefix=None, instance_id=None): job_name_elems = [app_prefix or "", instance_id or ""] - return '-'.join(elem for elem in job_name_elems if elem) + return "-".join(elem for elem in job_name_elems if elem) def pull_policy(params): # If this doesn't validate it returns None, that seems odd? if "k8s_pull_policy" in params: - if params['k8s_pull_policy'] in ["Always", "IfNotPresent", "Never"]: - return params['k8s_pull_policy'] + if params["k8s_pull_policy"] in ["Always", "IfNotPresent", "Never"]: + return params["k8s_pull_policy"] return None @@ -90,27 +94,22 @@ def find_pod_object_by_name(pykube_api, job_name, namespace=None): def is_pod_unschedulable(pykube_api, pod, namespace=None): - is_unschedulable = any(c.get("reason") == "Unschedulable" for c in pod.obj['status'].get('conditions', [])) - if pod.obj['status'].get('phase') == "Pending" and is_unschedulable: + is_unschedulable = any(c.get("reason") == "Unschedulable" for c in pod.obj["status"].get("conditions", [])) + if pod.obj["status"].get("phase") == "Pending" and is_unschedulable: return True return False def delete_job(job, cleanup="always"): - job_failed = (job.obj['status']['failed'] > 0 - if 'failed' in job.obj['status'] else False) + job_failed = job.obj["status"]["failed"] > 0 if "failed" in job.obj["status"] else False # Scale down the job just in case even if cleanup is never job.scale(replicas=0) api_delete = cleanup == "always" if not api_delete and cleanup == "onsuccess" and not job_failed: api_delete = True if api_delete: - delete_options = { - "apiVersion": "v1", - "kind": "DeleteOptions", - "propagationPolicy": "Background" - } + delete_options = {"apiVersion": "v1", "kind": "DeleteOptions", "propagationPolicy": "Background"} r = job.api.delete(json=delete_options, **job.api_kwargs()) job.api.raise_for_status(r) @@ -120,11 +119,7 @@ def delete_ingress(ingress, cleanup="always", job_failed=False): if not api_delete and cleanup == "onsuccess" and not job_failed: api_delete = True if api_delete: - delete_options = { - "apiVersion": "v1", - "kind": "DeleteOptions", - "propagationPolicy": "Background" - } + delete_options = {"apiVersion": "v1", "kind": "DeleteOptions", "propagationPolicy": "Background"} r = ingress.api.delete(json=delete_options, **ingress.api_kwargs()) ingress.api.raise_for_status(r) @@ -134,22 +129,18 @@ def delete_service(service, cleanup="always", job_failed=False): if not api_delete and cleanup == "onsuccess" and not job_failed: api_delete = True if api_delete: - delete_options = { - "apiVersion": "v1", - "kind": "DeleteOptions", - "propagationPolicy": "Background" - } + delete_options = {"apiVersion": "v1", "kind": "DeleteOptions", "propagationPolicy": "Background"} r = service.api.delete(json=delete_options, **service.api_kwargs()) service.api.raise_for_status(r) def job_object_dict(params, job_prefix, spec): k8s_job_obj = { - "apiVersion": params.get('k8s_job_api_version', DEFAULT_JOB_API_VERSION), + "apiVersion": params.get("k8s_job_api_version", DEFAULT_JOB_API_VERSION), "kind": "Job", "metadata": { - "generateName": f"{job_prefix}-", - "namespace": params.get('k8s_namespace', DEFAULT_NAMESPACE), + "generateName": f"{job_prefix}-", + "namespace": params.get("k8s_namespace", DEFAULT_NAMESPACE), }, "spec": spec, } @@ -158,11 +149,11 @@ def job_object_dict(params, job_prefix, spec): def service_object_dict(params, service_name, spec): k8s_service_obj = { - "apiVersion": params.get('k8s_service_api_version', DEFAULT_SERVICE_API_VERSION), + "apiVersion": params.get("k8s_service_api_version", DEFAULT_SERVICE_API_VERSION), "kind": "Service", "metadata": { - "name": service_name, - "namespace": params.get('k8s_namespace', DEFAULT_NAMESPACE), + "name": service_name, + "namespace": params.get("k8s_namespace", DEFAULT_NAMESPACE), }, } k8s_service_obj["metadata"].update(spec.pop("metadata", {})) @@ -172,11 +163,11 @@ def service_object_dict(params, service_name, spec): def ingress_object_dict(params, ingress_name, spec): k8s_ingress_obj = { - "apiVersion": params.get('k8s_ingress_api_version', DEFAULT_INGRESS_API_VERSION), + "apiVersion": params.get("k8s_ingress_api_version", DEFAULT_INGRESS_API_VERSION), "kind": "Ingress", "metadata": { - "name": ingress_name, - "namespace": params.get('k8s_namespace', DEFAULT_NAMESPACE), + "name": ingress_name, + "namespace": params.get("k8s_namespace", DEFAULT_NAMESPACE), # TODO: Add default annotations }, } @@ -201,10 +192,10 @@ def parse_pvc_param_line(pvc_param): read_only = mode == "r" claim_name, _, subpath = claim.partition("/") return { - 'name': claim_name.strip(), - 'subPath': subpath.strip(), - 'mountPath': mount_path.strip(), - 'readOnly': read_only + "name": claim_name.strip(), + "subPath": subpath.strip(), + "mountPath": mount_path.strip(), + "readOnly": read_only, } @@ -244,8 +235,8 @@ def generate_relative_mounts(pvc_param, files): if not pvc_param: return param_claim = parse_pvc_param_line(pvc_param) - claim_name = param_claim['name'] - base_subpath = PurePath(param_claim.get('subPath', "")) + claim_name = param_claim["name"] + base_subpath = PurePath(param_claim.get("subPath", "")) base_mount = PurePath(param_claim["mountPath"]) read_only = param_claim["readOnly"] volume_mounts = [] @@ -257,14 +248,15 @@ def generate_relative_mounts(pvc_param, files): relpath = file_path.relative_to(base_mount) subpath = base_subpath.joinpath(relpath) volume_mounts.append( - {'name': claim_name, 'mountPath': str(file_path), 'subPath': str(subpath), 'readOnly': read_only}) + {"name": claim_name, "mountPath": str(file_path), "subPath": str(subpath), "readOnly": read_only} + ) return volume_mounts def deduplicate_entries(obj_list): # remove duplicate entries in a list of dictionaries # based on: https://stackoverflow.com/a/9428041 - return [i for n, i in enumerate(obj_list) if i not in obj_list[n + 1:]] + return [i for n, i in enumerate(obj_list) if i not in obj_list[n + 1 :]] def get_volume_mounts_for_job(job_wrapper, data_claim=None, working_claim=None): @@ -273,7 +265,8 @@ def get_volume_mounts_for_job(job_wrapper, data_claim=None, working_claim=None): volume_mounts.extend(generate_relative_mounts(data_claim, job_wrapper.job_io.get_input_fnames())) # for individual output files, mount the parent folder of each output as there could be wildcard outputs output_folders = deduplicate_entries( - [str(PurePath(str(f)).parent) for f in job_wrapper.job_io.get_output_fnames()]) + [str(PurePath(str(f)).parent) for f in job_wrapper.job_io.get_output_fnames()] + ) volume_mounts.extend(generate_relative_mounts(data_claim, output_folders)) if working_claim: @@ -295,7 +288,7 @@ def galaxy_instance_id(params): setup of a Job that is being recovered or restarted after a downtime/reboot. """ if "k8s_galaxy_instance_id" in params: - raw_value = params['k8s_galaxy_instance_id'] + raw_value = params["k8s_galaxy_instance_id"] if re.match(r"(?!-)[a-z\d-]{1,20}(? 0: - return ('Tool file error', f'Outputs have conflicting parallelism attributes: {str(illegal_outputs)}') + return ("Tool file error", f"Outputs have conflicting parallelism attributes: {str(illegal_outputs)}") - stdout = '' - stderr = '' + stdout = "" + stderr = "" try: working_directory = job_wrapper.working_directory - task_dirs = [os.path.join(working_directory, x) for x in os.listdir(working_directory) if x.startswith('task_')] + task_dirs = [os.path.join(working_directory, x) for x in os.listdir(working_directory) if x.startswith("task_")] assert task_dirs, "Should be at least one sub-task!" # TODO: Output datasets can be very complex. This doesn't handle metadata files outputs = job_wrapper.job_io.get_output_hdas_and_fnames() output_paths = job_wrapper.job_io.get_output_fnames() pickone_done = [] - task_dirs = [os.path.join(working_directory, x) for x in os.listdir(working_directory) if x.startswith('task_')] - task_dirs.sort(key=lambda x: int(x.split('task_')[-1])) + task_dirs = [os.path.join(working_directory, x) for x in os.listdir(working_directory) if x.startswith("task_")] + task_dirs.sort(key=lambda x: int(x.split("task_")[-1])) for index, output in enumerate(outputs): output_file_name = str(output_paths[index]) # Use false_path if set, else real path. base_output_name = os.path.basename(output_file_name) @@ -151,10 +153,12 @@ def do_merge(job_wrapper, task_wrappers): # file f exists; some files may not exist if a task fails. output_files = [f for f in output_files if os.path.exists(f)] if output_files: - log.debug(f'files {output_files} ') + log.debug(f"files {output_files} ") if len(output_files) < len(task_dirs): - log.debug('merging only %i out of expected %i files for %s' - % (len(output_files), len(task_dirs), output_file_name)) + log.debug( + "merging only %i out of expected %i files for %s" + % (len(output_files), len(task_dirs), output_file_name) + ) # First two args to merge always output_files and path of dataset. More # complicated merge methods may require more parameters. Set those up here. extra_merge_arg_names = getfullargspec(output_type.merge).args[2:] @@ -162,10 +166,9 @@ def do_merge(job_wrapper, task_wrappers): if "output_dataset" in extra_merge_arg_names: extra_merge_args["output_dataset"] = output_dataset output_type.merge(output_files, output_file_name, **extra_merge_args) - log.debug(f'merge finished: {output_file_name}') + log.debug(f"merge finished: {output_file_name}") else: - msg = 'nothing to merge for %s (expected %i files)' \ - % (output_file_name, len(task_dirs)) + msg = "nothing to merge for %s (expected %i files)" % (output_file_name, len(task_dirs)) log.debug(msg) stderr += f"{msg}\n" elif output in pickone_outputs: @@ -179,7 +182,7 @@ def do_merge(job_wrapper, task_wrappers): log.exception(log_error) raise Exception(log_error) except Exception as e: - stdout = 'Error merging files' + stdout = "Error merging files" log.exception(stdout) stderr = util.unicodify(e) diff --git a/lib/galaxy/main_config.py b/lib/galaxy/main_config.py index 6c433cfe460..a3f9b8a126d 100644 --- a/lib/galaxy/main_config.py +++ b/lib/galaxy/main_config.py @@ -12,7 +12,6 @@ from typing import ( from galaxy.util.properties import find_config_file from galaxy.web_stack import get_app_kwds - DEFAULT_INI_APP = "main" DEFAULT_CONFIG_SECTION = "galaxy" @@ -28,7 +27,7 @@ def absolute_config_path(path, galaxy_root): def config_is_ini(config_file): - return config_file and (config_file.endswith('.ini') or config_file.endswith('.ini.sample')) + return config_file and (config_file.endswith(".ini") or config_file.endswith(".ini.sample")) def find_config(supplied_config, galaxy_root, app_name="galaxy"): @@ -36,7 +35,7 @@ def find_config(supplied_config, galaxy_root, app_name="galaxy"): return supplied_config if galaxy_root is None: - return os.path.abspath(f'{app_name}.yml') + return os.path.abspath(f"{app_name}.yml") # If not explicitly supplied an config, check galaxy.ini and then # just resort to sample if that has not been configured. @@ -52,6 +51,7 @@ def find_config(supplied_config, galaxy_root, app_name="galaxy"): class WebappSetupProps(NamedTuple): """Basic properties to provide information about the App and the environment variables used to resolve the App configuration.""" + app_name: str default_section_name: str env_config_file: str @@ -61,13 +61,13 @@ class WebappSetupProps(NamedTuple): class WebappConfig(NamedTuple): """The resolved configuration values for a Webapp.""" + global_conf: dict load_app_kwds: dict wsgi_preflight: bool = False class WebappConfigResolver: - def __init__(self, props: WebappSetupProps) -> None: self.props = props self.app_kwds = get_app_kwds(props.default_section_name, props.app_name) diff --git a/lib/galaxy/managers/annotatable.py b/lib/galaxy/managers/annotatable.py index abf3ca9ac06..00f60a78294 100644 --- a/lib/galaxy/managers/annotatable.py +++ b/lib/galaxy/managers/annotatable.py @@ -74,7 +74,7 @@ class AnnotatableSerializerMixin: serializers: Dict[str, Serializer] def add_serializers(self): - self.serializers['annotation'] = self.serialize_annotation + self.serializers["annotation"] = self.serialize_annotation def serialize_annotation(self, item, key, user=None, **context): """ @@ -88,7 +88,7 @@ class AnnotatableDeserializerMixin: deserializers: Dict[str, Deserializer] def add_deserializers(self): - self.deserializers['annotation'] = self.deserialize_annotation + self.deserializers["annotation"] = self.deserialize_annotation def deserialize_annotation(self, item, key, val, user=None, **context): """ @@ -121,10 +121,10 @@ class AnnotatableFilterMixin: def _add_parsers(self): self.fn_filter_parsers.update( { - 'annotation': { - 'op': { - 'has': self.filter_annotation_contains, - 'contains': self.filter_annotation_contains, + "annotation": { + "op": { + "has": self.filter_annotation_contains, + "contains": self.filter_annotation_contains, }, }, } diff --git a/lib/galaxy/managers/api_keys.py b/lib/galaxy/managers/api_keys.py index 4c7e7659ea6..1adbede04ec 100644 --- a/lib/galaxy/managers/api_keys.py +++ b/lib/galaxy/managers/api_keys.py @@ -2,7 +2,6 @@ from galaxy.structured_app import BasicSharedApp class ApiKeyManager: - def __init__(self, app: BasicSharedApp): self.app = app diff --git a/lib/galaxy/managers/base.py b/lib/galaxy/managers/base.py index 298ebc0b2b3..46c30ba1296 100644 --- a/lib/galaxy/managers/base.py +++ b/lib/galaxy/managers/base.py @@ -48,12 +48,17 @@ from sqlalchemy.orm import Query from sqlalchemy.orm.scoping import scoped_session from typing_extensions import Protocol -from galaxy import exceptions -from galaxy import model +from galaxy import ( + exceptions, + model, +) from galaxy.model import tool_shed_install from galaxy.schema import FilterQueryParams from galaxy.security.idencoding import IdEncodingHelper -from galaxy.structured_app import BasicSharedApp, MinimalManagerApp +from galaxy.structured_app import ( + BasicSharedApp, + MinimalManagerApp, +) from galaxy.web import url_for as gx_url_for log = logging.getLogger(__name__) @@ -88,20 +93,34 @@ def security_check(trans, item, check_ownership=False, check_accessible=False): # Verify ownership: there is a current user and that user is the same as the item's if check_ownership: if not trans.user: - raise exceptions.ItemOwnershipException("Must be logged in to manage Galaxy items", type='error') + raise exceptions.ItemOwnershipException("Must be logged in to manage Galaxy items", type="error") if item.user != trans.user: - raise exceptions.ItemOwnershipException(f"{item.__class__.__name__} is not owned by the current user", type='error') + raise exceptions.ItemOwnershipException( + f"{item.__class__.__name__} is not owned by the current user", type="error" + ) # Verify accessible: # if it's part of a lib - can they access via security # if it's something else (sharable) have they been added to the item's users_shared_with_dot_users if check_accessible: - if type(item) in (trans.app.model.LibraryFolder, trans.app.model.LibraryDatasetDatasetAssociation, trans.app.model.LibraryDataset): + if type(item) in ( + trans.app.model.LibraryFolder, + trans.app.model.LibraryDatasetDatasetAssociation, + trans.app.model.LibraryDataset, + ): if not trans.app.security_agent.can_access_library_item(trans.get_current_user_roles(), item, trans.user): - raise exceptions.ItemAccessibilityException(f"{item.__class__.__name__} is not accessible to the current user", type='error') + raise exceptions.ItemAccessibilityException( + f"{item.__class__.__name__} is not accessible to the current user", type="error" + ) else: - if (item.user != trans.user) and (not item.importable) and (trans.user not in item.users_shared_with_dot_users): - raise exceptions.ItemAccessibilityException(f"{item.__class__.__name__} is not accessible to the current user", type='error') + if ( + (item.user != trans.user) + and (not item.importable) + and (trans.user not in item.users_shared_with_dot_users) + ): + raise exceptions.ItemAccessibilityException( + f"{item.__class__.__name__} is not accessible to the current user", type="error" + ) return item @@ -110,7 +129,7 @@ def get_class(class_name): Returns the class object that a string denotes. Without this method, we'd have to do eval(). """ - if class_name == 'ToolShedRepository': + if class_name == "ToolShedRepository": item_class = tool_shed_install.ToolShedRepository else: if not hasattr(model, class_name): @@ -153,11 +172,13 @@ def get_object(trans, id, class_name, check_ownership=False, check_accessible=Fa if check_ownership or check_accessible: security_check(trans, item, check_ownership, check_accessible) if deleted is True and not item.deleted: - raise exceptions.ItemDeletionException('%s "%s" is not deleted' - % (class_name, getattr(item, 'name', id)), type="warning") + raise exceptions.ItemDeletionException( + '%s "%s" is not deleted' % (class_name, getattr(item, "name", id)), type="warning" + ) elif deleted is False and item.deleted: - raise exceptions.ItemDeletionException('%s "%s" is deleted' - % (class_name, getattr(item, 'name', id)), type="warning") + raise exceptions.ItemDeletionException( + '%s "%s" is deleted' % (class_name, getattr(item, "name", id)), type="warning" + ) return item @@ -188,6 +209,7 @@ class ModelManager: Provides common queries and CRUD operations as a (hopefully) light layer over the ORM. """ + model_class: Type[model._HasTable] foreign_key_name: str app: BasicSharedApp @@ -207,7 +229,14 @@ class ModelManager: return item # .... query foundation wrapper - def query(self, eagerloads: bool = True, filters=None, order_by=None, limit: Optional[int] = None, offset: Optional[int] = None) -> Query: + def query( + self, + eagerloads: bool = True, + filters=None, + order_by=None, + limit: Optional[int] = None, + offset: Optional[int] = None, + ) -> Query: """ Return a basic query from model_class, filters, order_by, and limit and offset. @@ -219,7 +248,9 @@ class ModelManager: query = query.enable_eagerloads(False) return self._filter_and_order_query(query, filters=filters, order_by=order_by, limit=limit, offset=offset) - def _filter_and_order_query(self, query: Query, filters=None, order_by=None, limit: Optional[int] = None, offset: Optional[int] = None) -> Query: + def _filter_and_order_query( + self, query: Query, filters=None, order_by=None, limit: Optional[int] = None, offset: Optional[int] = None + ) -> Query: # TODO: not a lot of functional cohesion here query = self._apply_orm_filters(query, filters) query = self._apply_order_by(query, order_by) @@ -267,7 +298,7 @@ class ModelManager: """ Returns a tuple of columns for the default order when getting multiple models. """ - return (self.model_class.table.c.create_time, ) + return (self.model_class.table.c.create_time,) def _apply_orm_limit_offset(self, query: Query, limit: Optional[int], offset: Optional[int]) -> Query: """ @@ -333,8 +364,7 @@ class ModelManager: orm_filters, fn_filters = self._split_filters(filters) if not fn_filters: # if no fn_filtering required, we can use the 'all orm' version with limit offset - return self._orm_list(filters=orm_filters, order_by=order_by, - limit=limit, offset=offset, **kwargs) + return self._orm_list(filters=orm_filters, order_by=order_by, limit=limit, offset=offset, **kwargs) # fn filters will change the number of items returnable by limit/offset - remove them here from the orm query query = self.query(filters=orm_filters, order_by=order_by, limit=None, offset=None, **kwargs) @@ -357,11 +387,11 @@ class ModelManager: if not isinstance(filters, list): filters = [filters] for filter_ in filters: - if not hasattr(filter_, 'filter_type'): + if not hasattr(filter_, "filter_type"): orm_filters.append(filter_) - elif filter_.filter_type == 'function': + elif filter_.filter_type == "function": fn_filters.append(filter_.filter) - elif filter_.filter_type == 'orm_function': + elif filter_.filter_type == "orm_function": orm_filters.append(filter_.filter(self.model_class)) else: orm_filters.append(filter_.filter) @@ -430,7 +460,7 @@ class ModelManager: If an id in ids is not found or if an item in items doesn't have a given id, they will not be in the returned list. """ - ID_ATTR_NAME = 'id' + ID_ATTR_NAME = "id" # TODO:?? aside from sqlalx.get mentioned above, I haven't seen an in-SQL way # to make this happen. This may not be the most efficient way either. # NOTE: that this isn't sorting by id - this is matching the order in items to the order in ids @@ -462,7 +492,7 @@ class ModelManager: """ Clone or copy an item. """ - raise exceptions.NotImplemented('Abstract method') + raise exceptions.NotImplemented("Abstract method") def update(self, item, new_values, flush=True, **kwargs): """ @@ -502,7 +532,7 @@ class ModelManager: # return item -T = TypeVar('T') +T = TypeVar("T") # ---- code for classes that use one *main* model manager @@ -515,7 +545,9 @@ class HasAModelManager(Generic[T]): """ #: the class used to create this serializer's generically accessible model_manager - model_manager_class: Type[T] # ideally this would be Type[ModelManager] but HistoryContentsManager cannot be a ModelManager + model_manager_class: Type[ + T + ] # ideally this would be Type[ModelManager] but HistoryContentsManager cannot be a ModelManager # examples where this doesn't really work are ConfigurationSerializer (no manager) # and contents (2 managers) app: MinimalManagerApp @@ -554,7 +586,6 @@ class SkipAttribute(Exception): class Serializer(Protocol): - def __call__(self, item: Any, key: str, **context) -> Any: ... @@ -576,6 +607,7 @@ class ModelSerializer(HasAModelManager[T]): keys_to_serialize = [ 'id', 'name', 'attr1', 'attr2', ... ] item_dict = MySerializer.serialize( my_item, keys_to_serialize ) """ + #: 'service' to use for getting urls - use class var to allow overriding when testing url_for = staticmethod(gx_url_for) default_view: Optional[str] @@ -609,11 +641,13 @@ class ModelSerializer(HasAModelManager[T]): Register a map of attribute keys -> serializing functions that will serialize the attribute. """ - self.serializers.update({ - 'id': self.serialize_id, - 'create_time': self.serialize_date, - 'update_time': self.serialize_date, - }) + self.serializers.update( + { + "id": self.serialize_id, + "create_time": self.serialize_date, + "update_time": self.serialize_date, + } + ) def add_view(self, view_name, key_list, include_keys_from=None): """ @@ -651,7 +685,7 @@ class ModelSerializer(HasAModelManager[T]): # ignore bad/unreg keys return returned - def skip(self, msg='skipped'): + def skip(self, msg="skipped"): """ To be called from inside a serializer to skip it. @@ -693,7 +727,7 @@ class ModelSerializer(HasAModelManager[T]): """ Serialize an type-id for `item`. """ - TYPE_ID_SEP = '-' + TYPE_ID_SEP = "-" type_id = getattr(item, key) if type_id is None: return None @@ -740,7 +774,7 @@ class ModelSerializer(HasAModelManager[T]): if view is None: view = self.default_view if view not in self.views: - raise ModelSerializingError('unknown view', view=view, available_views=self.views) + raise ModelSerializingError("unknown view", view=view, available_views=self.views) return self.views[view][:] @@ -759,7 +793,7 @@ class ModelValidator: :raises exceptions.RequestParameterInvalidException: if not an instance. """ if not isinstance(val, types): - msg = f'must be a type: {str(types)}' + msg = f"must be a type: {str(types)}" raise exceptions.RequestParameterInvalidException(msg, key=key, val=val) return val @@ -812,7 +846,7 @@ class ModelValidator: """ # TODO: is this correct? if val is None: - return '?' + return "?" # currently, data source sites like UCSC are able to set the genome build to non-local build names # afterwards, attempting to validate the whole model will choke here # for genome_build_shortname, longname in self.app.genome_builds.get_genome_build_names( trans=trans ): @@ -828,7 +862,6 @@ class ModelValidator: class Deserializer(Protocol): - def __call__(self, item: Any, key: Any, val: Any, **kwargs) -> Any: ... @@ -838,6 +871,7 @@ class ModelDeserializer(HasAModelManager[T]): An object that converts an incoming serialized dict into values that can be directly assigned to an item's attributes and assigns them. """ + validate = ModelValidator() app: MinimalManagerApp @@ -894,7 +928,7 @@ class ModelDeserializer(HasAModelManager[T]): return val def deserialize_basestring(self, item, key, val, convert_none_to_empty=False, **context): - val = '' if (convert_none_to_empty and val is None) else self.validate.basestring(key, val) + val = "" if (convert_none_to_empty and val is None) else self.validate.basestring(key, val) return self.default_deserializer(item, key, val, **context) def deserialize_bool(self, item, key, val, **context): @@ -939,6 +973,7 @@ class ModelFilterParser(HasAModelManager): These might be safely be replaced in the future by creating SQLAlchemy hybrid properties or more thoroughly mapping derived values. """ + # ??: this class kindof 'lives' in both the world of the controllers/param-parsing and to models/orm # (as the model informs how the filter params are parsed) # I have no great idea where this 'belongs', so it's here for now @@ -955,7 +990,9 @@ class ModelFilterParser(HasAModelManager): super().__init__(app, **kwargs) #: regex for testing/dicing iso8601 date strings, with optional time and ms, but allowing only UTC timezone - self.date_string_re = re.compile(r'^(\d{4}\-\d{2}\-\d{2})[T| ]{0,1}(\d{2}:\d{2}:\d{2}(?:\.\d{1,6}){0,1}){0,1}Z{0,1}$') + self.date_string_re = re.compile( + r"^(\d{4}\-\d{2}\-\d{2})[T| ]{0,1}(\d{2}:\d{2}:\d{2}(?:\.\d{1,6}){0,1}){0,1}Z{0,1}$" + ) # dictionary containing parsing data for ORM/SQLAlchemy-based filters # ..note: although kind of a pain in the ass and verbose, opt-in/allowlisting allows more control @@ -973,27 +1010,29 @@ class ModelFilterParser(HasAModelManager): Set up, extend, or alter `orm_filter_parsers` and `fn_filter_parsers`. """ # note: these are the default filters for all models - self.orm_filter_parsers.update({ - # (prob.) applicable to all models - 'id': {'op': ('in')}, - 'encoded_id': {'column': 'id', 'op': ('in'), 'val': self.parse_id_list}, - # dates can be directly passed through the orm into a filter (no need to parse into datetime object) - 'extension': {'op': ('eq', 'like', 'in')}, - 'create_time': {'op': ('le', 'ge', 'lt', 'gt'), 'val': self.parse_date}, - 'update_time': {'op': ('le', 'ge', 'lt', 'gt'), 'val': self.parse_date}, - }) + self.orm_filter_parsers.update( + { + # (prob.) applicable to all models + "id": {"op": ("in")}, + "encoded_id": {"column": "id", "op": ("in"), "val": self.parse_id_list}, + # dates can be directly passed through the orm into a filter (no need to parse into datetime object) + "extension": {"op": ("eq", "like", "in")}, + "create_time": {"op": ("le", "ge", "lt", "gt"), "val": self.parse_date}, + "update_time": {"op": ("le", "ge", "lt", "gt"), "val": self.parse_date}, + } + ) def build_filter_params( self, query_params: FilterQueryParams, - filter_attr_key: str = 'q', - filter_value_key: str = 'qv', - attr_op_split_char: str = '-', + filter_attr_key: str = "q", + filter_value_key: str = "qv", + attr_op_split_char: str = "-", ) -> List[Tuple[str, str, str]]: """ Builds a list of tuples containing filtering information in the form of (attribute, operator, value). """ - DEFAULT_OP = 'eq' + DEFAULT_OP = "eq" qdict = query_params.dict(exclude_defaults=True) if filter_attr_key not in qdict: return [] @@ -1059,12 +1098,13 @@ class ModelFilterParser(HasAModelManager): # by convention, assume most val parsers raise ValueError except ValueError as val_err: - raise exceptions.RequestParameterInvalidException('unparsable value for filter', - column=attr, operation=op, value=val, ValueError=str(val_err)) + raise exceptions.RequestParameterInvalidException( + "unparsable value for filter", column=attr, operation=op, value=val, ValueError=str(val_err) + ) # if neither of the above work, raise an error with how-to info # TODO: send back all valid filter keys in exception for added user help - raise exceptions.RequestParameterInvalidException('bad filter', column=attr, operation=op) + raise exceptions.RequestParameterInvalidException("bad filter", column=attr, operation=op) # ---- fn filters def _parse_fn_filter(self, attr, op, val): @@ -1077,13 +1117,13 @@ class ModelFilterParser(HasAModelManager): attr_map = self.fn_filter_parsers.get(attr, None) if not attr_map: return None - allowed_ops = attr_map['op'] + allowed_ops = attr_map["op"] # allowed ops is a map here, op => fn filter_fn = allowed_ops.get(op, None) if not filter_fn: return None # parse the val from string using the 'val' parser if present (otherwise, leave as string) - val_parser = attr_map.get('val', None) + val_parser = attr_map.get("val", None) if val_parser: val = val_parser(val) @@ -1106,8 +1146,8 @@ class ModelFilterParser(HasAModelManager): return self.parsed_filter(filter_type="orm_function", filter=column_map(attr, op, val)) # attr must be an allowlisted column by attr name or by key passed in column_map # note: column_map[ 'column' ] takes precedence - if 'column' in column_map: - attr = column_map['column'] + if "column" in column_map: + attr = column_map["column"] column = self.model_class.table.columns.get(attr) if column is None: # could be a property (hybrid_property, etc.) - assume we can make a filter from it @@ -1117,7 +1157,7 @@ class ModelFilterParser(HasAModelManager): return None # op must be allowlisted: contained in the list orm_filter_list[ attr ][ 'op' ] - allowed_ops = column_map['op'] + allowed_ops = column_map["op"] if op not in allowed_ops: return None op = self._convert_op_string_to_fn(column, op) @@ -1125,7 +1165,7 @@ class ModelFilterParser(HasAModelManager): return None # parse the val from string using the 'val' parser if present (otherwise, leave as string) - val_parser = column_map.get('val', None) + val_parser = column_map.get("val", None) if val_parser: val = val_parser(val) @@ -1133,7 +1173,7 @@ class ModelFilterParser(HasAModelManager): return self.parsed_filter(filter_type="orm", filter=orm_filter) #: these are the easier/shorter string equivalents to the python operator fn names that need '__' around them - UNDERSCORED_OPS = ('lt', 'le', 'eq', 'ne', 'ge', 'gt') + UNDERSCORED_OPS = ("lt", "le", "eq", "ne", "ge", "gt") def _convert_op_string_to_fn(self, column, op_string): """ @@ -1144,8 +1184,8 @@ class ModelFilterParser(HasAModelManager): fn_name = op_string if op_string in self.UNDERSCORED_OPS: fn_name = f"__{op_string}__" - elif op_string == 'in': - fn_name = 'in_' + elif op_string == "in": + fn_name = "in_" # get the column fn using the op_string and error if not a callable attr # TODO: special case 'not in' - or disallow? @@ -1157,15 +1197,15 @@ class ModelFilterParser(HasAModelManager): # ---- preset fn_filters: dictionaries of standard filter ops for standard datatypes def string_standard_ops(self, key): return { - 'op': { - 'eq': lambda i, v: v == getattr(i, key), - 'contains': lambda i, v: v in getattr(i, key), + "op": { + "eq": lambda i, v: v == getattr(i, key), + "contains": lambda i, v: v in getattr(i, key), } } # --- more parsers! yay! # TODO: These should go somewhere central - we've got ~6 parser modules/sections now - def parse_id_list(self, id_list_string, sep=','): + def parse_id_list(self, id_list_string, sep=","): """ Split `id_list_string` at `sep`. """ @@ -1173,7 +1213,7 @@ class ModelFilterParser(HasAModelManager): id_list = [self.app.security.decode_id(id_) for id_ in id_list_string.split(sep)] return id_list - def parse_int_list(self, int_list_string, sep=','): + def parse_int_list(self, int_list_string, sep=","): """ Split `int_list_string` at `sep` and parse as ints. """ @@ -1192,15 +1232,15 @@ class ModelFilterParser(HasAModelManager): try: epoch = float(date_string) datetime_obj = datetime.datetime.fromtimestamp(epoch) - return datetime_obj.isoformat(sep=' ') + return datetime_obj.isoformat(sep=" ") except ValueError: pass match = self.date_string_re.match(date_string) if match: - date_string = ' '.join(group for group in match.groups() if group) + date_string = " ".join(group for group in match.groups() if group) return date_string - raise ValueError('datetime strings must be in the ISO 8601 format and in the UTC') + raise ValueError("datetime strings must be in the ISO 8601 format and in the UTC") def parse_bool(bool_string: Union[str, bool]) -> bool: @@ -1208,9 +1248,9 @@ def parse_bool(bool_string: Union[str, bool]) -> bool: Parse a boolean from a string. """ # Be strict here to remove complexity of options (but allow already parsed). - if bool_string in ('True', True): + if bool_string in ("True", True): return True - if bool_string in ('False', False): + if bool_string in ("False", False): return False raise ValueError(f"invalid boolean: {str(bool_string)}") diff --git a/lib/galaxy/managers/citations.py b/lib/galaxy/managers/citations.py index 3e97901e192..84e9ab17bea 100644 --- a/lib/galaxy/managers/citations.py +++ b/lib/galaxy/managers/citations.py @@ -12,7 +12,6 @@ log = logging.getLogger(__name__) class CitationsManager: - def __init__(self, app: BasicSharedApp) -> None: self.app = app self.doi_cache = DoiCache(app.config) @@ -37,18 +36,17 @@ class CitationsManager: class DoiCache: - def __init__(self, config): cache_opts = { - 'cache.type': getattr(config, 'citation_cache_type', 'file'), - 'cache.data_dir': getattr(config, 'citation_cache_data_dir', None), - 'cache.lock_dir': getattr(config, 'citation_cache_lock_dir', None), + "cache.type": getattr(config, "citation_cache_type", "file"), + "cache.data_dir": getattr(config, "citation_cache_data_dir", None), + "cache.lock_dir": getattr(config, "citation_cache_lock_dir", None), } - self._cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache('doi') + self._cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache("doi") def _raw_get_bibtex(self, doi): doi_url = f"https://doi.org/{doi}" - headers = {'Accept': 'application/x-bibtex'} + headers = {"Accept": "application/x-bibtex"} req = requests.get(doi_url, headers=headers, timeout=DEFAULT_SOCKET_TIMEOUT) req.encoding = req.apparent_encoding return req.text @@ -62,7 +60,7 @@ def parse_citation(elem, citation_manager): """ Parse an abstract citation entry from the specified XML element. """ - citation_type = elem.attrib.get('type', None) + citation_type = elem.attrib.get("type", None) citation_class = CITATION_CLASSES.get(citation_type, None) if not citation_class: log.warning(f"Unknown or unspecified citation type: {citation_type}") @@ -75,7 +73,6 @@ def parse_citation(elem, citation_manager): class CitationCollection: - def __init__(self): self.citations = [] @@ -97,7 +94,6 @@ class CitationCollection: class BaseCitation: - def to_dict(self, citation_format): if citation_format == "bibtex": return dict( @@ -119,7 +115,6 @@ class BaseCitation: class BibtexCitation(BaseCitation): - def __init__(self, elem, citation_manager): self.raw_bibtex = elem.text.strip() @@ -152,7 +147,9 @@ class DoiCitation(BaseCitation): return """@MISC{{{doi}, DOI = {{{doi}}}, note = {{Failed to fetch BibTeX for DOI.}} - }}""".format(doi=self.__doi) + }}""".format( + doi=self.__doi + ) else: return self.raw_bibtex diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index 66af9720068..1d04ea4f2e4 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -5,20 +5,25 @@ Manager and serializer for cloud-based storages. import json import logging -from galaxy import model -from galaxy import util +from galaxy import ( + model, + util, +) from galaxy.exceptions import ( ItemAccessibilityException, MessageException, ObjectNotFound, RequestParameterInvalidException, - RequestParameterMissingException + RequestParameterMissingException, ) from galaxy.managers import sharable from galaxy.util import Params try: - from cloudbridge.factory import CloudProviderFactory, ProviderList + from cloudbridge.factory import ( + CloudProviderFactory, + ProviderList, + ) except ImportError: CloudProviderFactory = None ProviderList = None @@ -68,46 +73,50 @@ class CloudManager(sharable.SharableModelManager): :return: a cloudbridge connection to the specified provider. """ missing_credentials = [] - if provider == 'aws': - access = credentials.get('access_key', None) + if provider == "aws": + access = credentials.get("access_key", None) if access is None: access = credentials.get("AccessKeyId", None) if access is None: - missing_credentials.append('access_key') - secret = credentials.get('secret_key', None) + missing_credentials.append("access_key") + secret = credentials.get("secret_key", None) if secret is None: secret = credentials.get("SecretAccessKey", None) if secret is None: - missing_credentials.append('secret_key') + missing_credentials.append("secret_key") if len(missing_credentials) > 0: - raise RequestParameterMissingException("The following required key(s) are missing from the provided " - "credentials object: {}".format(missing_credentials)) + raise RequestParameterMissingException( + "The following required key(s) are missing from the provided " + "credentials object: {}".format(missing_credentials) + ) session_token = credentials.get("SessionToken") - config = {'aws_access_key': access, - 'aws_secret_key': secret, - "aws_session_token": session_token} + config = {"aws_access_key": access, "aws_secret_key": secret, "aws_session_token": session_token} connection = CloudProviderFactory().create_provider(ProviderList.AWS, config) elif provider == "azure": - subscription = credentials.get('subscription_id', None) + subscription = credentials.get("subscription_id", None) if subscription is None: - missing_credentials.append('subscription_id') - client = credentials.get('client_id', None) + missing_credentials.append("subscription_id") + client = credentials.get("client_id", None) if client is None: - missing_credentials.append('client_id') - secret = credentials.get('secret', None) + missing_credentials.append("client_id") + secret = credentials.get("secret", None) if secret is None: - missing_credentials.append('secret') - tenant = credentials.get('tenant', None) + missing_credentials.append("secret") + tenant = credentials.get("tenant", None) if tenant is None: - missing_credentials.append('tenant') + missing_credentials.append("tenant") if len(missing_credentials) > 0: - raise RequestParameterMissingException("The following required key(s) are missing from the provided " - "credentials object: {}".format(missing_credentials)) + raise RequestParameterMissingException( + "The following required key(s) are missing from the provided " + "credentials object: {}".format(missing_credentials) + ) - config = {'azure_subscription_id': subscription, - 'azure_client_id': client, - 'azure_secret': secret, - 'azure_tenant': tenant} + config = { + "azure_subscription_id": subscription, + "azure_client_id": client, + "azure_secret": secret, + "azure_tenant": tenant, + } storage_account = credentials.get("storage_account") if storage_account: config["azure_storage_account"] = storage_account @@ -116,40 +125,46 @@ class CloudManager(sharable.SharableModelManager): config["azure_resource_group"] = resource_group connection = CloudProviderFactory().create_provider(ProviderList.AZURE, config) elif provider == "openstack": - username = credentials.get('username', None) + username = credentials.get("username", None) if username is None: - missing_credentials.append('username') - password = credentials.get('password', None) + missing_credentials.append("username") + password = credentials.get("password", None) if password is None: - missing_credentials.append('password') - auth_url = credentials.get('auth_url', None) + missing_credentials.append("password") + auth_url = credentials.get("auth_url", None) if auth_url is None: - missing_credentials.append('auth_url') - prj_name = credentials.get('project_name', None) + missing_credentials.append("auth_url") + prj_name = credentials.get("project_name", None) if prj_name is None: - missing_credentials.append('project_name') - prj_domain_name = credentials.get('project_domain_name', None) + missing_credentials.append("project_name") + prj_domain_name = credentials.get("project_domain_name", None) if prj_domain_name is None: - missing_credentials.append('project_domain_name') - user_domain_name = credentials.get('user_domain_name', None) + missing_credentials.append("project_domain_name") + user_domain_name = credentials.get("user_domain_name", None) if user_domain_name is None: - missing_credentials.append('user_domain_name') + missing_credentials.append("user_domain_name") if len(missing_credentials) > 0: - raise RequestParameterMissingException("The following required key(s) are missing from the provided " - "credentials object: {}".format(missing_credentials)) - config = {'os_username': username, - 'os_password': password, - 'os_auth_url': auth_url, - 'os_project_name': prj_name, - 'os_project_domain_name': prj_domain_name, - 'os_user_domain_name': user_domain_name} + raise RequestParameterMissingException( + "The following required key(s) are missing from the provided " + "credentials object: {}".format(missing_credentials) + ) + config = { + "os_username": username, + "os_password": password, + "os_auth_url": auth_url, + "os_project_name": prj_name, + "os_project_domain_name": prj_domain_name, + "os_user_domain_name": user_domain_name, + } connection = CloudProviderFactory().create_provider(ProviderList.OPENSTACK, config) elif provider == "gcp": config = {"gcp_service_creds_dict": credentials} connection = CloudProviderFactory().create_provider(ProviderList.GCP, config) else: - raise RequestParameterInvalidException("Unrecognized provider '{}'; the following are the supported " - "providers: {}.".format(provider, SUPPORTED_PROVIDERS.keys())) + raise RequestParameterInvalidException( + "Unrecognized provider '{}'; the following are the supported " + "providers: {}.".format(provider, SUPPORTED_PROVIDERS.keys()) + ) # The authorization-assertion mechanism of Cloudbridge assumes a user has an elevated privileges, # such as Admin-level access to all resources (see https://github.com/CloudVE/cloudbridge/issues/135). @@ -169,29 +184,33 @@ class CloudManager(sharable.SharableModelManager): @staticmethod def _get_inputs(obj, key, input_args): space_to_tab = None - if input_args.get('space_to_tab', "").lower() == "true": + if input_args.get("space_to_tab", "").lower() == "true": space_to_tab = "Yes" - elif input_args.get('space_to_tab', "").lower() not in ["false", ""]: + elif input_args.get("space_to_tab", "").lower() not in ["false", ""]: raise RequestParameterInvalidException( "The valid values for `space_to_tab` argument are `true` and `false`; received {}".format( - input_args.get('space_to_tab'))) + input_args.get("space_to_tab") + ) + ) to_posix_lines = None - if input_args.get('to_posix_lines', "").lower() == "true": + if input_args.get("to_posix_lines", "").lower() == "true": to_posix_lines = "Yes" - elif input_args.get('to_posix_lines', "").lower() not in ["false", ""]: + elif input_args.get("to_posix_lines", "").lower() not in ["false", ""]: raise RequestParameterInvalidException( "The valid values for `to_posix_lines` argument are `true` and `false`; received {}".format( - input_args.get('to_posix_lines'))) + input_args.get("to_posix_lines") + ) + ) return { - 'dbkey': input_args.get("dbkey", "?"), - 'file_type': input_args.get("file_type", "auto"), - 'files_0|type': 'upload_dataset', - 'files_0|space_to_tab': space_to_tab, - 'files_0|to_posix_lines': to_posix_lines, - 'files_0|NAME': obj, - 'files_0|url_paste': key.generate_url(expires_in=SINGED_URL_TTL), + "dbkey": input_args.get("dbkey", "?"), + "file_type": input_args.get("file_type", "auto"), + "files_0|type": "upload_dataset", + "files_0|space_to_tab": space_to_tab, + "files_0|to_posix_lines": to_posix_lines, + "files_0|NAME": obj, + "files_0|url_paste": key.generate_url(expires_in=SINGED_URL_TTL), } def get(self, trans, history_id, bucket_name, objects, authz_id, input_args=None): @@ -233,14 +252,18 @@ class CloudManager(sharable.SharableModelManager): if input_args is None: input_args = {} - if not hasattr(trans.app, 'authnz_manager'): - err_msg = "The OpenID Connect protocol, a required feature for getting data from cloud, " \ - "is not enabled on this Galaxy instance." + if not hasattr(trans.app, "authnz_manager"): + err_msg = ( + "The OpenID Connect protocol, a required feature for getting data from cloud, " + "is not enabled on this Galaxy instance." + ) log.debug(err_msg) raise MessageException(err_msg) cloudauthz = trans.app.authnz_manager.try_get_authz_config(trans.sa_session, trans.user.id, authz_id) - credentials = trans.app.authnz_manager.get_cloud_access_credentials(cloudauthz, trans.sa_session, trans.user.id, trans.request) + credentials = trans.app.authnz_manager.get_cloud_access_credentials( + cloudauthz, trans.sa_session, trans.user.id, trans.request + ) connection = self.configure_provider(cloudauthz.provider, credentials) try: bucket = connection.storage.buckets.get(bucket_name) @@ -254,28 +277,35 @@ class CloudManager(sharable.SharableModelManager): try: key = bucket.objects.get(obj) except Exception as e: - raise MessageException(f"The following error occurred while getting the object {obj}: {util.unicodify(e)}") + raise MessageException( + f"The following error occurred while getting the object {obj}: {util.unicodify(e)}" + ) if key is None: log.exception( "Could not get object `{}` for user `{}`. Object may not exist, or the provided credentials are " - "invalid or not authorized to read the bucket/object.".format(obj, trans.user.id)) + "invalid or not authorized to read the bucket/object.".format(obj, trans.user.id) + ) raise ObjectNotFound( "Could not get the object `{}`. Please check if the object exists, and credentials are valid and " - "authorized to read the bucket and object. ".format(obj)) + "authorized to read the bucket and object. ".format(obj) + ) params = Params(self._get_inputs(obj, key, input_args), sanitize=False) incoming = params.__dict__ history = trans.sa_session.query(trans.app.model.History).get(history_id) if not history: raise ObjectNotFound(f"History with ID `{trans.app.security.encode_id(history_id)}` not found.") - output = trans.app.toolbox.get_tool('upload1').handle_input(trans, incoming, history=history) + output = trans.app.toolbox.get_tool("upload1").handle_input(trans, incoming, history=history) - job_errors = output.get('job_errors', []) + job_errors = output.get("job_errors", []) if job_errors: - raise ValueError('Following error occurred while getting the given object(s) from {}: {}'.format( - cloudauthz.provider, job_errors)) + raise ValueError( + "Following error occurred while getting the given object(s) from {}: {}".format( + cloudauthz.provider, job_errors + ) + ) else: - for d in output['out_data']: + for d in output["out_data"]: datasets.append(d[1].dataset) return datasets @@ -319,9 +349,11 @@ class CloudManager(sharable.SharableModelManager): if CloudProviderFactory is None: raise Exception(NO_CLOUDBRIDGE_ERROR_MESSAGE) - if not hasattr(trans.app, 'authnz_manager'): - err_msg = "The OpenID Connect protocol, a required feature for sending data to cloud, " \ - "is not enabled on this Galaxy instance." + if not hasattr(trans.app, "authnz_manager"): + err_msg = ( + "The OpenID Connect protocol, a required feature for sending data to cloud, " + "is not enabled on this Galaxy instance." + ) log.debug(err_msg) raise MessageException(err_msg) @@ -346,32 +378,23 @@ class CloudManager(sharable.SharableModelManager): "bucket": bucket_name, "object_label": object_label, "filename": hda, - "overwrite_existing": overwrite_existing + "overwrite_existing": overwrite_existing, } incoming = (util.Params(args, sanitize=False)).__dict__ d2c = trans.app.toolbox.get_tool(SEND_TOOL, SEND_TOOL_VERSION) if not d2c: log.debug(f"Failed to get the `send` tool per user `{trans.user.id}` request.") - failed.append(json.dumps( - { - "object": object_label, - "error": "Unable to get the `send` tool." - })) + failed.append(json.dumps({"object": object_label, "error": "Unable to get the `send` tool."})) continue res = d2c.execute(trans, incoming, history=history) job = res[0] - sent.append(json.dumps( - { - "object": object_label, - "job_id": trans.app.security.encode_id(job.id) - })) + sent.append(json.dumps({"object": object_label, "job_id": trans.app.security.encode_id(job.id)})) except Exception as e: err_msg = f"maybe invalid or unauthorized credentials. {util.unicodify(e)}" - log.debug("Failed to send the dataset `{}` per user `{}` request to cloud, {}".format( - object_label, trans.user.id, err_msg)) - failed.append(json.dumps( - { - "object": object_label, - "error": err_msg - })) + log.debug( + "Failed to send the dataset `{}` per user `{}` request to cloud, {}".format( + object_label, trans.user.id, err_msg + ) + ) + failed.append(json.dumps({"object": object_label, "error": err_msg})) return sent, failed diff --git a/lib/galaxy/managers/cloudauthzs.py b/lib/galaxy/managers/cloudauthzs.py index 005fbe780eb..392bc0147d8 100644 --- a/lib/galaxy/managers/cloudauthzs.py +++ b/lib/galaxy/managers/cloudauthzs.py @@ -6,11 +6,11 @@ import logging from typing import Dict from galaxy import model -from galaxy.exceptions import ( - InternalServerError, +from galaxy.exceptions import InternalServerError +from galaxy.managers import ( + base, + sharable, ) -from galaxy.managers import base -from galaxy.managers import sharable log = logging.getLogger(__name__) @@ -18,32 +18,36 @@ log = logging.getLogger(__name__) class CloudAuthzManager(sharable.SharableModelManager): model_class = model.CloudAuthz - foreign_key_name = 'cloudauthz' + foreign_key_name = "cloudauthz" class CloudAuthzsSerializer(base.ModelSerializer): """ Interface/service object for serializing cloud authorizations (cloudauthzs) into dictionaries. """ + model_manager_class = CloudAuthzManager def __init__(self, app, **kwargs): super().__init__(app, **kwargs) self.cloudauthzs_manager = self.manager - self.default_view = 'summary' - self.add_view('summary', [ - 'id', - 'model_class', - 'user_id', - 'provider', - 'config', - 'authn_id', - 'last_update', - 'last_activity', - 'create_time', - 'description' - ]) + self.default_view = "summary" + self.add_view( + "summary", + [ + "id", + "model_class", + "user_id", + "provider", + "config", + "authn_id", + "last_update", + "last_activity", + "create_time", + "description", + ], + ) def add_serializers(self): super().add_serializers() @@ -53,16 +57,18 @@ class CloudAuthzsSerializer(base.ModelSerializer): # k : serialized dictionary key (e.g., 'model_class', 'provider'). # **c: a dictionary containing 'trans' and 'user' objects. serializers: Dict[str, base.Serializer] = { - 'id': lambda item, key, **context: self.app.security.encode_id(item.id), - 'model_class': lambda item, key, **context: 'CloudAuthz', - 'user_id': lambda item, key, **context: self.app.security.encode_id(item.user_id), - 'provider': lambda item, key, **context: str(item.provider), - 'config': lambda item, key, **context: item.config, - 'authn_id': lambda item, key, **context: self.app.security.encode_id(item.authn_id) if item.authn_id else None, - 'last_update': lambda item, key, **context: str(item.last_update), - 'last_activity': lambda item, key, **context: str(item.last_activity), - 'create_time': lambda item, key, **context: str(item.create_time), - 'description': lambda item, key, **context: str(item.description) + "id": lambda item, key, **context: self.app.security.encode_id(item.id), + "model_class": lambda item, key, **context: "CloudAuthz", + "user_id": lambda item, key, **context: self.app.security.encode_id(item.user_id), + "provider": lambda item, key, **context: str(item.provider), + "config": lambda item, key, **context: item.config, + "authn_id": lambda item, key, **context: self.app.security.encode_id(item.authn_id) + if item.authn_id + else None, + "last_update": lambda item, key, **context: str(item.last_update), + "last_activity": lambda item, key, **context: str(item.last_activity), + "create_time": lambda item, key, **context: str(item.create_time), + "description": lambda item, key, **context: str(item.description), } self.serializers.update(serializers) @@ -72,16 +78,19 @@ class CloudAuthzsDeserializer(base.ModelDeserializer): Service object for validating and deserializing dictionaries that update/alter cloudauthz configurations. """ + model_manager_class = CloudAuthzManager def add_deserializers(self): super().add_deserializers() - self.deserializers.update({ - 'authn_id': self.deserialize_and_validate_authn_id, - 'provider': self.default_deserializer, - 'config': self.default_deserializer, - 'description': self.default_deserializer - }) + self.deserializers.update( + { + "authn_id": self.deserialize_and_validate_authn_id, + "provider": self.default_deserializer, + "config": self.default_deserializer, + "description": self.default_deserializer, + } + ) def deserialize_and_validate_authn_id(self, item, key, val, **context): """ @@ -104,7 +113,7 @@ class CloudAuthzsDeserializer(base.ModelDeserializer): :return: decoded authentication ID. """ - decoded_authn_id = self.app.security.decode_id(val, object_name='authz') + decoded_authn_id = self.app.security.decode_id(val, object_name="authz") trans = context.get("trans") if trans is None: diff --git a/lib/galaxy/managers/collections.py b/lib/galaxy/managers/collections.py index 3318d3f0aa9..0ed7c6e9778 100644 --- a/lib/galaxy/managers/collections.py +++ b/lib/galaxy/managers/collections.py @@ -1,14 +1,22 @@ import logging -from typing import Any, Dict, List, Union +from typing import ( + Any, + Dict, + List, + Union, +) -from sqlalchemy.orm import joinedload, Query +from sqlalchemy.orm import ( + joinedload, + Query, +) from galaxy import model from galaxy.datatypes.registry import Registry from galaxy.exceptions import ( ItemAccessibilityException, MessageException, - RequestParameterInvalidException + RequestParameterInvalidException, ) from galaxy.managers.collections_util import validate_input_element_identifiers from galaxy.model.dataset_collections import builder @@ -18,9 +26,7 @@ from galaxy.model.dataset_collections.type_description import COLLECTION_TYPE_DE from galaxy.model.mapping import GalaxyModelMapping from galaxy.model.tags import GalaxyTagHandler from galaxy.security.idencoding import IdEncodingHelper -from galaxy.util import ( - validation -) +from galaxy.util import validation from .hdas import ( HDAManager, HistoryDatasetAssociationNoHistoryException, @@ -28,7 +34,6 @@ from .hdas import ( from .histories import HistoryManager from .lddas import LDDAManager - log = logging.getLogger(__name__) ERROR_INVALID_ELEMENTS_SPECIFICATION = "Create called with invalid parameters, must specify element identifiers." @@ -40,6 +45,7 @@ class DatasetCollectionManager: Abstraction for interfacing with dataset collections instance - ideally abstracts out model and plugin details. """ + ELEMENTS_UNINITIALIZED = object() def __init__( @@ -61,15 +67,39 @@ class DatasetCollectionManager: self.tag_handler = tag_handler.create_tag_handler_session() self.ldda_manager = ldda_manager - def precreate_dataset_collection_instance(self, trans, parent, name, structure, implicit_inputs=None, implicit_output_name=None, tags=None, completed_collection=None): + def precreate_dataset_collection_instance( + self, + trans, + parent, + name, + structure, + implicit_inputs=None, + implicit_output_name=None, + tags=None, + completed_collection=None, + ): # TODO: prebuild all required HIDs and send them in so no need to flush in between. - dataset_collection = self.precreate_dataset_collection(structure, allow_unitialized_element=implicit_output_name is not None, completed_collection=completed_collection, implicit_output_name=implicit_output_name) + dataset_collection = self.precreate_dataset_collection( + structure, + allow_unitialized_element=implicit_output_name is not None, + completed_collection=completed_collection, + implicit_output_name=implicit_output_name, + ) instance = self._create_instance_for_collection( - trans, parent, name, dataset_collection, implicit_inputs=implicit_inputs, implicit_output_name=implicit_output_name, flush=False, tags=tags + trans, + parent, + name, + dataset_collection, + implicit_inputs=implicit_inputs, + implicit_output_name=implicit_output_name, + flush=False, + tags=tags, ) return instance - def precreate_dataset_collection(self, structure, allow_unitialized_element=True, completed_collection=None, implicit_output_name=None): + def precreate_dataset_collection( + self, structure, allow_unitialized_element=True, completed_collection=None, implicit_output_name=None + ): has_structure = not structure.is_leaf and structure.children_known if not has_structure and allow_unitialized_element: dataset_collection = model.DatasetCollectionElement.UNINITIALIZED_ELEMENT @@ -88,13 +118,19 @@ class DatasetCollectionManager: if completed_collection and implicit_output_name: job = completed_collection[index] if job: - it = (jtiodca.dataset_collection for jtiodca in job.output_dataset_collections if jtiodca.name == implicit_output_name) + it = ( + jtiodca.dataset_collection + for jtiodca in job.output_dataset_collections + if jtiodca.name == implicit_output_name + ) element = next(it, None) if element is None: if substructure.is_leaf: element = model.DatasetCollectionElement.UNINITIALIZED_ELEMENT else: - element = self.precreate_dataset_collection(substructure, allow_unitialized_element=allow_unitialized_element) + element = self.precreate_dataset_collection( + substructure, allow_unitialized_element=allow_unitialized_element + ) element = model.DatasetCollectionElement( collection=dataset_collection, @@ -107,10 +143,25 @@ class DatasetCollectionManager: return dataset_collection - def create(self, trans, parent, name, collection_type, element_identifiers=None, - elements=None, implicit_collection_info=None, trusted_identifiers=None, - hide_source_items=False, tags=None, copy_elements=False, history=None, - set_hid=True, flush=True, completed_job=None, output_name=None): + def create( + self, + trans, + parent, + name, + collection_type, + element_identifiers=None, + elements=None, + implicit_collection_info=None, + trusted_identifiers=None, + hide_source_items=False, + tags=None, + copy_elements=False, + history=None, + set_hid=True, + flush=True, + completed_job=None, + output_name=None, + ): """ PRECONDITION: security checks on ability to add to parent occurred during load. @@ -138,17 +189,36 @@ class DatasetCollectionManager: implicit_inputs = [] if implicit_collection_info: - implicit_inputs = implicit_collection_info.get('implicit_inputs', []) + implicit_inputs = implicit_collection_info.get("implicit_inputs", []) implicit_output_name = None if implicit_collection_info: implicit_output_name = implicit_collection_info["implicit_output_name"] return self._create_instance_for_collection( - trans, parent, name, dataset_collection, implicit_inputs=implicit_inputs, implicit_output_name=implicit_output_name, tags=tags, set_hid=set_hid, flush=flush, + trans, + parent, + name, + dataset_collection, + implicit_inputs=implicit_inputs, + implicit_output_name=implicit_output_name, + tags=tags, + set_hid=set_hid, + flush=flush, ) - def _create_instance_for_collection(self, trans, parent, name, dataset_collection, implicit_output_name=None, implicit_inputs=None, tags=None, set_hid=True, flush=True): + def _create_instance_for_collection( + self, + trans, + parent, + name, + dataset_collection, + implicit_output_name=None, + implicit_inputs=None, + tags=None, + set_hid=True, + flush=True, + ): if isinstance(parent, model.History): dataset_collection_instance: Union[ model.HistoryDatasetCollectionAssociation, @@ -157,9 +227,7 @@ class DatasetCollectionManager: collection=dataset_collection, name=name, ) - assert isinstance( - dataset_collection_instance, model.HistoryDatasetCollectionAssociation - ) + assert isinstance(dataset_collection_instance, model.HistoryDatasetCollectionAssociation) if implicit_inputs: for input_name, input_collection in implicit_inputs: dataset_collection_instance.add_implicit_input_collection(input_name, input_collection) @@ -194,8 +262,16 @@ class DatasetCollectionManager: tags = self._append_tags(dataset_collection_instance, implicit_inputs, tags) return self.__persist(dataset_collection_instance, flush=flush) - def create_dataset_collection(self, trans, collection_type, element_identifiers=None, elements=None, - hide_source_items=None, copy_elements=False, history=None): + def create_dataset_collection( + self, + trans, + collection_type, + element_identifiers=None, + elements=None, + hide_source_items=None, + copy_elements=False, + history=None, + ): # Make sure at least one of these is None. assert element_identifiers is None or elements is None @@ -209,18 +285,22 @@ class DatasetCollectionManager: # If we have elements, this is an internal request, don't need to load # objects from identifiers. if elements is None: - elements = self._element_identifiers_to_elements(trans, - collection_type_description=collection_type_description, - element_identifiers=element_identifiers, - hide_source_items=hide_source_items, - copy_elements=copy_elements, - history=history) + elements = self._element_identifiers_to_elements( + trans, + collection_type_description=collection_type_description, + element_identifiers=element_identifiers, + hide_source_items=hide_source_items, + copy_elements=copy_elements, + history=history, + ) if history: history.add_pending_items() else: if has_subcollections: # Nested collection - recursively create collections as needed. - self.__recursively_create_collections_for_elements(trans, elements, hide_source_items, copy_elements=copy_elements, history=history) + self.__recursively_create_collections_for_elements( + trans, elements, hide_source_items, copy_elements=copy_elements, history=history + ) # else if elements is set, it better be an ordered dict! if elements is not self.ELEMENTS_UNINITIALIZED: @@ -233,10 +313,7 @@ class DatasetCollectionManager: def get_converters_for_collection(self, trans, id, datatypes_registry: Registry, instance_type="history"): dataset_collection_instance = self.get_dataset_collection_instance( - trans, - id=id, - instance_type=instance_type, - check_ownership=True + trans, id=id, instance_type=instance_type, check_ownership=True ) dbkeys_and_extensions = dataset_collection_instance.dataset_dbkeys_and_extensions_summary suitable_converters = set() @@ -249,7 +326,7 @@ class DatasetCollectionManager: for tgt_type, tgt_val in new_converters.items(): converter = (tgt_type, tgt_val) set_of_new_converters.add(converter) - if (first_extension is True): + if first_extension is True: suitable_converters = set_of_new_converters most_recent_datatype = datatype first_extension = False @@ -259,35 +336,48 @@ class DatasetCollectionManager: most_recent_datatype = datatype suitable_tool_ids = list() for tool in suitable_converters: - tool_info = {"tool_id": tool[1].id, "name": tool[1].name, "target_type": tool[0], "original_type": most_recent_datatype} + tool_info = { + "tool_id": tool[1].id, + "name": tool[1].name, + "target_type": tool[0], + "original_type": most_recent_datatype, + } suitable_tool_ids.append(tool_info) return suitable_tool_ids - def _element_identifiers_to_elements(self, - trans, - collection_type_description, - element_identifiers, - hide_source_items=False, - copy_elements=False, - history=None): + def _element_identifiers_to_elements( + self, + trans, + collection_type_description, + element_identifiers, + hide_source_items=False, + copy_elements=False, + history=None, + ): if collection_type_description.has_subcollections(): # Nested collection - recursively create collections and update identifiers. - self.__recursively_create_collections_for_identifiers(trans, element_identifiers, hide_source_items, copy_elements, history=history) + self.__recursively_create_collections_for_identifiers( + trans, element_identifiers, hide_source_items, copy_elements, history=history + ) new_collection = False for element_identifier in element_identifiers: - if element_identifier.get("src") == "new_collection" and element_identifier.get('collection_type') == '': + if element_identifier.get("src") == "new_collection" and element_identifier.get("collection_type") == "": new_collection = True - elements = self.__load_elements(trans=trans, - element_identifiers=element_identifier['element_identifiers'], - hide_source_items=hide_source_items, - copy_elements=copy_elements, - history=history) + elements = self.__load_elements( + trans=trans, + element_identifiers=element_identifier["element_identifiers"], + hide_source_items=hide_source_items, + copy_elements=copy_elements, + history=history, + ) if not new_collection: - elements = self.__load_elements(trans=trans, - element_identifiers=element_identifiers, - hide_source_items=hide_source_items, - copy_elements=copy_elements, - history=history) + elements = self.__load_elements( + trans=trans, + element_identifiers=element_identifiers, + hide_source_items=hide_source_items, + copy_elements=copy_elements, + history=history, + ) return elements def _append_tags(self, dataset_collection_instance, implicit_inputs=None, tags=None): @@ -303,7 +393,9 @@ class DatasetCollectionManager: return builder.BoundCollectionBuilder(dataset_collection) def delete(self, trans, instance_type, id, recursive=False, purge=False): - dataset_collection_instance = self.get_dataset_collection_instance(trans, instance_type, id, check_ownership=True) + dataset_collection_instance = self.get_dataset_collection_instance( + trans, instance_type, id, check_ownership=True + ) dataset_collection_instance.deleted = True trans.sa_session.add(dataset_collection_instance) @@ -312,7 +404,9 @@ class DatasetCollectionManager: try: self.hda_manager.error_unless_owner(dataset, user=trans.get_user(), current_history=trans.history) except HistoryDatasetAssociationNoHistoryException: - log.info(f"Cannot delete HistoryDatasetAssociation {dataset.id}, HistoryDatasetAssociation has no associated History, cannot verify owner") + log.info( + f"Cannot delete HistoryDatasetAssociation {dataset.id}, HistoryDatasetAssociation has no associated History, cannot verify owner" + ) continue if not dataset.deleted: dataset.deleted = True @@ -323,13 +417,15 @@ class DatasetCollectionManager: trans.sa_session.flush() def update(self, trans, instance_type, id, payload): - dataset_collection_instance = self.get_dataset_collection_instance(trans, instance_type, id, check_ownership=True) + dataset_collection_instance = self.get_dataset_collection_instance( + trans, instance_type, id, check_ownership=True + ) if trans.user is None: anon_allowed_payload = {} - if 'deleted' in payload: - anon_allowed_payload['deleted'] = payload['deleted'] - if 'visible' in payload: - anon_allowed_payload['visible'] = payload['visible'] + if "deleted" in payload: + anon_allowed_payload["deleted"] = payload["deleted"] + if "visible" in payload: + anon_allowed_payload["visible"] = payload["visible"] payload = self._validate_and_parse_update_payload(anon_allowed_payload) else: payload = self._validate_and_parse_update_payload(payload) @@ -360,23 +456,27 @@ class DatasetCollectionManager: changed = dataset_collection_instance.set_from_dict(new_data) # the rest (often involving the trans) - do here - if 'annotation' in new_data.keys() and trans.get_user(): - dataset_collection_instance.add_item_annotation(trans.sa_session, trans.get_user(), dataset_collection_instance, new_data['annotation']) - changed['annotation'] = new_data['annotation'] + if "annotation" in new_data.keys() and trans.get_user(): + dataset_collection_instance.add_item_annotation( + trans.sa_session, trans.get_user(), dataset_collection_instance, new_data["annotation"] + ) + changed["annotation"] = new_data["annotation"] # the api promises a list of changed fields, but tags are not marked as changed to avoid the # flush, so we must handle changed tag responses manually new_tags = None - if 'tags' in new_data.keys() and trans.get_user(): + if "tags" in new_data.keys() and trans.get_user(): # set_tags_from_list will flush on its own, no need to add to 'changed' here and incur a second flush. - new_tags = self.tag_handler.set_tags_from_list(trans.get_user(), dataset_collection_instance, new_data['tags']) + new_tags = self.tag_handler.set_tags_from_list( + trans.get_user(), dataset_collection_instance, new_data["tags"] + ) if changed.keys(): trans.sa_session.flush() # set client tag field response after the flush if new_tags is not None: - changed['tags'] = dataset_collection_instance.make_tag_string_list() + changed["tags"] = dataset_collection_instance.make_tag_string_list() return changed @@ -385,12 +485,12 @@ class DatasetCollectionManager: for key, val in payload.items(): if val is None: continue - if key in ('name'): + if key in ("name"): val = validation.validate_and_sanitize_basestring(key, val) validated_payload[key] = val - if key in ('deleted', 'visible'): + if key in ("deleted", "visible"): validated_payload[key] = validation.validate_boolean(key, val) - elif key == 'tags': + elif key == "tags": validated_payload[key] = validation.validate_and_sanitize_basestring_list(key, val) return validated_payload @@ -406,7 +506,9 @@ class DatasetCollectionManager: context.flush() return dataset_collection_instance - def __recursively_create_collections_for_identifiers(self, trans, element_identifiers, hide_source_items, copy_elements, history=None): + def __recursively_create_collections_for_identifiers( + self, trans, element_identifiers, hide_source_items, copy_elements, history=None + ): for element_identifier in element_identifiers: try: if element_identifier.get("src") != "new_collection": @@ -430,7 +532,9 @@ class DatasetCollectionManager: return element_identifiers - def __recursively_create_collections_for_elements(self, trans, elements, hide_source_items, copy_elements, history=None): + def __recursively_create_collections_for_elements( + self, trans, elements, hide_source_items, copy_elements, history=None + ): if elements is self.ELEMENTS_UNINITIALIZED: return @@ -460,11 +564,13 @@ class DatasetCollectionManager: def __load_elements(self, trans, element_identifiers, hide_source_items=False, copy_elements=False, history=None): elements = {} for element_identifier in element_identifiers: - elements[element_identifier["name"]] = self.__load_element(trans, - element_identifier=element_identifier, - hide_source_items=hide_source_items, - copy_elements=copy_elements, - history=history) + elements[element_identifier["name"]] = self.__load_element( + trans, + element_identifier=element_identifier, + hide_source_items=hide_source_items, + copy_elements=copy_elements, + history=history, + ) return elements def __load_element(self, trans, element_identifier, hide_source_items, copy_elements, history=None): @@ -485,34 +591,38 @@ class DatasetCollectionManager: # dataset_identifier is dict {src=hda|ldda|hdca|new_collection, id=} try: - src_type = element_identifier.get('src', 'hda') + src_type = element_identifier.get("src", "hda") except AttributeError: raise MessageException(f"Dataset collection element definition ({element_identifier}) not dictionary-like.") - encoded_id = element_identifier.get('id') + encoded_id = element_identifier.get("id") if not src_type or not encoded_id: message_template = "Problem decoding element identifier %s - must contain a 'src' and a 'id'." message = message_template % element_identifier raise RequestParameterInvalidException(message) - tags = element_identifier.pop('tags', None) - tag_str = '' + tags = element_identifier.pop("tags", None) + tag_str = "" if tags: tag_str = ",".join(str(_) for _ in tags) - if src_type == 'hda': + if src_type == "hda": decoded_id = int(trans.app.security.decode_id(encoded_id)) hda = self.hda_manager.get_accessible(decoded_id, trans.user) if copy_elements: element = self.hda_manager.copy(hda, history=history or trans.history, hide_copy=True, flush=False) else: element = hda - if hide_source_items and self.hda_manager.get_owned(hda.id, user=trans.user, current_history=history or trans.history): + if hide_source_items and self.hda_manager.get_owned( + hda.id, user=trans.user, current_history=history or trans.history + ): hda.visible = False self.tag_handler.apply_item_tags(user=trans.user, item=element, tags_str=tag_str, flush=False) - elif src_type == 'ldda': + elif src_type == "ldda": element = self.ldda_manager.get(trans, encoded_id, check_accessible=True) - element = element.to_history_dataset_association(history or trans.history, add_to_history=True, visible=not hide_source_items) + element = element.to_history_dataset_association( + history or trans.history, add_to_history=True, visible=not hide_source_items + ) self.tag_handler.apply_item_tags(user=trans.user, item=element, tags_str=tag_str, flush=False) - elif src_type == 'hdca': + elif src_type == "hdca": # TODO: Option to copy? Force copy? Copy or allow if not owned? element = self.__get_history_collection_instance(trans, encoded_id).collection # TODO: ldca. @@ -528,8 +638,7 @@ class DatasetCollectionManager: return MatchingCollections.for_collections(collections_to_match, self.collection_type_descriptions) def get_dataset_collection_instance(self, trans, instance_type, id, **kwds): - """ - """ + """ """ if instance_type == "history": return self.__get_history_collection_instance(trans, id, **kwds) elif instance_type == "library": @@ -550,7 +659,9 @@ class DatasetCollectionManager: collection_type = rule_set.collection_type collection_type_description = self.collection_type_descriptions.for_collection_type(collection_type) - elements = self._build_elements_from_rule_data(collection_type_description, rule_set, data, sources, handle_dataset) + elements = self._build_elements_from_rule_data( + collection_type_description, rule_set, data, sources, handle_dataset + ) return elements def _build_elements_from_rule_data(self, collection_type_description, rule_set, data, sources, handle_dataset): @@ -573,7 +684,9 @@ class DatasetCollectionManager: elif identifier.lower() in ["r", "2", "r2", "reverse"]: identifier = "reverse" else: - raise Exception("Unknown indicator of paired status encountered - only values of F, R, 1, 2, R1, R2, forward, or reverse are allowed.") + raise Exception( + "Unknown indicator of paired status encountered - only values of F, R, 1, 2, R1, R2, forward, or reverse are allowed." + ) tags = [] if "group_tags" in mapping_as_dict: @@ -620,7 +733,11 @@ class DatasetCollectionManager: identifiers = parent_identifiers + [element.element_identifier] if not element.is_collection: data.append([]) - source = {"identifiers": identifiers, "dataset": element_object, "tags": element_object.make_tag_string_list()} + source = { + "identifiers": identifiers, + "dataset": element_object, + "tags": element_object.make_tag_string_list(), + } sources.append(source) else: child_collection_type_description = collection_type_description.child_collection_type_description() @@ -634,26 +751,38 @@ class DatasetCollectionManager: def __get_history_collection_instance(self, trans, id, check_ownership=False, check_accessible=True): instance_id = int(trans.app.security.decode_id(id)) - collection_instance = trans.sa_session.query(trans.app.model.HistoryDatasetCollectionAssociation).get(instance_id) + collection_instance = trans.sa_session.query(trans.app.model.HistoryDatasetCollectionAssociation).get( + instance_id + ) if not collection_instance: raise RequestParameterInvalidException(f"History dataset collection association {id} not found") - history = getattr(trans, 'history', collection_instance.history) + history = getattr(trans, "history", collection_instance.history) if check_ownership: self.history_manager.error_unless_owner(collection_instance.history, trans.user, current_history=history) if check_accessible: - self.history_manager.error_unless_accessible(collection_instance.history, trans.user, current_history=history) + self.history_manager.error_unless_accessible( + collection_instance.history, trans.user, current_history=history + ) return collection_instance def __get_library_collection_instance(self, trans, id, check_ownership=False, check_accessible=True): if check_ownership: - raise NotImplementedError("Functionality (getting library dataset collection with ownership check) unimplemented.") + raise NotImplementedError( + "Functionality (getting library dataset collection with ownership check) unimplemented." + ) instance_id = int(trans.security.decode_id(id)) - collection_instance = trans.sa_session.query(trans.app.model.LibraryDatasetCollectionAssociation).get(instance_id) + collection_instance = trans.sa_session.query(trans.app.model.LibraryDatasetCollectionAssociation).get( + instance_id + ) if not collection_instance: raise RequestParameterInvalidException(f"Library dataset collection association {id} not found") if check_accessible: - if not trans.app.security_agent.can_access_library_item(trans.get_current_user_roles(), collection_instance, trans.user): - raise ItemAccessibilityException("LibraryDatasetCollectionAssociation is not accessible to the current user", type='error') + if not trans.app.security_agent.can_access_library_item( + trans.get_current_user_roles(), collection_instance, trans.user + ): + raise ItemAccessibilityException( + "LibraryDatasetCollectionAssociation is not accessible to the current user", type="error" + ) return collection_instance def get_collection_contents(self, trans, parent_id, limit=None, offset=None): @@ -667,7 +796,7 @@ class DatasetCollectionManager: DCE = model.DatasetCollectionElement qry = Query(DCE).filter(DCE.dataset_collection_id == parent_id) qry = qry.order_by(DCE.element_index) - qry = qry.options(joinedload('child_collection'), joinedload('hda')) + qry = qry.options(joinedload("child_collection"), joinedload("hda")) if limit is not None: qry = qry.limit(int(limit)) if offset is not None: diff --git a/lib/galaxy/managers/collections_util.py b/lib/galaxy/managers/collections_util.py index 12b04706343..dcf5c68c4f9 100644 --- a/lib/galaxy/managers/collections_util.py +++ b/lib/galaxy/managers/collections_util.py @@ -1,13 +1,18 @@ import logging import math -from galaxy import exceptions, model +from galaxy import ( + exceptions, + model, +) from galaxy.util import string_as_bool log = logging.getLogger(__name__) ERROR_MESSAGE_UNKNOWN_SRC = "Unknown dataset source (src) %s." -ERROR_MESSAGE_NO_NESTED_IDENTIFIERS = "Dataset source new_collection requires nested element_identifiers for new collection." +ERROR_MESSAGE_NO_NESTED_IDENTIFIERS = ( + "Dataset source new_collection requires nested element_identifiers for new collection." +) ERROR_MESSAGE_NO_NAME = "Cannot load invalid dataset identifier - missing name - %s" ERROR_MESSAGE_NO_COLLECTION_TYPE = "No collection_type define for nested collection %s." ERROR_MESSAGE_INVALID_PARAMETER_FOUND = "Found invalid parameter %s in element identifier description %s." @@ -29,13 +34,13 @@ def api_payload_to_create_params(payload): element_identifiers=payload.get("element_identifiers"), name=payload.get("name", None), hide_source_items=string_as_bool(payload.get("hide_source_items", False)), - copy_elements=string_as_bool(payload.get("copy_elements", False)) + copy_elements=string_as_bool(payload.get("copy_elements", False)), ) return params def validate_input_element_identifiers(element_identifiers): - """ Scan through the list of element identifiers supplied by the API consumer + """Scan through the list of element identifiers supplied by the API consumer and verify the structure is valid. """ log.debug("Validating %d element identifiers for collection creation." % len(element_identifiers)) @@ -78,7 +83,9 @@ def get_collection(collection, name=""): hdas = [] if collection.has_subcollections: for element in collection.elements: - subnames, subhdas = get_collection_elements(element.child_collection, name=f"{name}/{element.element_identifier}") + subnames, subhdas = get_collection_elements( + element.child_collection, name=f"{name}/{element.element_identifier}" + ) names.extend(subnames) hdas.extend(subhdas) else: @@ -103,34 +110,40 @@ def get_collection_elements(collection, name=""): return names, hdas -def dictify_dataset_collection_instance(dataset_collection_instance, parent, security, url_builder, view="element", fuzzy_count=None): +def dictify_dataset_collection_instance( + dataset_collection_instance, parent, security, url_builder, view="element", fuzzy_count=None +): hdca_view = "element" if view in ["element", "element-reference"] else "collection" dict_value = dataset_collection_instance.to_dict(view=hdca_view) encoded_id = security.encode_id(dataset_collection_instance.id) if isinstance(parent, model.History): encoded_history_id = security.encode_id(parent.id) - dict_value['url'] = url_builder('history_content_typed', history_id=encoded_history_id, id=encoded_id, type="dataset_collection") + dict_value["url"] = url_builder( + "history_content_typed", history_id=encoded_history_id, id=encoded_id, type="dataset_collection" + ) elif isinstance(parent, model.LibraryFolder): encoded_library_id = security.encode_id(parent.library_root.id) encoded_folder_id = security.encode_id(parent.id) # TODO: Work in progress - this end-point is not right yet... - dict_value['url'] = url_builder('library_content', library_id=encoded_library_id, id=encoded_id, folder_id=encoded_folder_id) + dict_value["url"] = url_builder( + "library_content", library_id=encoded_library_id, id=encoded_id, folder_id=encoded_folder_id + ) - dict_value['contents_url'] = url_builder( - 'contents_dataset_collection', + dict_value["contents_url"] = url_builder( + "contents_dataset_collection", hdca_id=encoded_id, - parent_id=security.encode_id(dataset_collection_instance.collection_id) + parent_id=security.encode_id(dataset_collection_instance.collection_id), ) if view in ["element", "element-reference"]: collection = dataset_collection_instance.collection rank_fuzzy_counts = gen_rank_fuzzy_counts(collection.collection_type, fuzzy_count) elements, rest_fuzzy_counts = get_fuzzy_count_elements(collection, rank_fuzzy_counts) if view == "element": - dict_value['populated'] = collection.populated + dict_value["populated"] = collection.populated element_func = dictify_element else: element_func = dictify_element_reference - dict_value['elements'] = [element_func(_, rank_fuzzy_counts=rest_fuzzy_counts) for _ in elements] + dict_value["elements"] = [element_func(_, rank_fuzzy_counts=rest_fuzzy_counts) for _ in elements] security.encode_all_ids(dict_value, recursive=True) # TODO: Use Kyle's recursive formulation of this. return dict_value @@ -156,11 +169,14 @@ def dictify_element_reference(element, rank_fuzzy_counts=None, recursive=True, s if recursive: child_collection = element.child_collection elements, rest_fuzzy_counts = get_fuzzy_count_elements(child_collection, rank_fuzzy_counts) - object_details["elements"] = [dictify_element_reference(_, rank_fuzzy_counts=rest_fuzzy_counts, recursive=recursive) for _ in elements] + object_details["elements"] = [ + dictify_element_reference(_, rank_fuzzy_counts=rest_fuzzy_counts, recursive=recursive) + for _ in elements + ] object_details["element_count"] = child_collection.element_count else: object_details["state"] = element_object.state - object_details["hda_ldda"] = 'hda' + object_details["hda_ldda"] = "hda" object_details["history_id"] = element_object.history_id dictified["object"] = object_details @@ -265,15 +281,19 @@ def gen_rank_fuzzy_counts(collection_type, fuzzy_count=None): list_count = len(rank_collection_types) - paired_count paired_fuzzy_count_mult = 1 if paired_count == 0 else 2 << (paired_count - 1) list_fuzzy_count_mult = math.floor((fuzzy_count * 1.0) / paired_fuzzy_count_mult) - list_rank_fuzzy_count = int(math.floor(math.pow(list_fuzzy_count_mult, 1.0 / list_count)) + 1) if list_count > 0 else 1.0 + list_rank_fuzzy_count = ( + int(math.floor(math.pow(list_fuzzy_count_mult, 1.0 / list_count)) + 1) if list_count > 0 else 1.0 + ) pair_rank_fuzzy_count = 2 if list_rank_fuzzy_count > fuzzy_count: list_rank_fuzzy_count = fuzzy_count if pair_rank_fuzzy_count > fuzzy_count: pair_rank_fuzzy_count = fuzzy_count - rank_fuzzy_counts = [pair_rank_fuzzy_count if rt == "paired" else list_rank_fuzzy_count for rt in rank_collection_types] + rank_fuzzy_counts = [ + pair_rank_fuzzy_count if rt == "paired" else list_rank_fuzzy_count for rt in rank_collection_types + ] return rank_fuzzy_counts -__all__ = ('api_payload_to_create_params', 'dictify_dataset_collection_instance') +__all__ = ("api_payload_to_create_params", "dictify_dataset_collection_instance") diff --git a/lib/galaxy/managers/configuration.py b/lib/galaxy/managers/configuration.py index 2201abe6e1f..55b5c2115f1 100644 --- a/lib/galaxy/managers/configuration.py +++ b/lib/galaxy/managers/configuration.py @@ -22,9 +22,10 @@ from galaxy.managers.markdown_util import weasyprint_available from galaxy.schema import SerializationParams from galaxy.schema.fields import EncodedDatabaseIdField from galaxy.web.framework.base import server_starttime + log = logging.getLogger(__name__) -VERSION_JSON_FILE = 'version.json' +VERSION_JSON_FILE = "version.json" class ConfigurationManager: @@ -34,9 +35,7 @@ class ConfigurationManager: self._app = app def get_configuration( - self, - trans: ProvidesUserContext, - serialization_params: SerializationParams + self, trans: ProvidesUserContext, serialization_params: SerializationParams ) -> Dict[str, Any]: is_admin = trans.user_is_admin host = getattr(trans, "host", None) @@ -56,9 +55,9 @@ class ConfigurationManager: with open(json_file) as f: extra_info = json.load(f) except OSError: - log.info('Galaxy extra version JSON file %s not loaded.', json_file) + log.info("Galaxy extra version JSON file %s not loaded.", json_file) else: - version_info['extra'] = extra_info + version_info["extra"] = extra_info return version_info def decode_id( @@ -66,7 +65,7 @@ class ConfigurationManager: encoded_id: EncodedDatabaseIdField, ) -> Dict[str, int]: # Handle the special case for library folders - if ((len(encoded_id) % 16 == 1) and encoded_id.startswith('F')): + if (len(encoded_id) % 16 == 1) and encoded_id.startswith("F"): encoded_id = cast(EncodedDatabaseIdField, encoded_id[1:]) decoded_id = self._app.security.decode_id(encoded_id) return {"decoded_id": decoded_id} @@ -79,7 +78,7 @@ class ConfigurationManager: except AttributeError: pass else: - entry = {'id': id, 'lineage': lineage_dict} + entry = {"id": id, "lineage": lineage_dict} rval.append(entry) return rval @@ -89,14 +88,15 @@ class ConfigurationManager: # server. A dedicated endpoint should probably be added to do that instead. def tool_conf_to_dict(conf): return dict( - config_filename=conf['config_filename'], - tool_path=conf['tool_path'], + config_filename=conf["config_filename"], + tool_path=conf["tool_path"], ) + confs = self._app.toolbox.dynamic_confs(include_migrated_tool_conf=True) return list(map(tool_conf_to_dict, confs)) def reload_toolbox(self): - self._app.queue_worker.send_control_task('reload_toolbox') + self._app.queue_worker.send_control_task("reload_toolbox") # TODO: this is a bit of an odd duck. It uses the serializer structure from managers @@ -108,14 +108,13 @@ class ConfigSerializer(base.ModelSerializer): def __init__(self, app): super().__init__(app) - self.default_view = 'all' - self.add_view('all', list(self.serializers.keys())) + self.default_view = "all" + self.add_view("all", list(self.serializers.keys())) def default_serializer(self, config, key): return getattr(config, key, None) def add_serializers(self): - def _defaults_to(default) -> base.Serializer: return lambda item, key, **context: getattr(item, key, default) @@ -129,87 +128,87 @@ class ConfigSerializer(base.ModelSerializer): self.serializers: Dict[str, base.Serializer] = { # TODO: this is available from user data, remove - 'is_admin_user': lambda *a, **c: False, - 'brand': _use_config, - 'display_galaxy_brand': _use_config, - 'logo_url': _use_config, - 'logo_src': _use_config, - 'logo_src_secondary': _use_config, - 'terms_url': _use_config, - 'myexperiment_target_url': _use_config, - 'wiki_url': _use_config, - 'search_url': _use_config, - 'mailing_lists': _defaults_to(self.app.config.mailing_lists_url), - 'screencasts_url': _use_config, - 'citation_url': _use_config, - 'support_url': _use_config, - 'quota_url': _use_config, - 'helpsite_url': _use_config, - 'lims_doc_url': _defaults_to("https://usegalaxy.org/u/rkchak/p/sts"), - 'default_locale': _use_config, - 'enable_tool_recommendations': _use_config, - 'enable_account_interface': _use_config, - 'tool_recommendation_model_path': _use_config, - 'admin_tool_recommendations_path': _use_config, - 'overwrite_model_recommendations': _use_config, - 'topk_recommendations': _use_config, - 'allow_user_impersonation': _use_config, - 'allow_user_creation': _defaults_to(False), # schema default is True - 'use_remote_user': _defaults_to(None), # schema default is False; or config.single_user - 'single_user': _config_is_truthy, - 'enable_oidc': _use_config, - 'oidc': _use_config, - 'enable_quotas': _use_config, - 'remote_user_logout_href': _use_config, - 'datatypes_disable_auto': _use_config, - 'allow_user_dataset_purge': _defaults_to(False), # schema default is True - 'ga_code': _use_config, - 'plausible_server': _use_config, - 'plausible_domain': _use_config, - 'markdown_to_pdf_available': lambda item, key, **context: weasyprint_available(), - 'matomo_server': _use_config, - 'matomo_site_id': _use_config, - 'enable_unique_workflow_defaults': _use_config, - 'enable_beta_markdown_export': _use_config, - 'simplified_workflow_run_ui': _use_config, - 'simplified_workflow_run_ui_target_history': _use_config, - 'simplified_workflow_run_ui_job_cache': _use_config, - 'simplified_workflow_run_ui': _use_config, - 'has_user_tool_filters': _defaults_to(False), + "is_admin_user": lambda *a, **c: False, + "brand": _use_config, + "display_galaxy_brand": _use_config, + "logo_url": _use_config, + "logo_src": _use_config, + "logo_src_secondary": _use_config, + "terms_url": _use_config, + "myexperiment_target_url": _use_config, + "wiki_url": _use_config, + "search_url": _use_config, + "mailing_lists": _defaults_to(self.app.config.mailing_lists_url), + "screencasts_url": _use_config, + "citation_url": _use_config, + "support_url": _use_config, + "quota_url": _use_config, + "helpsite_url": _use_config, + "lims_doc_url": _defaults_to("https://usegalaxy.org/u/rkchak/p/sts"), + "default_locale": _use_config, + "enable_tool_recommendations": _use_config, + "enable_account_interface": _use_config, + "tool_recommendation_model_path": _use_config, + "admin_tool_recommendations_path": _use_config, + "overwrite_model_recommendations": _use_config, + "topk_recommendations": _use_config, + "allow_user_impersonation": _use_config, + "allow_user_creation": _defaults_to(False), # schema default is True + "use_remote_user": _defaults_to(None), # schema default is False; or config.single_user + "single_user": _config_is_truthy, + "enable_oidc": _use_config, + "oidc": _use_config, + "enable_quotas": _use_config, + "remote_user_logout_href": _use_config, + "datatypes_disable_auto": _use_config, + "allow_user_dataset_purge": _defaults_to(False), # schema default is True + "ga_code": _use_config, + "plausible_server": _use_config, + "plausible_domain": _use_config, + "markdown_to_pdf_available": lambda item, key, **context: weasyprint_available(), + "matomo_server": _use_config, + "matomo_site_id": _use_config, + "enable_unique_workflow_defaults": _use_config, + "enable_beta_markdown_export": _use_config, + "simplified_workflow_run_ui": _use_config, + "simplified_workflow_run_ui_target_history": _use_config, + "simplified_workflow_run_ui_job_cache": _use_config, + "simplified_workflow_run_ui": _use_config, + "has_user_tool_filters": _defaults_to(False), # TODO: is there no 'correct' way to get an api url? controller='api', action='tools' is a hack # at any rate: the following works with path_prefix but is still brittle # TODO: change this to (more generic) upload_path and incorporate config.nginx_upload_path into building it - 'nginx_upload_path': lambda item, key, **context: getattr(item, key, False), - 'chunk_upload_size': _use_config, - 'ftp_upload_site': _use_config, - 'version_major': _defaults_to(None), - 'version_minor': _defaults_to(None), - 'require_login': _use_config, - 'inactivity_box_content': _use_config, - 'visualizations_visible': _use_config, - 'interactivetools_enable': _use_config, - 'aws_estimate': _use_config, - 'message_box_content': _use_config, - 'message_box_visible': _use_config, - 'message_box_class': _use_config, - 'server_startttime': lambda item, key, **context: server_starttime, - 'mailing_join_addr': _defaults_to('galaxy-announce-join@bx.psu.edu'), # should this be the schema default? - 'server_mail_configured': lambda item, key, **context: bool(item.smtp_server), - 'registration_warning_message': _use_config, - 'welcome_url': _use_config, - 'show_welcome_with_login': _defaults_to(True), # schema default is False - 'cookie_domain': _use_config, - 'python': _defaults_to((sys.version_info.major, sys.version_info.minor)), - 'select_type_workflow_threshold': _use_config, - 'file_sources_configured': lambda item, key, **context: self.app.file_sources.custom_sources_configured, - 'panel_views': lambda item, key, **context: self.app.toolbox.panel_view_dicts(), - 'default_panel_view': _use_config, - 'upload_from_form_button': _use_config, - 'release_doc_base_url': _use_config, - 'expose_user_email': _use_config, - 'enable_tool_source_display': _use_config, - 'user_library_import_dir_available': lambda item, key, **context: bool(item.get('user_library_import_dir')), - 'welcome_directory': _use_config, + "nginx_upload_path": lambda item, key, **context: getattr(item, key, False), + "chunk_upload_size": _use_config, + "ftp_upload_site": _use_config, + "version_major": _defaults_to(None), + "version_minor": _defaults_to(None), + "require_login": _use_config, + "inactivity_box_content": _use_config, + "visualizations_visible": _use_config, + "interactivetools_enable": _use_config, + "aws_estimate": _use_config, + "message_box_content": _use_config, + "message_box_visible": _use_config, + "message_box_class": _use_config, + "server_startttime": lambda item, key, **context: server_starttime, + "mailing_join_addr": _defaults_to("galaxy-announce-join@bx.psu.edu"), # should this be the schema default? + "server_mail_configured": lambda item, key, **context: bool(item.smtp_server), + "registration_warning_message": _use_config, + "welcome_url": _use_config, + "show_welcome_with_login": _defaults_to(True), # schema default is False + "cookie_domain": _use_config, + "python": _defaults_to((sys.version_info.major, sys.version_info.minor)), + "select_type_workflow_threshold": _use_config, + "file_sources_configured": lambda item, key, **context: self.app.file_sources.custom_sources_configured, + "panel_views": lambda item, key, **context: self.app.toolbox.panel_view_dicts(), + "default_panel_view": _use_config, + "upload_from_form_button": _use_config, + "release_doc_base_url": _use_config, + "expose_user_email": _use_config, + "enable_tool_source_display": _use_config, + "user_library_import_dir_available": lambda item, key, **context: bool(item.get("user_library_import_dir")), + "welcome_directory": _use_config, } @@ -222,12 +221,13 @@ class AdminConfigSerializer(ConfigSerializer): def _defaults_to(default): return lambda config, key, **context: getattr(config, key, default) - self.serializers.update({ - # TODO: this is available from user serialization: remove - 'is_admin_user': lambda *a, **context: True, - - 'library_import_dir': _defaults_to(None), - 'user_library_import_dir': _defaults_to(None), - 'allow_library_path_paste': _defaults_to(False), - 'allow_user_deletion': _defaults_to(False), - }) + self.serializers.update( + { + # TODO: this is available from user serialization: remove + "is_admin_user": lambda *a, **context: True, + "library_import_dir": _defaults_to(None), + "user_library_import_dir": _defaults_to(None), + "allow_library_path_paste": _defaults_to(False), + "allow_user_deletion": _defaults_to(False), + } + ) diff --git a/lib/galaxy/managers/context.py b/lib/galaxy/managers/context.py index d4c850e70c8..aee813e7ceb 100644 --- a/lib/galaxy/managers/context.py +++ b/lib/galaxy/managers/context.py @@ -37,7 +37,11 @@ A method that requires a user but not a history should declare its import abc import string from json import dumps -from typing import Callable, List, Optional +from typing import ( + Callable, + List, + Optional, +) from galaxy.exceptions import UserActivationRequiredException from galaxy.model import ( @@ -55,7 +59,7 @@ from galaxy.util import bunch class ProvidesAppContext: - """ For transaction-like objects to provide Galaxy convenience layer for + """For transaction-like objects to provide Galaxy convenience layer for database and event handling. Mixed in class must provide `app` property. @@ -63,8 +67,7 @@ class ProvidesAppContext: @abc.abstractproperty def app(self) -> MinimalManagerApp: - """Provide access to the Galaxy ``app`` object. - """ + """Provide access to the Galaxy ``app`` object.""" @abc.abstractproperty def url_builder(self) -> Optional[Callable[..., str]]: @@ -149,7 +152,7 @@ class ProvidesAppContext: context = app.model.context context.expunge_all() # This is a bit hacky, should refctor this. Maybe refactor to app -> expunge_all() - if hasattr(app, 'install_model'): + if hasattr(app, "install_model"): install_model = app.install_model if install_model != app.model: install_model.context.expunge_all() @@ -187,7 +190,7 @@ class ProvidesAppContext: class ProvidesUserContext(ProvidesAppContext): - """ For transaction-like objects to provide Galaxy convenience layer for + """For transaction-like objects to provide Galaxy convenience layer for reasoning about users. Mixed in class must provide `user` and `app` @@ -247,15 +250,17 @@ class ProvidesUserContext(ProvidesAppContext): identifier_attr = self.app.config.ftp_upload_dir_identifier identifier_value = getattr(self.user, identifier_attr) template = self.app.config.ftp_upload_dir_template - path = string.Template(template).safe_substitute(dict( - ftp_upload_dir=base_dir, - ftp_upload_dir_identifier=identifier_value, - )) + path = string.Template(template).safe_substitute( + dict( + ftp_upload_dir=base_dir, + ftp_upload_dir_identifier=identifier_value, + ) + ) return path class ProvidesHistoryContext(ProvidesUserContext): - """ For transaction-like objects to provide Galaxy convenience layer for + """For transaction-like objects to provide Galaxy convenience layer for reasoning about histories. Mixed in class must provide `user`, `history`, and `app` @@ -270,8 +275,7 @@ class ProvidesHistoryContext(ProvidesUserContext): """ def db_dataset_for(self, dbkey) -> Optional[HistoryDatasetAssociation]: - """Optionally return the db_file dataset associated/needed by `dataset`. - """ + """Optionally return the db_file dataset associated/needed by `dataset`.""" # If no history, return None. if self.history is None: return None @@ -282,14 +286,12 @@ class ProvidesHistoryContext(ProvidesUserContext): return None non_ready_or_ok = set(Dataset.non_ready_states) non_ready_or_ok.add(HistoryDatasetAssociation.states.OK) - datasets = self.sa_session.query( - HistoryDatasetAssociation - ).filter_by( - deleted=False, - history_id=self.history.id, - extension="len" - ).filter( - HistoryDatasetAssociation.table.c._state.in_(non_ready_or_ok), + datasets = ( + self.sa_session.query(HistoryDatasetAssociation) + .filter_by(deleted=False, history_id=self.history.id, extension="len") + .filter( + HistoryDatasetAssociation.table.c._state.in_(non_ready_or_ok), + ) ) valid_ds = None for ds in datasets: diff --git a/lib/galaxy/managers/datasets.py b/lib/galaxy/managers/datasets.py index cf3bd0128bf..67ff9afa6ff 100644 --- a/lib/galaxy/managers/datasets.py +++ b/lib/galaxy/managers/datasets.py @@ -4,11 +4,16 @@ Manager and Serializer for Datasets. import glob import logging import os -from typing import Dict, List, Type, TypeVar +from typing import ( + Dict, + List, + Type, + TypeVar, +) from galaxy import ( exceptions, - model + model, ) from galaxy.datatypes import sniff from galaxy.managers import ( @@ -16,22 +21,23 @@ from galaxy.managers import ( deletable, rbac_secured, secured, - users + users, ) from galaxy.structured_app import MinimalManagerApp from galaxy.util.checkers import check_binary log = logging.getLogger(__name__) -T = TypeVar('T') +T = TypeVar("T") class DatasetManager(base.ModelManager, secured.AccessibleManagerMixin, deletable.PurgableManagerMixin): """ Manipulate datasets: the components contained in DatasetAssociations/DatasetInstances/HDAs/LDDAs """ + model_class = model.Dataset - foreign_key_name = 'dataset' + foreign_key_name = "dataset" app: MinimalManagerApp # TODO:?? get + error_if_uploading is common pattern, should upload check be worked into access/owed? @@ -47,7 +53,7 @@ class DatasetManager(base.ModelManager, secured.AccessibleManagerMixin, deletabl Create and return a new Dataset object. """ # default to NEW state on new datasets - kwargs.update(dict(state=(kwargs.get('state', model.Dataset.states.NEW)))) + kwargs.update(dict(state=(kwargs.get("state", model.Dataset.states.NEW)))) dataset = model.Dataset(**kwargs) self.session().add(dataset) @@ -58,7 +64,7 @@ class DatasetManager(base.ModelManager, secured.AccessibleManagerMixin, deletabl return dataset def copy(self, dataset, **kwargs): - raise exceptions.NotImplemented('Datasets cannot be copied') + raise exceptions.NotImplemented("Datasets cannot be copied") def purge(self, dataset, flush=True): """ @@ -80,7 +86,7 @@ class DatasetManager(base.ModelManager, secured.AccessibleManagerMixin, deletabl # TODO: how to allow admin bypass? def error_unless_dataset_purge_allowed(self, msg=None): if not self.app.config.allow_user_dataset_purge: - msg = msg or 'This instance does not allow user dataset purging' + msg = msg or "This instance does not allow user dataset purging" raise exceptions.ConfigDoesNotAllowException(msg) # .... accessibility @@ -110,15 +116,15 @@ class DatasetManager(base.ModelManager, secured.AccessibleManagerMixin, deletabl # TODO: SecurityAgentDatasetRBACPermissions( object ): -class DatasetRBACPermissions: +class DatasetRBACPermissions: def __init__(self, app): self.app = app self.access = rbac_secured.AccessDatasetRBACPermission(app) self.manage = rbac_secured.ManageDatasetRBACPermission(app) # TODO: temporary facade over security_agent - def available_roles(self, trans, dataset, controller='root'): + def available_roles(self, trans, dataset, controller="root"): return self.app.security_agent.get_legitimate_roles(trans, dataset, controller) def get(self, dataset, flush=True): @@ -152,38 +158,39 @@ class DatasetSerializer(base.ModelSerializer[DatasetManager], deletable.Purgable # needed for admin test self.user_manager = user_manager - self.default_view = 'summary' - self.add_view('summary', [ - 'id', - 'create_time', - 'update_time', - 'state', - 'deleted', - 'purged', - 'purgable', - # 'object_store_id', - # 'external_filename', - # 'extra_files_path', - 'file_size', - 'total_size', - 'uuid', - ]) + self.default_view = "summary" + self.add_view( + "summary", + [ + "id", + "create_time", + "update_time", + "state", + "deleted", + "purged", + "purgable", + # 'object_store_id', + # 'external_filename', + # 'extra_files_path', + "file_size", + "total_size", + "uuid", + ], + ) # could do visualizations and/or display_apps def add_serializers(self): super().add_serializers() deletable.PurgableSerializerMixin.add_serializers(self) serializers: Dict[str, base.Serializer] = { - 'create_time': self.serialize_date, - 'update_time': self.serialize_date, - - 'uuid': lambda item, key, **context: str(item.uuid) if item.uuid else None, - 'file_name': self.serialize_file_name, - 'extra_files_path': self.serialize_extra_files_path, - 'permissions': self.serialize_permissions, - - 'total_size': lambda item, key, **context: int(item.get_total_size()), - 'file_size': lambda item, key, **context: int(item.get_size()) + "create_time": self.serialize_date, + "update_time": self.serialize_date, + "uuid": lambda item, key, **context: str(item.uuid) if item.uuid else None, + "file_name": self.serialize_file_name, + "extra_files_path": self.serialize_extra_files_path, + "permissions": self.serialize_permissions, + "total_size": lambda item, key, **context: int(item.get_total_size()), + "file_size": lambda item, key, **context: int(item.get_size()), } self.serializers.update(serializers) @@ -213,8 +220,7 @@ class DatasetSerializer(base.ModelSerializer[DatasetManager], deletable.Purgable self.skip() def serialize_permissions(self, item, key, user=None, **context): - """ - """ + """ """ dataset = item trans = context.get("trans") if not self.dataset_manager.permissions.manage.is_permitted(dataset, user, trans=trans): @@ -223,21 +229,20 @@ class DatasetSerializer(base.ModelSerializer[DatasetManager], deletable.Purgable management_permissions = self.dataset_manager.permissions.manage.by_dataset(dataset) access_permissions = self.dataset_manager.permissions.access.by_dataset(dataset) permissions = { - 'manage': [self.app.security.encode_id(perm.role.id) for perm in management_permissions], - 'access': [self.app.security.encode_id(perm.role.id) for perm in access_permissions], + "manage": [self.app.security.encode_id(perm.role.id) for perm in management_permissions], + "access": [self.app.security.encode_id(perm.role.id) for perm in access_permissions], } return permissions # ============================================================================= AKA DatasetInstanceManager -class DatasetAssociationManager(base.ModelManager, - secured.AccessibleManagerMixin, - deletable.PurgableManagerMixin): +class DatasetAssociationManager(base.ModelManager, secured.AccessibleManagerMixin, deletable.PurgableManagerMixin): """ DatasetAssociation/DatasetInstances are intended to be working proxies to a Dataset, associated with either a library or a user/history (HistoryDatasetAssociation). """ + # DA's were meant to be proxies - but were never fully implemented as them # Instead, a dataset association HAS a dataset but contains metadata specific to a library (lda) or user (hda) model_class: Type[model.DatasetInstance] @@ -278,7 +283,7 @@ class DatasetAssociationManager(base.ModelManager, return dataset_assoc def by_user(self, user): - raise exceptions.NotImplemented('Abstract Method') + raise exceptions.NotImplemented("Abstract Method") # .... associated job def creating_job(self, dataset_assoc): @@ -328,7 +333,7 @@ class DatasetAssociationManager(base.ModelManager, """Return a list of file paths for composite files, an empty list otherwise.""" if not self.is_composite(dataset_assoc): return [] - return glob.glob(os.path.join(dataset_assoc.dataset.extra_files_path, '*')) + return glob.glob(os.path.join(dataset_assoc.dataset.extra_files_path, "*")) def serialize_dataset_association_roles(self, trans, dataset_assoc): if hasattr(dataset_assoc, "library_dataset_dataset_association"): @@ -343,12 +348,22 @@ class DatasetAssociationManager(base.ModelManager, access_roles = set(dataset.get_access_roles(security_agent)) manage_roles = set(dataset.get_manage_permissions_roles(security_agent)) - access_dataset_role_list = [(access_role.name, trans.security.encode_id(access_role.id)) for access_role in access_roles] - manage_dataset_role_list = [(manage_role.name, trans.security.encode_id(manage_role.id)) for manage_role in manage_roles] + access_dataset_role_list = [ + (access_role.name, trans.security.encode_id(access_role.id)) for access_role in access_roles + ] + manage_dataset_role_list = [ + (manage_role.name, trans.security.encode_id(manage_role.id)) for manage_role in manage_roles + ] rval = dict(access_dataset_roles=access_dataset_role_list, manage_dataset_roles=manage_dataset_role_list) if library_dataset is not None: - modify_roles = set(security_agent.get_roles_for_action(library_dataset, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY)) - modify_item_role_list = [(modify_role.name, trans.security.encode_id(modify_role.id)) for modify_role in modify_roles] + modify_roles = set( + security_agent.get_roles_for_action( + library_dataset, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY + ) + ) + modify_item_role_list = [ + (modify_role.name, trans.security.encode_id(modify_role.id)) for modify_role in modify_roles + ] rval["modify_item_roles"] = modify_item_role_list return rval @@ -357,7 +372,9 @@ class DatasetAssociationManager(base.ModelManager, data = trans.sa_session.query(self.model_class).get(dataset_assoc.id) if data.datatype.is_datatype_change_allowed(): if not data.ok_to_edit_metadata(): - raise exceptions.ItemAccessibilityException('This dataset is currently being used as input or output. You cannot change datatype until the jobs have completed or you have canceled them.') + raise exceptions.ItemAccessibilityException( + "This dataset is currently being used as input or output. You cannot change datatype until the jobs have completed or you have canceled them." + ) else: path = data.dataset.file_name is_binary = check_binary(path) @@ -372,25 +389,32 @@ class DatasetAssociationManager(base.ModelManager, """Trigger a job that detects and sets metadata on a given dataset association (ldda or hda)""" data = trans.sa_session.query(self.model_class).get(dataset_assoc.id) if not data.ok_to_edit_metadata(): - raise exceptions.ItemAccessibilityException('This dataset is currently being used as input or output. You cannot edit metadata until the jobs have completed or you have canceled them.') + raise exceptions.ItemAccessibilityException( + "This dataset is currently being used as input or output. You cannot edit metadata until the jobs have completed or you have canceled them." + ) else: if overwrite: for name, spec in data.metadata.spec.items(): # We need to be careful about the attributes we are resetting - if name not in ['name', 'info', 'dbkey', 'base_name']: - if spec.get('default'): - setattr(data.metadata, name, spec.unwrap(spec.get('default'))) + if name not in ["name", "info", "dbkey", "base_name"]: + if spec.get("default"): + setattr(data.metadata, name, spec.unwrap(spec.get("default"))) job, *_ = self.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( - self.app.datatypes_registry.set_external_metadata_tool, trans, incoming={'input1': data, 'validate': validate}, - overwrite=overwrite) + self.app.datatypes_registry.set_external_metadata_tool, + trans, + incoming={"input1": data, "validate": validate}, + overwrite=overwrite, + ) self.app.job_manager.enqueue(job, tool=self.app.datatypes_registry.set_external_metadata_tool) def update_permissions(self, trans, dataset_assoc, **kwd): - action = kwd.get('action', 'set_permissions') - if action not in ['remove_restrictions', 'make_private', 'set_permissions']: - raise exceptions.RequestParameterInvalidException('The mandatory parameter "action" has an invalid value. ' - 'Allowed values are: "remove_restrictions", "make_private", "set_permissions"') + action = kwd.get("action", "set_permissions") + if action not in ["remove_restrictions", "make_private", "set_permissions"]: + raise exceptions.RequestParameterInvalidException( + 'The mandatory parameter "action" has an invalid value. ' + 'Allowed values are: "remove_restrictions", "make_private", "set_permissions"' + ) if hasattr(dataset_assoc, "library_dataset_dataset_association"): library_dataset = dataset_assoc dataset = library_dataset.library_dataset_dataset_association.dataset @@ -401,22 +425,26 @@ class DatasetAssociationManager(base.ModelManager, current_user_roles = trans.get_current_user_roles() can_manage = trans.app.security_agent.can_manage_dataset(current_user_roles, dataset) or trans.user_is_admin if not can_manage: - raise exceptions.InsufficientPermissionsException('You do not have proper permissions to manage permissions on this dataset.') + raise exceptions.InsufficientPermissionsException( + "You do not have proper permissions to manage permissions on this dataset." + ) - if action == 'remove_restrictions': + if action == "remove_restrictions": trans.app.security_agent.make_dataset_public(dataset) if not trans.app.security_agent.dataset_is_public(dataset): - raise exceptions.InternalServerError('An error occurred while making dataset public.') - elif action == 'make_private': + raise exceptions.InternalServerError("An error occurred while making dataset public.") + elif action == "make_private": if not trans.app.security_agent.dataset_is_private_to_user(trans, dataset): private_role = trans.app.security_agent.get_private_user_role(trans.user) - dp = trans.app.model.DatasetPermissions(trans.app.security_agent.permitted_actions.DATASET_ACCESS.action, dataset, private_role) + dp = trans.app.model.DatasetPermissions( + trans.app.security_agent.permitted_actions.DATASET_ACCESS.action, dataset, private_role + ) trans.sa_session.add(dp) trans.sa_session.flush() if not trans.app.security_agent.dataset_is_private_to_user(trans, dataset): # Check again and inform the user if dataset is not private. - raise exceptions.InternalServerError('An error occurred and the dataset is NOT private.') - elif action == 'set_permissions': + raise exceptions.InternalServerError("An error occurred and the dataset is NOT private.") + elif action == "set_permissions": def to_role_id(encoded_role_id): role_id = base.decode_id(self.app, encoded_role_id) @@ -429,12 +457,12 @@ class DatasetAssociationManager(base.ModelManager, else: return None - access_roles = parameters_roles_or_none('access') - manage_roles = parameters_roles_or_none('manage') - modify_roles = parameters_roles_or_none('modify') + access_roles = parameters_roles_or_none("access") + manage_roles = parameters_roles_or_none("manage") + modify_roles = parameters_roles_or_none("modify") role_ids_dict = { - 'DATASET_MANAGE_PERMISSIONS': manage_roles, - 'DATASET_ACCESS': access_roles, + "DATASET_MANAGE_PERMISSIONS": manage_roles, + "DATASET_ACCESS": access_roles, } if library_dataset is not None: role_ids_dict["LIBRARY_MODIFY"] = modify_roles @@ -445,9 +473,7 @@ class DatasetAssociationManager(base.ModelManager, raise exceptions.NotImplemented() -class _UnflattenedMetadataDatasetAssociationSerializer(base.ModelSerializer[T], - deletable.PurgableSerializerMixin): - +class _UnflattenedMetadataDatasetAssociationSerializer(base.ModelSerializer[T], deletable.PurgableSerializerMixin): def __init__(self, app): self.dataset_serializer = app[DatasetSerializer] super().__init__(app) @@ -457,55 +483,48 @@ class _UnflattenedMetadataDatasetAssociationSerializer(base.ModelSerializer[T], deletable.PurgableSerializerMixin.add_serializers(self) serializers: Dict[str, base.Serializer] = { - 'create_time': self.serialize_date, - 'update_time': self.serialize_date, - + "create_time": self.serialize_date, + "update_time": self.serialize_date, # underlying dataset - 'dataset': lambda item, key, **context: self.dataset_serializer.serialize_to_view(item.dataset, view='summary', **context), - 'dataset_id': self._proxy_to_dataset(proxy_key='id'), + "dataset": lambda item, key, **context: self.dataset_serializer.serialize_to_view( + item.dataset, view="summary", **context + ), + "dataset_id": self._proxy_to_dataset(proxy_key="id"), # TODO: why is this named uuid!? The da doesn't have a uuid - it's the underlying dataset's uuid! - 'uuid': self._proxy_to_dataset(proxy_key='uuid'), + "uuid": self._proxy_to_dataset(proxy_key="uuid"), # 'dataset_uuid': self._proxy_to_dataset( key='uuid' ), - 'file_name': self._proxy_to_dataset(serializer=self.dataset_serializer.serialize_file_name), - 'extra_files_path': self._proxy_to_dataset(serializer=self.dataset_serializer.serialize_extra_files_path), - 'permissions': self._proxy_to_dataset(serializer=self.dataset_serializer.serialize_permissions), + "file_name": self._proxy_to_dataset(serializer=self.dataset_serializer.serialize_file_name), + "extra_files_path": self._proxy_to_dataset(serializer=self.dataset_serializer.serialize_extra_files_path), + "permissions": self._proxy_to_dataset(serializer=self.dataset_serializer.serialize_permissions), # TODO: do the sizes proxy accurately/in the same way? - 'size': lambda item, key, **context: int(item.get_size()), - 'file_size': lambda item, key, **context: self.serializers['size'](item, key, **context), - 'nice_size': lambda item, key, **context: item.get_size(nice_size=True), - + "size": lambda item, key, **context: int(item.get_size()), + "file_size": lambda item, key, **context: self.serializers["size"](item, key, **context), + "nice_size": lambda item, key, **context: item.get_size(nice_size=True), # common to lddas and hdas - from mapping.py - 'copied_from_history_dataset_association_id': self.serialize_id, - 'copied_from_library_dataset_dataset_association_id': self.serialize_id, - 'info': lambda item, key, **context: item.info.strip() if isinstance(item.info, str) else item.info, - 'blurb': lambda item, key, **context: item.blurb, - 'peek': lambda item, key, **context: item.display_peek() if item.peek and item.peek != 'no peek' else None, - - 'meta_files': self.serialize_meta_files, - 'metadata': self.serialize_metadata, - - 'creating_job': self.serialize_creating_job, - 'rerunnable': self.serialize_rerunnable, - - 'parent_id': self.serialize_id, - 'designation': lambda item, key, **context: item.designation, - + "copied_from_history_dataset_association_id": self.serialize_id, + "copied_from_library_dataset_dataset_association_id": self.serialize_id, + "info": lambda item, key, **context: item.info.strip() if isinstance(item.info, str) else item.info, + "blurb": lambda item, key, **context: item.blurb, + "peek": lambda item, key, **context: item.display_peek() if item.peek and item.peek != "no peek" else None, + "meta_files": self.serialize_meta_files, + "metadata": self.serialize_metadata, + "creating_job": self.serialize_creating_job, + "rerunnable": self.serialize_rerunnable, + "parent_id": self.serialize_id, + "designation": lambda item, key, **context: item.designation, # 'extended_metadata': self.serialize_extended_metadata, # 'extended_metadata_id': self.serialize_id, - # remapped - 'genome_build': lambda item, key, **context: item.dbkey, - + "genome_build": lambda item, key, **context: item.dbkey, # derived (not mapped) attributes - 'data_type': lambda item, key, **context: f"{item.datatype.__class__.__module__}.{item.datatype.__class__.__name__}", - - 'converted': self.serialize_converted_datasets, + "data_type": lambda item, key, **context: f"{item.datatype.__class__.__module__}.{item.datatype.__class__.__name__}", + "converted": self.serialize_converted_datasets, # TODO: metadata/extra files } self.serializers.update(serializers) # this an abstract superclass, so no views created # because of that: we need to add a few keys that will use the default serializer - self.serializable_keyset.update(['name', 'state', 'tool_version', 'extension', 'visible', 'dbkey']) + self.serializable_keyset.update(["name", "state", "tool_version", "extension", "visible", "dbkey"]) def _proxy_to_dataset(self, serializer: base.Serializer = None, proxy_key=None): # dataset associations are (rough) proxies to datasets - access their serializer using this remapping fn @@ -515,7 +534,7 @@ class _UnflattenedMetadataDatasetAssociationSerializer(base.ModelSerializer[T], serializer = self.dataset_serializer.serializers.get(proxy_key) if serializer: return lambda item, key, **context: serializer(item.dataset, proxy_key or key, **context) - raise TypeError('kwarg serializer or key needed') + raise TypeError("kwarg serializer or key needed") def serialize_meta_files(self, item, key, **context): """ @@ -526,11 +545,16 @@ class _UnflattenedMetadataDatasetAssociationSerializer(base.ModelSerializer[T], for meta_type in dataset_assoc.metadata_file_types: if getattr(dataset_assoc.metadata, meta_type, None): meta_files.append( - dict(file_type=meta_type, - download_url=self.url_for('history_contents_metadata_file', - history_id=self.app.security.encode_id(dataset_assoc.history_id), - history_content_id=self.app.security.encode_id(dataset_assoc.id), - metadata_file=meta_type))) + dict( + file_type=meta_type, + download_url=self.url_for( + "history_contents_metadata_file", + history_id=self.app.security.encode_id(dataset_assoc.history_id), + history_content_id=self.app.security.encode_id(dataset_assoc.id), + metadata_file=meta_type, + ), + ) + ) return meta_files def serialize_metadata(self, item, key, excluded=None, **context): @@ -570,7 +594,7 @@ class _UnflattenedMetadataDatasetAssociationSerializer(base.ModelSerializer[T], """ dataset = item if dataset.creating_job: - return self.serialize_id(dataset.creating_job, 'id') + return self.serialize_id(dataset.creating_job, "id") else: return None @@ -597,7 +621,7 @@ class _UnflattenedMetadataDatasetAssociationSerializer(base.ModelSerializer[T], id_map = {} for converted in dataset_assoc.implicitly_converted_datasets: if not converted.deleted and converted.dataset: - id_map[converted.type] = self.serialize_id(converted.dataset, 'id') + id_map[converted.type] = self.serialize_id(converted.dataset, "id") return id_map @@ -607,7 +631,7 @@ class DatasetAssociationSerializer(_UnflattenedMetadataDatasetAssociationSeriali def add_serializers(self): super().add_serializers() # remove the single nesting key here - del self.serializers['metadata'] + del self.serializers["metadata"] def serialize(self, dataset_assoc, keys, **context): """ @@ -615,12 +639,12 @@ class DatasetAssociationSerializer(_UnflattenedMetadataDatasetAssociationSeriali """ # if 'metadata' isn't removed from keys here serialize will retrieve the un-serializable MetadataCollection # TODO: remove these when metadata is sub-object - KEYS_HANDLED_SEPARATELY = ('metadata', ) + KEYS_HANDLED_SEPARATELY = ("metadata",) left_to_handle = self._pluck_from_list(keys, KEYS_HANDLED_SEPARATELY) serialized = super().serialize(dataset_assoc, keys, **context) # add metadata directly to the dict instead of as a sub-object - if 'metadata' in left_to_handle: + if "metadata" in left_to_handle: metadata = self._prefixed_metadata(dataset_assoc) serialized.update(metadata) return serialized @@ -645,7 +669,7 @@ class DatasetAssociationSerializer(_UnflattenedMetadataDatasetAssociationSeriali prefixing each key with 'metadata_'. """ # build the original, nested dictionary - metadata = self.serialize_metadata(dataset_assoc, 'metadata') + metadata = self.serialize_metadata(dataset_assoc, "metadata") # prefix each key within and return prefixed = {} @@ -656,22 +680,22 @@ class DatasetAssociationSerializer(_UnflattenedMetadataDatasetAssociationSeriali class DatasetAssociationDeserializer(base.ModelDeserializer, deletable.PurgableDeserializerMixin): - def add_deserializers(self): super().add_deserializers() deletable.PurgableDeserializerMixin.add_deserializers(self) - self.deserializers.update({ - 'name': self.deserialize_basestring, - 'info': self.deserialize_basestring, - 'datatype': self.deserialize_datatype, - }) + self.deserializers.update( + { + "name": self.deserialize_basestring, + "info": self.deserialize_basestring, + "datatype": self.deserialize_datatype, + } + ) self.deserializable_keyset.update(self.deserializers.keys()) -# TODO: untested + # TODO: untested def deserialize_metadata(self, dataset_assoc, metadata_key, metadata_dict, **context): - """ - """ + """ """ self.validate.matches_type(metadata_key, metadata_dict, dict) returned = {} for key, val in metadata_dict.items(): @@ -679,12 +703,11 @@ class DatasetAssociationDeserializer(base.ModelDeserializer, deletable.PurgableD return returned def deserialize_metadatum(self, dataset_assoc, key, val, **context): - """ - """ + """ """ if key not in dataset_assoc.datatype.metadata_spec: return metadata_specification = dataset_assoc.datatype.metadata_spec[key] - if metadata_specification.get('readonly'): + if metadata_specification.get("readonly"): return unwrapped_val = metadata_specification.unwrap(val) setattr(dataset_assoc.metadata, key, unwrapped_val) @@ -700,37 +723,41 @@ class DatasetAssociationDeserializer(base.ModelDeserializer, deletable.PurgableD if not target_datatype.is_datatype_change_allowed(): raise exceptions.RequestParameterInvalidException("The target datatype does not allow datatype changes.") if not item.ok_to_edit_metadata(): - raise exceptions.RequestParameterInvalidException("Dataset metadata could not be updated because it is used as input or output of a running job.") + raise exceptions.RequestParameterInvalidException( + "Dataset metadata could not be updated because it is used as input or output of a running job." + ) item.change_datatype(val) sa_session = self.app.model.context sa_session.flush() trans = context.get("trans") - assert trans, "Logic error in Galaxy, deserialize_datatype not send a transation object" # TODO: restructure this for stronger typing - job, *_ = self.app.datatypes_registry.set_external_metadata_tool.tool_action.execute(self.app.datatypes_registry.set_external_metadata_tool, trans, incoming={'input1': item}, overwrite=False) # overwrite is False as per existing behavior + assert ( + trans + ), "Logic error in Galaxy, deserialize_datatype not send a transation object" # TODO: restructure this for stronger typing + job, *_ = self.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( + self.app.datatypes_registry.set_external_metadata_tool, trans, incoming={"input1": item}, overwrite=False + ) # overwrite is False as per existing behavior trans.app.job_manager.enqueue(job, tool=trans.app.datatypes_registry.set_external_metadata_tool) return item.datatype class DatasetAssociationFilterParser(base.ModelFilterParser, deletable.PurgableFiltersMixin): - def _add_parsers(self): super()._add_parsers() deletable.PurgableFiltersMixin._add_parsers(self) - self.orm_filter_parsers.update({ - 'name': {'op': ('eq', 'contains', 'like')}, - 'state': {'column': '_state', 'op': ('eq', 'in')}, - 'visible': {'op': ('eq'), 'val': base.parse_bool}, - }) - self.fn_filter_parsers.update({ - 'genome_build': self.string_standard_ops('dbkey'), - 'data_type': { - 'op': { - 'eq': self.eq_datatype, - 'isinstance': self.isinstance_datatype - } + self.orm_filter_parsers.update( + { + "name": {"op": ("eq", "contains", "like")}, + "state": {"column": "_state", "op": ("eq", "in")}, + "visible": {"op": ("eq"), "val": base.parse_bool}, } - }) + ) + self.fn_filter_parsers.update( + { + "genome_build": self.string_standard_ops("dbkey"), + "data_type": {"op": {"eq": self.eq_datatype, "isinstance": self.isinstance_datatype}}, + } + ) def eq_datatype(self, dataset_assoc, class_str): """ @@ -746,7 +773,7 @@ class DatasetAssociationFilterParser(base.ModelFilterParser, deletable.PurgableF """ parse_datatype_fn = self.app.datatypes_registry.get_datatype_class_by_name comparison_classes: List[Type] = [] - for class_str in class_strs.split(','): + for class_str in class_strs.split(","): datatype_class = parse_datatype_fn(class_str) if datatype_class: comparison_classes.append(datatype_class) diff --git a/lib/galaxy/managers/datatypes.py b/lib/galaxy/managers/datatypes.py index 6913bc32104..37f4df8c953 100644 --- a/lib/galaxy/managers/datatypes.py +++ b/lib/galaxy/managers/datatypes.py @@ -18,9 +18,7 @@ from galaxy.datatypes.registry import Registry def view_index( - datatypes_registry: Registry, - extension_only: Optional[bool] = True, - upload_only: Optional[bool] = True + datatypes_registry: Registry, extension_only: Optional[bool] = True, upload_only: Optional[bool] = True ) -> Union[List[DatatypeDetails], List[str]]: if extension_only: if upload_only: @@ -30,7 +28,7 @@ def view_index( else: rval = [] for datatype_info_dict in datatypes_registry.datatype_info_dicts: - if upload_only and not datatype_info_dict.get('display_in_upload'): + if upload_only and not datatype_info_dict.get("display_in_upload"): continue rval.append(datatype_info_dict) return rval @@ -60,20 +58,18 @@ def view_mapping(datatypes_registry: Registry) -> DatatypesMap: def view_types_and_mapping( - datatypes_registry: Registry, - extension_only: Optional[bool] = True, - upload_only: Optional[bool] = True + datatypes_registry: Registry, extension_only: Optional[bool] = True, upload_only: Optional[bool] = True ) -> DatatypesCombinedMap: return DatatypesCombinedMap( datatypes=view_index(datatypes_registry, extension_only, upload_only), - datatypes_mapping=view_mapping(datatypes_registry) + datatypes_mapping=view_mapping(datatypes_registry), ) def view_sniffers(datatypes_registry: Registry) -> List[str]: rval: List[str] = [] for sniffer_elem in datatypes_registry.sniffer_elems: - datatype = sniffer_elem.get('type') + datatype = sniffer_elem.get("type") if datatype is not None: rval.append(datatype) return rval @@ -83,11 +79,13 @@ def view_converters(datatypes_registry: Registry) -> DatatypeConverterList: converters = [] for (source_type, targets) in datatypes_registry.datatype_converters.items(): for target_type in targets: - converters.append({ - 'source': source_type, - 'target': target_type, - 'tool_id': targets[target_type].id, - }) + converters.append( + { + "source": source_type, + "target": target_type, + "tool_id": targets[target_type].id, + } + ) return parse_obj_as(DatatypeConverterList, converters) diff --git a/lib/galaxy/managers/deletable.py b/lib/galaxy/managers/deletable.py index a9c8ce6d6a8..b3aaf3802f9 100644 --- a/lib/galaxy/managers/deletable.py +++ b/lib/galaxy/managers/deletable.py @@ -8,7 +8,11 @@ models have some backing/supporting resources that can be removed as well the supporting resources as well. These models also have the boolean attribute 'purged'. """ -from typing import Any, Dict, Set +from typing import ( + Any, + Dict, + Set, +) from galaxy.model import _HasTable from .base import ( @@ -35,20 +39,20 @@ class DeletableManagerMixin: """ Mark as deleted and return. """ - return self._session_setattr(item, 'deleted', True, flush=flush) + return self._session_setattr(item, "deleted", True, flush=flush) def undelete(self, item, flush=True, **kwargs): """ Mark as not deleted and return. """ - return self._session_setattr(item, 'deleted', False, flush=flush) + return self._session_setattr(item, "deleted", False, flush=flush) class DeletableSerializerMixin: serializable_keyset: Set[str] def add_serializers(self): - self.serializable_keyset.add('deleted') + self.serializable_keyset.add("deleted") # TODO: these are of questionable value if we don't want to enable users to delete/purge via update @@ -56,7 +60,7 @@ class DeletableDeserializerMixin: deserializers: Dict[str, Deserializer] def add_deserializers(self): - self.deserializers['deleted'] = self.deserialize_deleted + self.deserializers["deleted"] = self.deserialize_deleted def deserialize_deleted(self, item, key, val, **context): """ @@ -77,9 +81,7 @@ class DeletableFiltersMixin: orm_filter_parsers: OrmFilterParsersType def _add_parsers(self): - self.orm_filter_parsers.update({ - 'deleted': {'op': ('eq'), 'val': parse_bool} - }) + self.orm_filter_parsers.update({"deleted": {"op": ("eq"), "val": parse_bool}}) class PurgableManagerMixin(DeletableManagerMixin): @@ -99,7 +101,7 @@ class PurgableManagerMixin(DeletableManagerMixin): Override this in subclasses to do the additional resource removal. """ self.delete(item, flush=False) - return self._session_setattr(item, 'purged', True, flush=flush) + return self._session_setattr(item, "purged", True, flush=flush) class PurgableSerializerMixin(DeletableSerializerMixin): @@ -107,7 +109,7 @@ class PurgableSerializerMixin(DeletableSerializerMixin): def add_serializers(self): DeletableSerializerMixin.add_serializers(self) - self.serializable_keyset.add('purged') + self.serializable_keyset.add("purged") class PurgableDeserializerMixin(DeletableDeserializerMixin): @@ -115,7 +117,7 @@ class PurgableDeserializerMixin(DeletableDeserializerMixin): def add_deserializers(self): DeletableDeserializerMixin.add_deserializers(self) - self.deserializers['purged'] = self.deserialize_purged + self.deserializers["purged"] = self.deserialize_purged def deserialize_purged(self, item, key, val, **context): """ @@ -131,9 +133,6 @@ class PurgableDeserializerMixin(DeletableDeserializerMixin): class PurgableFiltersMixin(DeletableFiltersMixin): - def _add_parsers(self): DeletableFiltersMixin._add_parsers(self) - self.orm_filter_parsers.update({ - 'purged': {'op': ('eq'), 'val': parse_bool} - }) + self.orm_filter_parsers.update({"purged": {"op": ("eq"), "val": parse_bool}}) diff --git a/lib/galaxy/managers/display_applications.py b/lib/galaxy/managers/display_applications.py index de88e8b9a9d..ec8572e8238 100644 --- a/lib/galaxy/managers/display_applications.py +++ b/lib/galaxy/managers/display_applications.py @@ -1,6 +1,7 @@ import logging from typing import ( - Any, Dict, + Any, + Dict, List, ) @@ -29,13 +30,15 @@ class DisplayApplicationsManager: """ rval = [] for display_app in self.datatypes_registry.display_applications.values(): - rval.append({ - 'id': display_app.id, - 'name': display_app.name, - 'version': display_app.version, - 'filename_': display_app._filename, - 'links': [{'name': link.name} for link in display_app.links.values()] - }) + rval.append( + { + "id": display_app.id, + "name": display_app.name, + "version": display_app.version, + "filename_": display_app._filename, + "links": [{"name": link.name} for link in display_app.links.values()], + } + ) return rval def reload(self, ids: List[str]) -> Dict[str, Any]: @@ -46,17 +49,21 @@ class DisplayApplicationsManager: :type ids: list """ self._app.queue_worker.send_control_task( - 'reload_display_application', - noop_self=True, - kwargs={'display_application_ids': ids} + "reload_display_application", noop_self=True, kwargs={"display_application_ids": ids} ) reloaded, failed = self.datatypes_registry.reload_display_applications(ids) if not reloaded and failed: - message = 'Unable to reload any of the %i requested display applications ("%s").' % (len(failed), '", "'.join(failed)) + message = 'Unable to reload any of the %i requested display applications ("%s").' % ( + len(failed), + '", "'.join(failed), + ) elif failed: - message = 'Reloaded %i display applications ("%s"), but failed to reload %i display applications ("%s").' % (len(reloaded), '", "'.join(reloaded), len(failed), '", "'.join(failed)) + message = ( + 'Reloaded %i display applications ("%s"), but failed to reload %i display applications ("%s").' + % (len(reloaded), '", "'.join(reloaded), len(failed), '", "'.join(failed)) + ) elif not reloaded: - message = 'You need to request at least one display application to reload.' + message = "You need to request at least one display application to reload." else: message = 'Reloaded %i requested display applications ("%s").' % (len(reloaded), '", "'.join(reloaded)) - return {'message': message, 'reloaded': reloaded, 'failed': failed} + return {"message": message, "reloaded": reloaded, "failed": failed} diff --git a/lib/galaxy/managers/executables.py b/lib/galaxy/managers/executables.py index 776f7299367..8fb95eacdaf 100644 --- a/lib/galaxy/managers/executables.py +++ b/lib/galaxy/managers/executables.py @@ -34,6 +34,4 @@ def artifact_class(trans, as_dict): return artifact_class, as_dict, object_id -__all__ = ( - 'artifact_class', -) +__all__ = ("artifact_class",) diff --git a/lib/galaxy/managers/folders.py b/lib/galaxy/managers/folders.py index 3dda0f85bca..4fe5daea419 100644 --- a/lib/galaxy/managers/folders.py +++ b/lib/galaxy/managers/folders.py @@ -5,7 +5,7 @@ import logging from sqlalchemy.orm.exc import ( MultipleResultsFound, - NoResultFound + NoResultFound, ) from galaxy import util @@ -45,11 +45,15 @@ class FolderManager: :raises: InconsistentDatabase, RequestParameterInvalidException, InternalServerError """ try: - folder = trans.sa_session.query(trans.app.model.LibraryFolder).filter(trans.app.model.LibraryFolder.table.c.id == decoded_folder_id).one() + folder = ( + trans.sa_session.query(trans.app.model.LibraryFolder) + .filter(trans.app.model.LibraryFolder.table.c.id == decoded_folder_id) + .one() + ) except MultipleResultsFound: - raise InconsistentDatabase('Multiple folders found with the same id.') + raise InconsistentDatabase("Multiple folders found with the same id.") except NoResultFound: - raise RequestParameterInvalidException('No folder found with the id provided.') + raise RequestParameterInvalidException("No folder found with the id provided.") except Exception as e: raise InternalServerError(f"Error loading from the database.{util.unicodify(e)}") folder = self.secure(trans, folder, check_manageable, check_accessible) @@ -88,10 +92,10 @@ class FolderManager: :raises: AuthenticationRequired, InsufficientPermissionsException """ if not trans.user: - raise AuthenticationRequired("Must be logged in to manage Galaxy items.", type='error') + raise AuthenticationRequired("Must be logged in to manage Galaxy items.", type="error") current_user_roles = trans.get_current_user_roles() if not trans.app.security_agent.can_modify_library_item(current_user_roles, folder): - raise InsufficientPermissionsException("You don't have permissions to modify this folder.", type='error') + raise InsufficientPermissionsException("You don't have permissions to modify this folder.", type="error") else: return folder @@ -105,10 +109,10 @@ class FolderManager: :raises: AuthenticationRequired, InsufficientPermissionsException """ if not trans.user: - raise AuthenticationRequired("Must be logged in to manage Galaxy items.", type='error') + raise AuthenticationRequired("Must be logged in to manage Galaxy items.", type="error") current_user_roles = trans.get_current_user_roles() if not trans.app.security_agent.can_manage_library_item(current_user_roles, folder): - raise InsufficientPermissionsException("You don't have permissions to manage this folder.", type='error') + raise InsufficientPermissionsException("You don't have permissions to manage this folder.", type="error") else: return folder @@ -130,15 +134,15 @@ class FolderManager: :rtype: dictionary """ - folder_dict = folder.to_dict(view='element') + folder_dict = folder.to_dict(view="element") folder_dict = trans.security.encode_all_ids(folder_dict, True) - folder_dict['id'] = f"F{folder_dict['id']}" - if folder_dict['parent_id'] is not None: - folder_dict['parent_id'] = f"F{folder_dict['parent_id']}" - folder_dict['update_time'] = folder.update_time + folder_dict["id"] = f"F{folder_dict['id']}" + if folder_dict["parent_id"] is not None: + folder_dict["parent_id"] = f"F{folder_dict['parent_id']}" + folder_dict["update_time"] = folder.update_time return folder_dict - def create(self, trans, parent_folder_id, new_folder_name, new_folder_description=''): + def create(self, trans, parent_folder_id, new_folder_name, new_folder_description=""): """ Create a new folder under the given folder. @@ -156,8 +160,12 @@ class FolderManager: """ parent_folder = self.get(trans, parent_folder_id) current_user_roles = trans.get_current_user_roles() - if not (trans.user_is_admin or trans.app.security_agent.can_add_library_item(current_user_roles, parent_folder)): - raise InsufficientPermissionsException('You do not have proper permission to create folders under given folder.') + if not ( + trans.user_is_admin or trans.app.security_agent.can_add_library_item(current_user_roles, parent_folder) + ): + raise InsufficientPermissionsException( + "You do not have proper permission to create folders under given folder." + ) new_folder = trans.app.model.LibraryFolder(name=new_folder_name, description=new_folder_description) # We are associating the last used genome build with folders, so we will always # initialize a new folder with the first dbkey in genome builds list which is currently @@ -238,16 +246,34 @@ class FolderManager: :rtype: dictionary """ # Omit duplicated roles by converting to set - modify_roles = set(trans.app.security_agent.get_roles_for_action(folder, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY)) - manage_roles = set(trans.app.security_agent.get_roles_for_action(folder, trans.app.security_agent.permitted_actions.LIBRARY_MANAGE)) - add_roles = set(trans.app.security_agent.get_roles_for_action(folder, trans.app.security_agent.permitted_actions.LIBRARY_ADD)) + modify_roles = set( + trans.app.security_agent.get_roles_for_action( + folder, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY + ) + ) + manage_roles = set( + trans.app.security_agent.get_roles_for_action( + folder, trans.app.security_agent.permitted_actions.LIBRARY_MANAGE + ) + ) + add_roles = set( + trans.app.security_agent.get_roles_for_action( + folder, trans.app.security_agent.permitted_actions.LIBRARY_ADD + ) + ) - modify_folder_role_list = [(modify_role.name, trans.security.encode_id(modify_role.id)) for modify_role in modify_roles] - manage_folder_role_list = [(manage_role.name, trans.security.encode_id(manage_role.id)) for manage_role in manage_roles] + modify_folder_role_list = [ + (modify_role.name, trans.security.encode_id(modify_role.id)) for modify_role in modify_roles + ] + manage_folder_role_list = [ + (manage_role.name, trans.security.encode_id(manage_role.id)) for manage_role in manage_roles + ] add_library_item_role_list = [(add_role.name, trans.security.encode_id(add_role.id)) for add_role in add_roles] - return dict(modify_folder_role_list=modify_folder_role_list, - manage_folder_role_list=manage_folder_role_list, - add_library_item_role_list=add_library_item_role_list) + return dict( + modify_folder_role_list=modify_folder_role_list, + manage_folder_role_list=manage_folder_role_list, + add_library_item_role_list=add_library_item_role_list, + ) def can_add_item(self, trans, folder): """ @@ -256,7 +282,11 @@ class FolderManager: if trans.user_is_admin: return True current_user_roles = trans.get_current_user_roles() - add_roles = set(trans.app.security_agent.get_roles_for_action(folder, trans.app.security_agent.permitted_actions.LIBRARY_ADD)) + add_roles = set( + trans.app.security_agent.get_roles_for_action( + folder, trans.app.security_agent.permitted_actions.LIBRARY_ADD + ) + ) for role in current_user_roles: if role in add_roles: return True @@ -274,10 +304,10 @@ class FolderManager: :raises: MalformedId """ - if ((len(encoded_folder_id) % 16 == 1) and encoded_folder_id.startswith('F')): + if (len(encoded_folder_id) % 16 == 1) and encoded_folder_id.startswith("F"): cut_id = encoded_folder_id[1:] else: - raise MalformedId(f'Malformed folder id ( {str(encoded_folder_id)} ) specified, unable to decode.') + raise MalformedId(f"Malformed folder id ( {str(encoded_folder_id)} ) specified, unable to decode.") return cut_id def decode_folder_id(self, trans, encoded_folder_id): diff --git a/lib/galaxy/managers/genomes.py b/lib/galaxy/managers/genomes.py index 9ea0f3a565e..47cde121746 100644 --- a/lib/galaxy/managers/genomes.py +++ b/lib/galaxy/managers/genomes.py @@ -13,7 +13,6 @@ from galaxy.structured_app import StructuredApp class GenomesManager: - def __init__(self, app: StructuredApp): self._app = app self.genomes = app.genomes @@ -22,36 +21,22 @@ class GenomesManager: return self.genomes.get_dbkeys(user, chrom_info) def get_genome( - self, - trans: ProvidesUserContext, - id: str, - num: int, - chrom: str, - low: int, - high: int, - reference: bool + self, trans: ProvidesUserContext, id: str, num: int, chrom: str, low: int, high: int, reference: bool ) -> Any: if reference: region = self.genomes.reference(trans, dbkey=id, chrom=chrom, low=low, high=high) - return {'dataset_type': 'refseq', 'data': region.sequence} + return {"dataset_type": "refseq", "data": region.sequence} else: return self.genomes.chroms(trans, dbkey=id, num=num, chrom=chrom, low=low) - def get_sequence( - self, - trans: ProvidesUserContext, - id: str, - chrom: str, - low: int, - high: int - ) -> Any: + def get_sequence(self, trans: ProvidesUserContext, id: str, chrom: str, low: int, high: int) -> Any: region = self.genomes.reference(trans, dbkey=id, chrom=chrom, low=low, high=high) return region.sequence def get_indexes(self, id: str, index_type: str) -> Any: - index_extensions = {'fasta_indexes': '.fai'} + index_extensions = {"fasta_indexes": ".fai"} if index_type not in index_extensions: - raise RequestParameterInvalidException(f'Invalid index type: {index_type}') + raise RequestParameterInvalidException(f"Invalid index type: {index_type}") tbl_entries = self._app.tool_data_tables.data_tables[index_type].data ext = index_extensions[index_type] @@ -60,15 +45,15 @@ class GenomesManager: with open(index_filename) as f: return f.read() except OSError: - raise ReferenceDataError(f'Failed to load index file for {id}') + raise ReferenceDataError(f"Failed to load index file for {id}") def _get_index_filename(self, id, tbl_entries, ext, index_type): try: paths = [x[-1] for x in tbl_entries if id in x] file_name = paths.pop() except TypeError: - raise ReferenceDataError(f'Data tables not found for {index_type}') + raise ReferenceDataError(f"Data tables not found for {index_type}") except IndexError: - raise ReferenceDataError(f'Data tables not found for {index_type} for {id}') + raise ReferenceDataError(f"Data tables not found for {index_type} for {id}") else: return f"{file_name}{ext}" diff --git a/lib/galaxy/managers/group_roles.py b/lib/galaxy/managers/group_roles.py index 2a149dd7df9..728c2bdc0d9 100644 --- a/lib/galaxy/managers/group_roles.py +++ b/lib/galaxy/managers/group_roles.py @@ -6,9 +6,7 @@ from typing import ( from galaxy import model from galaxy.app import MinimalManagerApp -from galaxy.exceptions import ( - ObjectNotFound, -) +from galaxy.exceptions import ObjectNotFound from galaxy.managers.base import decode_id from galaxy.managers.context import ProvidesAppContext from galaxy.schema.fields import EncodedDatabaseIdField @@ -80,11 +78,14 @@ class GroupRolesManager: raise ObjectNotFound(f"Role with id {encoded_role_id} was not found.") return role - def _get_group_role(self, trans: ProvidesAppContext, group: model.Group, role: model.Role) -> Optional[model.GroupRoleAssociation]: - return trans.sa_session.query(model.GroupRoleAssociation).filter( - model.GroupRoleAssociation.group == group, - model.GroupRoleAssociation.role == role - ).one_or_none() + def _get_group_role( + self, trans: ProvidesAppContext, group: model.Group, role: model.Role + ) -> Optional[model.GroupRoleAssociation]: + return ( + trans.sa_session.query(model.GroupRoleAssociation) + .filter(model.GroupRoleAssociation.group == group, model.GroupRoleAssociation.role == role) + .one_or_none() + ) def _add_role_to_group(self, trans: ProvidesAppContext, group: model.Group, role: model.Role): gra = model.GroupRoleAssociation(group, role) diff --git a/lib/galaxy/managers/group_users.py b/lib/galaxy/managers/group_users.py index 1bbef7ec4a2..5cc6438f08d 100644 --- a/lib/galaxy/managers/group_users.py +++ b/lib/galaxy/managers/group_users.py @@ -8,9 +8,7 @@ from typing import ( from galaxy import model from galaxy.app import MinimalManagerApp -from galaxy.exceptions import ( - ObjectNotFound, -) +from galaxy.exceptions import ObjectNotFound from galaxy.managers.base import decode_id from galaxy.managers.context import ProvidesAppContext from galaxy.schema.fields import EncodedDatabaseIdField @@ -36,7 +34,9 @@ class GroupUsersManager: rval.append(group_user) return rval - def show(self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField) -> Dict[str, Any]: + def show( + self, trans: ProvidesAppContext, id: EncodedDatabaseIdField, group_id: EncodedDatabaseIdField + ) -> Dict[str, Any]: """ Returns information about a group user. """ @@ -89,11 +89,14 @@ class GroupUsersManager: raise ObjectNotFound(f"User with id {encoded_user_id} was not found.") return user - def _get_group_user(self, trans: ProvidesAppContext, group: model.Group, user: model.User) -> Optional[model.UserGroupAssociation]: - return trans.sa_session.query(model.UserGroupAssociation).filter( - model.UserGroupAssociation.user == user, - model.UserGroupAssociation.group == group - ).one_or_none() + def _get_group_user( + self, trans: ProvidesAppContext, group: model.Group, user: model.User + ) -> Optional[model.UserGroupAssociation]: + return ( + trans.sa_session.query(model.UserGroupAssociation) + .filter(model.UserGroupAssociation.user == user, model.UserGroupAssociation.group == group) + .one_or_none() + ) def _add_user_to_group(self, trans: ProvidesAppContext, group: model.Group, user: model.User): gra = model.UserGroupAssociation(user, group) @@ -109,5 +112,5 @@ class GroupUsersManager: return { "id": encoded_user_id, "email": user.email, - "url": url_for('group_user', group_id=encoded_group_id, id=encoded_user_id) + "url": url_for("group_user", group_id=encoded_group_id, id=encoded_user_id), } diff --git a/lib/galaxy/managers/groups.py b/lib/galaxy/managers/groups.py index d1d47468251..b401158ffde 100644 --- a/lib/galaxy/managers/groups.py +++ b/lib/galaxy/managers/groups.py @@ -31,9 +31,9 @@ class GroupsManager: """ rval = [] for group in trans.sa_session.query(model.Group).filter(model.Group.deleted == false()): - item = group.to_dict(value_mapper={'id': trans.security.encode_id}) + item = group.to_dict(value_mapper={"id": trans.security.encode_id}) encoded_id = trans.security.encode_id(group.id) - item['url'] = url_for('group', id=encoded_id) + item["url"] = url_for("group", id=encoded_id) rval.append(item) return rval @@ -41,23 +41,23 @@ class GroupsManager: """ Creates a new group. """ - name = payload.get('name', None) + name = payload.get("name", None) if name is None: raise ObjectAttributeMissingException("Missing required name") self._check_duplicated_group_name(trans, name) group = model.Group(name=name) trans.sa_session.add(group) - encoded_user_ids = payload.get('user_ids', []) + encoded_user_ids = payload.get("user_ids", []) users = self._get_users_by_encoded_ids(trans, encoded_user_ids) - encoded_role_ids = payload.get('role_ids', []) + encoded_role_ids = payload.get("role_ids", []) roles = self._get_roles_by_encoded_ids(trans, encoded_role_ids) trans.app.security_agent.set_entity_group_associations(groups=[group], roles=roles, users=users) trans.sa_session.flush() encoded_id = trans.security.encode_id(group.id) - item = group.to_dict(view='element', value_mapper={'id': trans.security.encode_id}) - item['url'] = url_for('group', id=encoded_id) + item = group.to_dict(view="element", value_mapper={"id": trans.security.encode_id}) + item["url"] = url_for("group", id=encoded_id) return [item] def show(self, trans: ProvidesAppContext, encoded_id: EncodedDatabaseIdField): @@ -65,10 +65,10 @@ class GroupsManager: Displays information about a group. """ group = self._get_group(trans, encoded_id) - item = group.to_dict(view='element', value_mapper={'id': trans.security.encode_id}) - item['url'] = url_for('group', id=encoded_id) - item['users_url'] = url_for('group_users', group_id=encoded_id) - item['roles_url'] = url_for('group_roles', group_id=encoded_id) + item = group.to_dict(view="element", value_mapper={"id": trans.security.encode_id}) + item["url"] = url_for("group", id=encoded_id) + item["users_url"] = url_for("group_users", group_id=encoded_id) + item["roles_url"] = url_for("group_roles", group_id=encoded_id) return item def update(self, trans: ProvidesAppContext, encoded_id: EncodedDatabaseIdField, payload: Dict[str, Any]): @@ -76,16 +76,18 @@ class GroupsManager: Modifies a group. """ group = self._get_group(trans, encoded_id) - name = payload.get('name', None) + name = payload.get("name", None) if name: self._check_duplicated_group_name(trans, name) group.name = name trans.sa_session.add(group) - encoded_user_ids = payload.get('user_ids', []) + encoded_user_ids = payload.get("user_ids", []) users = self._get_users_by_encoded_ids(trans, encoded_user_ids) - encoded_role_ids = payload.get('role_ids', []) + encoded_role_ids = payload.get("role_ids", []) roles = self._get_roles_by_encoded_ids(trans, encoded_role_ids) - trans.app.security_agent.set_entity_group_associations(groups=[group], roles=roles, users=users, delete_existing_assocs=False) + trans.app.security_agent.set_entity_group_associations( + groups=[group], roles=roles, users=users, delete_existing_assocs=False + ) trans.sa_session.flush() def _decode_id(self, encoded_id: EncodedDatabaseIdField) -> int: @@ -105,12 +107,16 @@ class GroupsManager: raise ObjectNotFound(f"Group with id {encoded_id} was not found.") return group - def _get_users_by_encoded_ids(self, trans: ProvidesAppContext, encoded_user_ids: List[EncodedDatabaseIdField]) -> List[model.User]: + def _get_users_by_encoded_ids( + self, trans: ProvidesAppContext, encoded_user_ids: List[EncodedDatabaseIdField] + ) -> List[model.User]: decoded_user_ids = self._decode_ids(encoded_user_ids) users = trans.sa_session.query(model.User).filter(model.User.table.c.id.in_(decoded_user_ids)).all() return users - def _get_roles_by_encoded_ids(self, trans: ProvidesAppContext, encoded_role_ids: List[EncodedDatabaseIdField]) -> List[model.Role]: + def _get_roles_by_encoded_ids( + self, trans: ProvidesAppContext, encoded_role_ids: List[EncodedDatabaseIdField] + ) -> List[model.Role]: decoded_role_ids = self._decode_ids(encoded_role_ids) roles = trans.sa_session.query(model.Role).filter(model.Role.id.in_(decoded_role_ids)).all() return roles diff --git a/lib/galaxy/managers/hdas.py b/lib/galaxy/managers/hdas.py index fd59c1585a9..50f6ac76582 100644 --- a/lib/galaxy/managers/hdas.py +++ b/lib/galaxy/managers/hdas.py @@ -7,14 +7,18 @@ history. import gettext import logging import os -from typing import Any, Dict, List +from typing import ( + Any, + Dict, + List, +) from sqlalchemy.orm.session import object_session from galaxy import ( datatypes, exceptions, - model + model, ) from galaxy.managers import ( annotatable, @@ -25,7 +29,10 @@ from galaxy.managers import ( users, ) from galaxy.model.tags import GalaxyTagHandler -from galaxy.structured_app import MinimalManagerApp, StructuredApp +from galaxy.structured_app import ( + MinimalManagerApp, + StructuredApp, +) log = logging.getLogger(__name__) @@ -34,15 +41,18 @@ class HistoryDatasetAssociationNoHistoryException(Exception): pass -class HDAManager(datasets.DatasetAssociationManager, - secured.OwnableManagerMixin, - taggable.TaggableManagerMixin, - annotatable.AnnotatableManagerMixin): +class HDAManager( + datasets.DatasetAssociationManager, + secured.OwnableManagerMixin, + taggable.TaggableManagerMixin, + annotatable.AnnotatableManagerMixin, +): """ Interface/service object for interacting with HDAs. """ + model_class = model.HistoryDatasetAssociation - foreign_key_name = 'history_dataset_association' + foreign_key_name = "history_dataset_association" tag_assoc = model.HistoryDatasetAssociationTagAssociation annotation_assoc = model.HistoryDatasetAssociationAnnotationAssociation @@ -60,8 +70,7 @@ class HDAManager(datasets.DatasetAssociationManager, self.tag_handler = tag_handler def get_owned_ids(self, object_ids, history=None): - """Get owned IDs. - """ + """Get owned IDs.""" filters = [self.model_class.table.c.id.in_(object_ids), self.model_class.table.c.history_id == history.id] return self.list(filters=filters) @@ -104,12 +113,13 @@ class HDAManager(datasets.DatasetAssociationManager, it will be automatically set. """ if not dataset: - kwargs['create_dataset'] = True - hda = model.HistoryDatasetAssociation(history=history, dataset=dataset, - sa_session=self.app.model.context, **kwargs) + kwargs["create_dataset"] = True + hda = model.HistoryDatasetAssociation( + history=history, dataset=dataset, sa_session=self.app.model.context, **kwargs + ) if history: - history.add_dataset(hda, set_hid=('hid' not in kwargs)) + history.add_dataset(hda, set_hid=("hid" not in kwargs)) # TODO:?? some internal sanity check here (or maybe in add_dataset) to make sure hids are not duped? self.session().add(hda) @@ -121,7 +131,7 @@ class HDAManager(datasets.DatasetAssociationManager, """ Copy hda, including annotation and tags, add to history and return the given HDA. """ - copy = hda.copy(parent_id=kwargs.get('parent_id'), copy_hid=False, copy_tags=hda.tags, flush=flush) + copy = hda.copy(parent_id=kwargs.get("parent_id"), copy_hid=False, copy_tags=hda.tags, flush=flush) if hide_copy: copy.visible = False if history: @@ -148,6 +158,7 @@ class HDAManager(datasets.DatasetAssociationManager, def purge(self, hda, flush=True): if self.app.config.enable_celery_tasks: from galaxy.celery.tasks import purge_hda + purge_hda.delay(hda_id=hda.id) else: self._purge(hda, flush=flush) @@ -180,8 +191,7 @@ class HDAManager(datasets.DatasetAssociationManager, Return True if the hda's job was resubmitted at any point. """ job_states = model.Job.states - query = (self._job_state_history_query(hda) - .filter(model.JobStateHistory.state == job_states.RESUBMITTED)) + query = self._job_state_history_query(hda).filter(model.JobStateHistory.state == job_states.RESUBMITTED) return self.app.model.context.query(query.exists()).scalar() def _job_state_history_query(self, hda): @@ -195,10 +205,12 @@ class HDAManager(datasets.DatasetAssociationManager, # TODO: this does not play well with copied hdas # NOTE: don't eagerload (JODA will load the hda were using!) hda_id = hda.id - query = (session.query(JobToOutputDatasetAssociation, JobStateHistory) - .filter(JobToOutputDatasetAssociation.dataset_id == hda_id) - .filter(JobStateHistory.job_id == JobToOutputDatasetAssociation.job_id) - .enable_eagerloads(False)) + query = ( + session.query(JobToOutputDatasetAssociation, JobStateHistory) + .filter(JobToOutputDatasetAssociation.dataset_id == hda_id) + .filter(JobStateHistory.job_id == JobToOutputDatasetAssociation.job_id) + .enable_eagerloads(False) + ) return query def data_conversion_status(self, hda): @@ -244,8 +256,9 @@ class HDAManager(datasets.DatasetAssociationManager, # The user associated the DATASET_ACCESS permission on the dataset with 1 or more roles. We # need to ensure that they did not associate roles that would cause accessibility problems. security_agent = trans.app.security_agent - permissions, in_roles, error, message = \ - security_agent.derive_roles_from_access(trans, hda.dataset.id, 'root', **role_ids_dict) + permissions, in_roles, error, message = security_agent.derive_roles_from_access( + trans, hda.dataset.id, "root", **role_ids_dict + ) if error: # Keep the original role associations for the DATASET_ACCESS permission on the dataset. access_action = security_agent.get_action(security_agent.permitted_actions.DATASET_ACCESS.action) @@ -260,9 +273,10 @@ class HDAManager(datasets.DatasetAssociationManager, class HDASerializer( # datasets._UnflattenedMetadataDatasetAssociationSerializer, - datasets.DatasetAssociationSerializer[HDAManager], - taggable.TaggableSerializerMixin, - annotatable.AnnotatableSerializerMixin): + datasets.DatasetAssociationSerializer[HDAManager], + taggable.TaggableSerializerMixin, + annotatable.AnnotatableSerializerMixin, +): model_manager_class = HDAManager app: StructuredApp @@ -270,125 +284,139 @@ class HDASerializer( # datasets._UnflattenedMetadataDatasetAssociationSerialize super().__init__(app) self.hda_manager = self.manager - self.default_view = 'summary' - self.add_view('summary', [ - 'id', - 'type_id', - 'name', - 'history_id', - 'hid', - 'history_content_type', - 'dataset_id', - 'state', - 'extension', - 'deleted', 'purged', 'visible', - 'tags', - 'type', - 'url', - 'create_time', - 'update_time', - ]) - self.add_view('detailed', [ - 'model_class', - 'history_id', 'hid', - # why include if model_class is there? - 'hda_ldda', - 'copied_from_ldda_id', - # TODO: accessible needs to go away - 'accessible', + self.default_view = "summary" + self.add_view( + "summary", + [ + "id", + "type_id", + "name", + "history_id", + "hid", + "history_content_type", + "dataset_id", + "state", + "extension", + "deleted", + "purged", + "visible", + "tags", + "type", + "url", + "create_time", + "update_time", + ], + ) + self.add_view( + "detailed", + [ + "model_class", + "history_id", + "hid", + # why include if model_class is there? + "hda_ldda", + "copied_from_ldda_id", + # TODO: accessible needs to go away + "accessible", + # remapped + "genome_build", + "misc_info", + "misc_blurb", + "file_ext", + "file_size", + "resubmitted", + "metadata", + "meta_files", + "data_type", + "peek", + "creating_job", + "rerunnable", + "uuid", + "permissions", + "file_name", + "display_apps", + "display_types", + "visualizations", + "validated_state", + "validated_state_message", + # 'url', + "download_url", + "annotation", + "api_type", + "created_from_basename", + "hashes", + "sources", + ], + include_keys_from="summary", + ) - # remapped - 'genome_build', 'misc_info', 'misc_blurb', - 'file_ext', 'file_size', - - 'resubmitted', - 'metadata', 'meta_files', 'data_type', - 'peek', - - 'creating_job', - 'rerunnable', - - 'uuid', - 'permissions', - 'file_name', - - 'display_apps', - 'display_types', - 'visualizations', - - 'validated_state', - 'validated_state_message', - - # 'url', - 'download_url', - - 'annotation', - - 'api_type', - 'created_from_basename', - 'hashes', - 'sources', - ], include_keys_from='summary') - - self.add_view('extended', [ - 'tool_version', 'parent_id', 'designation', - ], include_keys_from='detailed') + self.add_view( + "extended", + [ + "tool_version", + "parent_id", + "designation", + ], + include_keys_from="detailed", + ) # keyset returned to create show a dataset where the owner has no access - self.add_view('inaccessible', [ - 'accessible', - 'id', 'name', 'history_id', 'hid', 'history_content_type', - 'state', 'deleted', 'visible' - ]) + self.add_view( + "inaccessible", + ["accessible", "id", "name", "history_id", "hid", "history_content_type", "state", "deleted", "visible"], + ) # fields for new beta web client, there is no summary/detailed split any more - self.add_view('betawebclient', [ - # common to hdca - 'create_time', - 'deleted', - 'hid', - 'history_content_type', - 'history_id', - 'id', - 'name', - 'tags', - 'type', - 'type_id', - 'update_time', - 'url', - 'visible', - # dataset only - 'accessible', - 'api_type', - 'annotation', - 'created_from_basename', - 'creating_job', - 'dataset_id', - 'data_type', - 'display_apps', - 'display_types', - 'download_url', - 'extension', - 'file_ext', - 'file_name', - 'file_size', - 'genome_build', - 'hda_ldda', - 'meta_files', - 'misc_blurb', - 'misc_info', - 'model_class', - 'peek', - 'purged', - 'rerunnable', - 'resubmitted', - 'state', - 'uuid', - 'validated_state', - 'validated_state_message', - 'hashes', - 'sources', - ]) + self.add_view( + "betawebclient", + [ + # common to hdca + "create_time", + "deleted", + "hid", + "history_content_type", + "history_id", + "id", + "name", + "tags", + "type", + "type_id", + "update_time", + "url", + "visible", + # dataset only + "accessible", + "api_type", + "annotation", + "created_from_basename", + "creating_job", + "dataset_id", + "data_type", + "display_apps", + "display_types", + "download_url", + "extension", + "file_ext", + "file_name", + "file_size", + "genome_build", + "hda_ldda", + "meta_files", + "misc_blurb", + "misc_info", + "model_class", + "peek", + "purged", + "rerunnable", + "resubmitted", + "state", + "uuid", + "validated_state", + "validated_state_message", + "hashes", + "sources", + ], + ) def serialize_copied_from_ldda_id(self, item, key, **context): """ @@ -404,44 +432,45 @@ class HDASerializer( # datasets._UnflattenedMetadataDatasetAssociationSerialize annotatable.AnnotatableSerializerMixin.add_serializers(self) serializers: Dict[str, base.Serializer] = { - 'model_class': lambda item, key, **context: 'HistoryDatasetAssociation', - 'history_content_type': lambda item, key, **context: 'dataset', - 'hda_ldda': lambda item, key, **context: 'hda', - 'type_id': self.serialize_type_id, - 'copied_from_ldda_id': self.serialize_copied_from_ldda_id, - 'history_id': self.serialize_id, - + "model_class": lambda item, key, **context: "HistoryDatasetAssociation", + "history_content_type": lambda item, key, **context: "dataset", + "hda_ldda": lambda item, key, **context: "hda", + "type_id": self.serialize_type_id, + "copied_from_ldda_id": self.serialize_copied_from_ldda_id, + "history_id": self.serialize_id, # remapped - 'misc_info': self._remap_from('info'), - 'misc_blurb': self._remap_from('blurb'), - 'file_ext': self._remap_from('extension'), - 'file_path': self._remap_from('file_name'), - 'resubmitted': lambda item, key, **context: self.hda_manager.has_been_resubmitted(item), - 'display_apps': self.serialize_display_apps, - 'display_types': self.serialize_old_display_applications, - 'visualizations': self.serialize_visualization_links, - + "misc_info": self._remap_from("info"), + "misc_blurb": self._remap_from("blurb"), + "file_ext": self._remap_from("extension"), + "file_path": self._remap_from("file_name"), + "resubmitted": lambda item, key, **context: self.hda_manager.has_been_resubmitted(item), + "display_apps": self.serialize_display_apps, + "display_types": self.serialize_old_display_applications, + "visualizations": self.serialize_visualization_links, # 'url' : url_for( 'history_content_typed', history_id=encoded_history_id, id=encoded_id, type="dataset" ), # TODO: this intermittently causes a routes.GenerationException - temp use the legacy route to prevent this # see also: https://trello.com/c/5d6j4X5y # see also: https://sentry.galaxyproject.org/galaxy/galaxy-main/group/20769/events/9352883/ - 'url': lambda item, key, **context: self.url_for('history_content', - history_id=self.app.security.encode_id(item.history_id), - id=self.app.security.encode_id(item.id)), - 'urls': self.serialize_urls, - + "url": lambda item, key, **context: self.url_for( + "history_content", + history_id=self.app.security.encode_id(item.history_id), + id=self.app.security.encode_id(item.id), + ), + "urls": self.serialize_urls, # TODO: backwards compat: need to go away - 'download_url': lambda item, key, **context: self.url_for('history_contents_display', - history_id=self.app.security.encode_id(item.history.id), - history_content_id=self.app.security.encode_id(item.id)), - 'parent_id': self.serialize_id, + "download_url": lambda item, key, **context: self.url_for( + "history_contents_display", + history_id=self.app.security.encode_id(item.history.id), + history_content_id=self.app.security.encode_id(item.id), + ), + "parent_id": self.serialize_id, # TODO: to DatasetAssociationSerializer - 'accessible': lambda item, key, user=None, **c: self.manager.is_accessible(item, user, **c), - 'api_type': lambda item, key, **context: 'file', - 'type': lambda item, key, **context: 'file', - 'created_from_basename': lambda item, key, **context: item.created_from_basename, - 'hashes': lambda item, key, **context: [h.to_dict() for h in item.hashes], - 'sources': lambda item, key, **context: [s.to_dict() for s in item.sources], + "accessible": lambda item, key, user=None, **c: self.manager.is_accessible(item, user, **c), + "api_type": lambda item, key, **context: "file", + "type": lambda item, key, **context: "file", + "created_from_basename": lambda item, key, **context: item.created_from_basename, + "hashes": lambda item, key, **context: [h.to_dict() for h in item.hashes], + "sources": lambda item, key, **context: [s.to_dict() for s in item.sources], } self.serializers.update(serializers) @@ -451,7 +480,7 @@ class HDASerializer( # datasets._UnflattenedMetadataDatasetAssociationSerialize """ # TODO: to DatasetAssociationSerializer if not self.manager.is_accessible(hda, user, **context): - keys = self._view_to_keys('inaccessible') + keys = self._view_to_keys("inaccessible") return super().serialize(hda, keys, user=user, **context) def serialize_display_apps(self, item, key, trans=None, **context): @@ -464,11 +493,13 @@ class HDASerializer( # datasets._UnflattenedMetadataDatasetAssociationSerialize app_links = [] for link_app in display_app.links.values(): - app_links.append({ - 'target': link_app.url.get('target_frame', '_blank'), - 'href': link_app.get_display_url(hda, trans), - 'text': gettext.gettext(link_app.name) - }) + app_links.append( + { + "target": link_app.url.get("target_frame", "_blank"), + "href": link_app.get_display_url(hda, trans), + "text": gettext.gettext(link_app.name), + } + ) if app_links: display_apps.append(dict(label=display_app.name, links=app_links)) @@ -492,11 +523,9 @@ class HDASerializer( # datasets._UnflattenedMetadataDatasetAssociationSerialize app_links = [] for display_name, display_link in display_links: - app_links.append({ - 'target': target_frame, - 'href': display_link, - 'text': gettext.gettext(display_name) - }) + app_links.append( + {"target": target_frame, "href": display_link, "text": gettext.gettext(display_name)} + ) if app_links: display_apps.append(dict(label=display_label, links=app_links)) @@ -521,28 +550,32 @@ class HDASerializer( # datasets._UnflattenedMetadataDatasetAssociationSerialize url_for = self.url_for encoded_id = self.app.security.encode_id(hda.id) urls = { - 'purge': url_for(controller='dataset', action='purge_async', dataset_id=encoded_id), - 'display': url_for(controller='dataset', action='display', dataset_id=encoded_id, preview=True), - 'edit': url_for(controller='dataset', action='edit', dataset_id=encoded_id), - 'download': url_for(controller='dataset', action='display', - dataset_id=encoded_id, to_ext=hda.extension), - 'report_error': url_for(controller='dataset', action='errors', id=encoded_id), - 'rerun': url_for(controller='tool_runner', action='rerun', id=encoded_id), - 'show_params': url_for(controller='dataset', action='details', dataset_id=encoded_id), - 'visualization': url_for(controller='visualization', action='index', - id=encoded_id, model='HistoryDatasetAssociation'), - 'meta_download': url_for(controller='dataset', action='get_metadata_file', - hda_id=encoded_id, metadata_name=''), + "purge": url_for(controller="dataset", action="purge_async", dataset_id=encoded_id), + "display": url_for(controller="dataset", action="display", dataset_id=encoded_id, preview=True), + "edit": url_for(controller="dataset", action="edit", dataset_id=encoded_id), + "download": url_for(controller="dataset", action="display", dataset_id=encoded_id, to_ext=hda.extension), + "report_error": url_for(controller="dataset", action="errors", id=encoded_id), + "rerun": url_for(controller="tool_runner", action="rerun", id=encoded_id), + "show_params": url_for(controller="dataset", action="details", dataset_id=encoded_id), + "visualization": url_for( + controller="visualization", action="index", id=encoded_id, model="HistoryDatasetAssociation" + ), + "meta_download": url_for( + controller="dataset", action="get_metadata_file", hda_id=encoded_id, metadata_name="" + ), } return urls -class HDADeserializer(datasets.DatasetAssociationDeserializer, - taggable.TaggableDeserializerMixin, - annotatable.AnnotatableDeserializerMixin): +class HDADeserializer( + datasets.DatasetAssociationDeserializer, + taggable.TaggableDeserializerMixin, + annotatable.AnnotatableDeserializerMixin, +): """ Interface/service object for validating and deserializing dictionaries into histories. """ + model_manager_class = HDAManager def __init__(self, app: MinimalManagerApp): @@ -555,19 +588,22 @@ class HDADeserializer(datasets.DatasetAssociationDeserializer, taggable.TaggableDeserializerMixin.add_deserializers(self) annotatable.AnnotatableDeserializerMixin.add_deserializers(self) - self.deserializers.update({ - 'visible': self.deserialize_bool, - # remapped - 'genome_build': lambda item, key, val, **c: self.deserialize_genome_build(item, 'dbkey', val), - 'misc_info': lambda item, key, val, **c: self.deserialize_basestring(item, 'info', val, - convert_none_to_empty=True), - }) + self.deserializers.update( + { + "visible": self.deserialize_bool, + # remapped + "genome_build": lambda item, key, val, **c: self.deserialize_genome_build(item, "dbkey", val), + "misc_info": lambda item, key, val, **c: self.deserialize_basestring( + item, "info", val, convert_none_to_empty=True + ), + } + ) self.deserializable_keyset.update(self.deserializers.keys()) -class HDAFilterParser(datasets.DatasetAssociationFilterParser, - taggable.TaggableFilterMixin, - annotatable.AnnotatableFilterMixin): +class HDAFilterParser( + datasets.DatasetAssociationFilterParser, taggable.TaggableFilterMixin, annotatable.AnnotatableFilterMixin +): model_manager_class = HDAManager model_class = model.HistoryDatasetAssociation diff --git a/lib/galaxy/managers/hdcas.py b/lib/galaxy/managers/hdcas.py index 9f6b4fe9b92..ec9b23d1649 100644 --- a/lib/galaxy/managers/hdcas.py +++ b/lib/galaxy/managers/hdcas.py @@ -14,14 +14,16 @@ from galaxy.managers import ( deletable, hdas, secured, - taggable + taggable, ) from galaxy.managers.collections_util import get_hda_and_element_identifiers from galaxy.model.tags import GalaxyTagHandler -from galaxy.structured_app import MinimalManagerApp, StructuredApp +from galaxy.structured_app import ( + MinimalManagerApp, + StructuredApp, +) from galaxy.util.zipstream import ZipstreamWrapper - log = logging.getLogger(__name__) @@ -48,17 +50,19 @@ def set_collection_attributes(dataset_element, *payload): # TODO: to DatasetCollectionInstanceManager class HDCAManager( - base.ModelManager, - secured.AccessibleManagerMixin, - secured.OwnableManagerMixin, - deletable.PurgableManagerMixin, - taggable.TaggableManagerMixin, - annotatable.AnnotatableManagerMixin): + base.ModelManager, + secured.AccessibleManagerMixin, + secured.OwnableManagerMixin, + deletable.PurgableManagerMixin, + taggable.TaggableManagerMixin, + annotatable.AnnotatableManagerMixin, +): """ Interface/service object for interacting with HDCAs. """ + model_class = model.HistoryDatasetCollectionAssociation - foreign_key_name = 'history_dataset_collection_association' + foreign_key_name = "history_dataset_collection_association" tag_assoc = model.HistoryDatasetCollectionTagAssociation annotation_assoc = model.HistoryDatasetCollectionAssociationAnnotationAssociation @@ -79,10 +83,10 @@ class HDCAManager( """ returned = [] # lots of nesting going on within the nesting - collection = content.collection if hasattr(content, 'collection') else content - this_parents = (content, ) + parents + collection = content.collection if hasattr(content, "collection") else content + this_parents = (content,) + parents for element in collection.elements: - next_parents = (element, ) + this_parents + next_parents = (element,) + this_parents if element.is_collection: processed_list = self.map_datasets(element.child_collection, fn, *next_parents) returned.extend(processed_list) @@ -108,28 +112,21 @@ class DCESerializer(base.ModelSerializer): self.hda_serializer = hdas.HDASerializer(app) self.dc_serializer = DCSerializer(app, dce_serializer=self) - self.default_view = 'summary' - self.add_view('summary', [ - 'id', 'model_class', - 'element_index', - 'element_identifier', - 'element_type', - 'object' - ]) + self.default_view = "summary" + self.add_view("summary", ["id", "model_class", "element_index", "element_identifier", "element_type", "object"]) def add_serializers(self): super().add_serializers() - self.serializers.update({ - 'model_class': lambda *a, **c: 'DatasetCollectionElement', - 'object': self.serialize_object - }) + self.serializers.update( + {"model_class": lambda *a, **c: "DatasetCollectionElement", "object": self.serialize_object} + ) def serialize_object(self, item, key, **context): if item.hda: - return self.hda_serializer.serialize_to_view(item.hda, view='summary', **context) + return self.hda_serializer.serialize_to_view(item.hda, view="summary", **context) if item.child_collection: - return self.dc_serializer.serialize_to_view(item.child_collection, view='detailed', **context) - return 'object' + return self.dc_serializer.serialize_to_view(item.child_collection, view="detailed", **context) + return "object" class DCSerializer(base.ModelSerializer): @@ -141,32 +138,41 @@ class DCSerializer(base.ModelSerializer): super().__init__(app) self.dce_serializer = dce_serializer or DCESerializer(app) - self.default_view = 'summary' - self.add_view('summary', [ - 'id', - 'create_time', - 'update_time', - 'collection_type', - 'populated_state', - 'populated_state_message', - 'element_count', - ]) - self.add_view('detailed', [ - 'populated', - 'elements', - ], include_keys_from='summary') + self.default_view = "summary" + self.add_view( + "summary", + [ + "id", + "create_time", + "update_time", + "collection_type", + "populated_state", + "populated_state_message", + "element_count", + ], + ) + self.add_view( + "detailed", + [ + "populated", + "elements", + ], + include_keys_from="summary", + ) def add_serializers(self): super().add_serializers() - self.serializers.update({ - 'model_class': lambda *a, **c: 'DatasetCollection', - 'elements': self.serialize_elements, - }) + self.serializers.update( + { + "model_class": lambda *a, **c: "DatasetCollection", + "elements": self.serialize_elements, + } + ) def serialize_elements(self, item, key, **context): returned = [] for element in item.elements: - serialized = self.dce_serializer.serialize_to_view(element, view='summary', **context) + serialized = self.dce_serializer.serialize_to_view(element, view="summary", **context) returned.append(serialized) return returned @@ -175,25 +181,34 @@ class DCASerializer(base.ModelSerializer): """ Base (abstract) Serializer class for HDCAs and LDCAs. """ + app: StructuredApp def __init__(self, app: StructuredApp, dce_serializer=None): super().__init__(app) self.dce_serializer = dce_serializer or DCESerializer(app) - self.default_view = 'summary' - self.add_view('summary', [ - 'id', - 'create_time', 'update_time', - 'collection_type', - 'populated_state', - 'populated_state_message', - 'element_count', - ]) - self.add_view('detailed', [ - 'populated', - 'elements', - ], include_keys_from='summary') + self.default_view = "summary" + self.add_view( + "summary", + [ + "id", + "create_time", + "update_time", + "collection_type", + "populated_state", + "populated_state_message", + "element_count", + ], + ) + self.add_view( + "detailed", + [ + "populated", + "elements", + ], + include_keys_from="summary", + ) def add_serializers(self): super().add_serializers() @@ -201,14 +216,14 @@ class DCASerializer(base.ModelSerializer): self.dc_serializer = DCSerializer(self.app) # then set the serializers to point to it for those attrs collection_keys = [ - 'create_time', - 'update_time', - 'collection_type', - 'populated', - 'populated_state', - 'populated_state_message', - 'elements', - 'element_count', + "create_time", + "update_time", + "collection_type", + "populated", + "populated_state", + "populated_state_message", + "elements", + "element_count", ] for key in collection_keys: self.serializers[key] = self._proxy_to_dataset_collection(key=key) @@ -221,13 +236,10 @@ class DCASerializer(base.ModelSerializer): return lambda i, k, **c: self.dc_serializer.serialize(i.collection, [k], **c)[k] if serializer: return lambda i, k, **c: serializer(i.collection, key or k, **c) - raise TypeError('kwarg serializer or key needed') + raise TypeError("kwarg serializer or key needed") -class HDCASerializer( - DCASerializer, - taggable.TaggableSerializerMixin, - annotatable.AnnotatableSerializerMixin): +class HDCASerializer(DCASerializer, taggable.TaggableSerializerMixin, annotatable.AnnotatableSerializerMixin): """ Serializer for HistoryDatasetCollectionAssociations. """ @@ -236,87 +248,92 @@ class HDCASerializer( super().__init__(app) self.hdca_manager = HDCAManager(app) - self.default_view = 'summary' - self.add_view('summary', [ - 'id', - 'type_id', - 'name', - 'history_id', 'hid', - 'history_content_type', - - 'collection_type', - 'populated_state', - 'populated_state_message', - 'element_count', - - 'job_source_id', - 'job_source_type', - - 'name', - 'type_id', - 'deleted', - # 'purged', - 'visible', - 'type', 'url', - 'create_time', 'update_time', - 'tags', # TODO: detail view only (maybe), - 'contents_url' - ]) - self.add_view('detailed', [ - 'populated', - 'elements' - ], include_keys_from='summary') + self.default_view = "summary" + self.add_view( + "summary", + [ + "id", + "type_id", + "name", + "history_id", + "hid", + "history_content_type", + "collection_type", + "populated_state", + "populated_state_message", + "element_count", + "job_source_id", + "job_source_type", + "name", + "type_id", + "deleted", + # 'purged', + "visible", + "type", + "url", + "create_time", + "update_time", + "tags", # TODO: detail view only (maybe), + "contents_url", + ], + ) + self.add_view("detailed", ["populated", "elements"], include_keys_from="summary") # fields for new beta web client, there is no summary/detailed split any more - self.add_view('betawebclient', [ - # common to hda - 'create_time', - 'deleted', - 'hid', - 'history_content_type', - 'history_id', - 'id', - 'name', - 'tags', - 'type', - 'type_id', - 'update_time', - 'url', - 'visible', - # hdca only - 'collection_id', - 'collection_type', - 'contents_url', - 'element_count', - 'job_source_id', - 'job_source_type', - 'job_state_summary', - 'populated', - 'populated_state', - 'populated_state_message', - 'elements_datatypes', - ]) + self.add_view( + "betawebclient", + [ + # common to hda + "create_time", + "deleted", + "hid", + "history_content_type", + "history_id", + "id", + "name", + "tags", + "type", + "type_id", + "update_time", + "url", + "visible", + # hdca only + "collection_id", + "collection_type", + "contents_url", + "element_count", + "job_source_id", + "job_source_type", + "job_state_summary", + "populated", + "populated_state", + "populated_state_message", + "elements_datatypes", + ], + ) def add_serializers(self): super().add_serializers() taggable.TaggableSerializerMixin.add_serializers(self) annotatable.AnnotatableSerializerMixin.add_serializers(self) serializers: Dict[str, base.Serializer] = { - 'model_class': lambda item, key, **context: self.hdca_manager.model_class.__class__.__name__, + "model_class": lambda item, key, **context: self.hdca_manager.model_class.__class__.__name__, # TODO: remove - 'type': lambda item, key, **context: 'collection', + "type": lambda item, key, **context: "collection", # part of a history and container - 'history_id': self.serialize_id, - 'history_content_type': lambda item, key, **context: self.hdca_manager.model_class.content_type, - 'type_id': self.serialize_type_id, - 'job_source_id': self.serialize_id, - 'url': lambda item, key, **context: self.url_for('history_content_typed', - history_id=self.app.security.encode_id(item.history_id), - id=self.app.security.encode_id(item.id), - type=self.hdca_manager.model_class.content_type), - 'contents_url': self.generate_contents_url, - 'job_state_summary': self.serialize_job_state_summary, - 'elements_datatypes': self.serialize_elements_datatypes, + "history_id": self.serialize_id, + "history_content_type": lambda item, key, **context: self.hdca_manager.model_class.content_type, + "type_id": self.serialize_type_id, + "job_source_id": self.serialize_id, + "url": lambda item, key, **context: self.url_for( + "history_content_typed", + history_id=self.app.security.encode_id(item.history_id), + id=self.app.security.encode_id(item.id), + type=self.hdca_manager.model_class.content_type, + ), + "contents_url": self.generate_contents_url, + "job_state_summary": self.serialize_job_state_summary, + "elements_datatypes": self.serialize_elements_datatypes, } self.serializers.update(serializers) @@ -324,15 +341,15 @@ class HDCASerializer( encode_id = self.app.security.encode_id trans = context.get("trans") url_for = trans.url_builder if trans and trans.url_builder else self.url_for - contents_url = url_for('contents_dataset_collection', - hdca_id=encode_id(item.id), - parent_id=encode_id(item.collection_id)) + contents_url = url_for( + "contents_dataset_collection", hdca_id=encode_id(item.id), parent_id=encode_id(item.collection_id) + ) return contents_url def serialize_job_state_summary(self, item, key, **context): states = item.job_state_summary.__dict__.copy() - del states['_sa_instance_state'] - del states['hdca_id'] + del states["_sa_instance_state"] + del states["hdca_id"] return states def serialize_elements_datatypes(self, item, key, **context): diff --git a/lib/galaxy/managers/histories.py b/lib/galaxy/managers/histories.py index 098a91a044f..59ce2a84bc8 100644 --- a/lib/galaxy/managers/histories.py +++ b/lib/galaxy/managers/histories.py @@ -19,15 +19,13 @@ from sqlalchemy import ( desc, ) -from galaxy import ( - exceptions as glx_exceptions, - model -) +from galaxy import exceptions as glx_exceptions +from galaxy import model from galaxy.managers import ( deletable, hdas, history_contents, - sharable + sharable, ) from galaxy.managers.base import ( Serializer, @@ -45,7 +43,7 @@ log = logging.getLogger(__name__) class HistoryManager(sharable.SharableModelManager, deletable.PurgableManagerMixin, SortableManager): model_class = model.History - foreign_key_name = 'history' + foreign_key_name = "history" user_share_model = model.HistoryUserShareAssociation tag_assoc = model.HistoryTagAssociation @@ -54,7 +52,13 @@ class HistoryManager(sharable.SharableModelManager, deletable.PurgableManagerMix # TODO: incorporate imp/exp (or alias to) - def __init__(self, app: MinimalManagerApp, hda_manager: hdas.HDAManager, contents_manager: history_contents.HistoryContentsManager, contents_filters: history_contents.HistoryContentsFilters): + def __init__( + self, + app: MinimalManagerApp, + hda_manager: hdas.HDAManager, + contents_manager: history_contents.HistoryContentsManager, + contents_filters: history_contents.HistoryContentsFilters, + ): super().__init__(app) self.hda_manager = hda_manager self.contents_manager = contents_manager @@ -149,28 +153,29 @@ class HistoryManager(sharable.SharableModelManager, deletable.PurgableManagerMix """Return an ORM compatible order_by using the given string""" # TODO: generalize into class # TODO: general (enough) columns - if order_by_string in ('create_time', 'create_time-dsc'): + if order_by_string in ("create_time", "create_time-dsc"): return desc(self.model_class.create_time) - if order_by_string == 'create_time-asc': + if order_by_string == "create_time-asc": return asc(self.model_class.create_time) - if order_by_string in ('update_time', 'update_time-dsc'): + if order_by_string in ("update_time", "update_time-dsc"): return desc(self.model_class.update_time) - if order_by_string == 'update_time-asc': + if order_by_string == "update_time-asc": return asc(self.model_class.update_time) - if order_by_string in ('name', 'name-asc'): + if order_by_string in ("name", "name-asc"): return asc(self.model_class.name) - if order_by_string == 'name-dsc': + if order_by_string == "name-dsc": return desc(self.model_class.name) # TODO: history columns - if order_by_string in ('size', 'size-dsc'): + if order_by_string in ("size", "size-dsc"): return desc(self.model_class.disk_size) - if order_by_string == 'size-asc': + if order_by_string == "size-asc": return asc(self.model_class.disk_size) # TODO: add functional/non-orm orders (such as rating) if default: return self.parse_order_by(default) - raise glx_exceptions.RequestParameterInvalidException('Unkown order_by', order_by=order_by_string, - available=['create_time', 'update_time', 'name', 'size']) + raise glx_exceptions.RequestParameterInvalidException( + "Unkown order_by", order_by=order_by_string, available=["create_time", "update_time", "name", "size"] + ) def non_ready_jobs(self, history): """Return the currently running job objects associated with this history. @@ -180,15 +185,18 @@ class HistoryManager(sharable.SharableModelManager, deletable.PurgableManagerMix """ # TODO: defer to jobModelManager (if there was one) # TODO: genericize the params to allow other filters - jobs = (self.session().query(model.Job) + jobs = ( + self.session() + .query(model.Job) .filter(model.Job.history == history) - .filter(model.Job.state.in_(model.Job.non_ready_states))) + .filter(model.Job.state.in_(model.Job.non_ready_states)) + ) return jobs def queue_history_import(self, trans, archive_type, archive_source): # Run job to do import. - history_imp_tool = trans.app.toolbox.get_tool('__IMPORT_HISTORY__') - incoming = {'__ARCHIVE_SOURCE__': archive_source, '__ARCHIVE_TYPE__': archive_type} + history_imp_tool = trans.app.toolbox.get_tool("__IMPORT_HISTORY__") + incoming = {"__ARCHIVE_SOURCE__": archive_source, "__ARCHIVE_TYPE__": archive_type} job, *_ = history_imp_tool.execute(trans, incoming=incoming) trans.app.job_manager.enqueue(job, tool=history_imp_tool) return job @@ -197,13 +205,13 @@ class HistoryManager(sharable.SharableModelManager, deletable.PurgableManagerMix def legacy_serve_ready_history_export(self, trans, jeha): assert jeha.ready if jeha.compressed: - trans.response.set_content_type('application/x-gzip') + trans.response.set_content_type("application/x-gzip") else: - trans.response.set_content_type('application/x-tar') + trans.response.set_content_type("application/x-tar") disposition = f'attachment; filename="{jeha.export_name}"' trans.response.headers["Content-Disposition"] = disposition archive = trans.app.object_store.get_filename(jeha.dataset) - return open(archive, mode='rb') + return open(archive, mode="rb") def get_ready_history_export_file_path(self, trans, jeha) -> str: """ @@ -213,28 +221,30 @@ class HistoryManager(sharable.SharableModelManager, deletable.PurgableManagerMix assert jeha.ready return trans.app.object_store.get_filename(jeha.dataset) - def queue_history_export(self, trans, history, gzip=True, include_hidden=False, include_deleted=False, directory_uri=None, file_name=None): + def queue_history_export( + self, trans, history, gzip=True, include_hidden=False, include_deleted=False, directory_uri=None, file_name=None + ): # Convert options to booleans. if isinstance(gzip, str): - gzip = (gzip in ['True', 'true', 'T', 't']) + gzip = gzip in ["True", "true", "T", "t"] if isinstance(include_hidden, str): - include_hidden = (include_hidden in ['True', 'true', 'T', 't']) + include_hidden = include_hidden in ["True", "true", "T", "t"] if isinstance(include_deleted, str): - include_deleted = (include_deleted in ['True', 'true', 'T', 't']) + include_deleted = include_deleted in ["True", "true", "T", "t"] params = { - 'history_to_export': history, - 'compress': gzip, - 'include_hidden': include_hidden, - 'include_deleted': include_deleted + "history_to_export": history, + "compress": gzip, + "include_hidden": include_hidden, + "include_deleted": include_deleted, } if directory_uri is None: - export_tool_id = '__EXPORT_HISTORY__' + export_tool_id = "__EXPORT_HISTORY__" else: - params['directory_uri'] = directory_uri - params['file_name'] = file_name or None - export_tool_id = '__EXPORT_HISTORY_TO_URI__' + params["directory_uri"] = directory_uri + params["file_name"] = file_name or None + export_tool_id = "__EXPORT_HISTORY_TO_URI__" # Run job to do export. history_exp_tool = trans.app.toolbox.get_tool(export_tool_id) @@ -291,21 +301,28 @@ class HistoryManager(sharable.SharableModelManager, deletable.PurgableManagerMix extra.cannot_change = list(cannot_change_dict.values()) extra.accessible_count = total_dataset_count - len(extra.can_change) - len(extra.cannot_change) if not extra.accessible_count and not extra.can_change and not share_anyway: - errors.add("The history you are sharing do not contain any datasets that can be accessed by the users with which you are sharing.") + errors.add( + "The history you are sharing do not contain any datasets that can be accessed by the users with which you are sharing." + ) extra.can_share = not errors and (extra.accessible_count == total_dataset_count or option is not None) return extra def is_history_shared_with(self, history, user) -> bool: - return bool(self.session().query(self.user_share_model).filter( - and_( - self.user_share_model.table.c.user_id == user.id, - self.user_share_model.table.c.history_id == history.id, + return bool( + self.session() + .query(self.user_share_model) + .filter( + and_( + self.user_share_model.table.c.user_id == user.id, + self.user_share_model.table.c.history_id == history.id, + ) ) - ).first()) + .first() + ) def make_members_public(self, trans, item): - """ Make the non-purged datasets in history public. + """Make the non-purged datasets in history public. Performs permissions check. """ for hda in item.activatable_datasets: @@ -321,7 +338,6 @@ class HistoryManager(sharable.SharableModelManager, deletable.PurgableManagerMix class HistoryExportView: - def __init__(self, app: MinimalManagerApp): self.app = app @@ -335,7 +351,9 @@ class HistoryExportView: encoded_jeha_id = trans.security.encode_id(jeha.id) api_url = trans.url_builder("history_archive_download", id=history_id, jeha_id=encoded_jeha_id) external_url = trans.url_builder("history_archive_download", id=history_id, jeha_id="latest", qualified=True) - external_permanent_url = trans.url_builder("history_archive_download", id=history_id, jeha_id=encoded_jeha_id, qualified=True) + external_permanent_url = trans.url_builder( + "history_archive_download", id=history_id, jeha_id=encoded_jeha_id, qualified=True + ) rval["download_url"] = api_url rval["external_download_latest_url"] = external_url rval["external_download_permanent_url"] = external_permanent_url @@ -359,7 +377,9 @@ class HistoryExportView: def _history(self, trans, history_id): if history_id is not None: - history = self.app.history_manager.get_accessible(trans.security.decode_id(history_id), trans.user, current_history=trans.history) + history = self.app.history_manager.get_accessible( + trans.security.decode_id(history_id), trans.user, current_history=trans.history + ) else: history = trans.history return history @@ -369,10 +389,17 @@ class HistorySerializer(sharable.SharableModelSerializer, deletable.PurgableSeri """ Interface/service object for serializing histories into dictionaries. """ - model_manager_class = HistoryManager - SINGLE_CHAR_ABBR = 'h' - def __init__(self, app: MinimalManagerApp, hda_manager: hdas.HDAManager, hda_serializer: hdas.HDASerializer, history_contents_serializer: history_contents.HistoryContentsSerializer): + model_manager_class = HistoryManager + SINGLE_CHAR_ABBR = "h" + + def __init__( + self, + app: MinimalManagerApp, + hda_manager: hdas.HDAManager, + hda_serializer: hdas.HDASerializer, + history_contents_serializer: history_contents.HistoryContentsSerializer, + ): super().__init__(app) self.history_manager = self.manager @@ -380,83 +407,97 @@ class HistorySerializer(sharable.SharableModelSerializer, deletable.PurgableSeri self.hda_serializer = hda_serializer self.history_contents_serializer = history_contents_serializer - self.default_view = 'summary' - self.add_view('summary', [ - 'id', - 'model_class', - 'name', - 'deleted', - 'purged', - # 'count' - 'url', - # TODO: why these? - 'published', - 'annotation', - 'tags', - 'update_time', - ]) - self.add_view('detailed', [ - 'contents_url', - 'empty', - 'size', - 'user_id', - 'create_time', - 'update_time', - 'importable', - 'slug', - 'username_and_slug', - 'genome_build', - # TODO: remove the next three - instead getting the same info from the 'hdas' list - 'state', - 'state_details', - 'state_ids', - # 'community_rating', - # 'user_rating', - ], include_keys_from='summary') + self.default_view = "summary" + self.add_view( + "summary", + [ + "id", + "model_class", + "name", + "deleted", + "purged", + # 'count' + "url", + # TODO: why these? + "published", + "annotation", + "tags", + "update_time", + ], + ) + self.add_view( + "detailed", + [ + "contents_url", + "empty", + "size", + "user_id", + "create_time", + "update_time", + "importable", + "slug", + "username_and_slug", + "genome_build", + # TODO: remove the next three - instead getting the same info from the 'hdas' list + "state", + "state_details", + "state_ids", + # 'community_rating', + # 'user_rating', + ], + include_keys_from="summary", + ) # in the Historys' case, each of these views includes the keys from the previous #: ..note: this is a custom view for newer (2016/3) UI and should be considered volatile - self.add_view('dev-detailed', [ - 'contents_url', - 'size', - 'user_id', - 'create_time', - 'update_time', - 'importable', - 'slug', - 'username_and_slug', - 'genome_build', - # 'contents_states', - 'contents_active', - 'hid_counter', - ], include_keys_from='summary') + self.add_view( + "dev-detailed", + [ + "contents_url", + "size", + "user_id", + "create_time", + "update_time", + "importable", + "slug", + "username_and_slug", + "genome_build", + # 'contents_states', + "contents_active", + "hid_counter", + ], + include_keys_from="summary", + ) # beta web client fields, no summary/detailed/dev-detailed blah - self.add_view('betawebclient', [ - 'annotation', - 'contents_active', - 'contents_url', - 'create_time', - 'deleted', - 'empty', - 'genome_build', - 'hid_counter', - 'id', - 'importable', - 'name', - 'nice_size', - 'published', - 'purged', - # 'shared', - 'size', - 'slug', - 'state', - 'tags', - 'update_time', - 'url', - 'username_and_slug', - 'user_id', - ]) + self.add_view( + "betawebclient", + [ + "annotation", + "contents_active", + "contents_url", + "create_time", + "deleted", + "empty", + "genome_build", + "hid_counter", + "id", + "importable", + "name", + "nice_size", + "published", + "purged", + # 'shared', + "size", + "slug", + "state", + "tags", + "update_time", + "url", + "username_and_slug", + "user_id", + ], + ) # assumes: outgoing to json.dumps and sanitized def add_serializers(self): @@ -464,29 +505,30 @@ class HistorySerializer(sharable.SharableModelSerializer, deletable.PurgableSeri deletable.PurgableSerializerMixin.add_serializers(self) serializers: Dict[str, Serializer] = { - 'model_class': lambda item, key, **context: 'History', - 'size': lambda item, key, **context: int(item.disk_size), - 'nice_size': lambda item, key, **context: item.disk_nice_size, - 'state': self.serialize_history_state, - - 'url': lambda item, key, **context: self.url_for('history', id=self.app.security.encode_id(item.id)), - 'contents_url': lambda item, key, **context: self.url_for('history_contents', - history_id=self.app.security.encode_id(item.id)), - - 'empty': lambda item, key, **context: (len(item.datasets) + len(item.dataset_collections)) <= 0, - 'count': lambda item, key, **context: len(item.datasets), - 'hdas': lambda item, key, **context: [self.app.security.encode_id(hda.id) for hda in item.datasets], - 'state_details': self.serialize_state_counts, - 'state_ids': self.serialize_state_ids, - 'contents': self.serialize_contents, - 'non_ready_jobs': lambda item, key, **context: [self.app.security.encode_id(job.id) for job - in self.manager.non_ready_jobs(item)], - - 'contents_states': self.serialize_contents_states, - 'contents_active': self.serialize_contents_active, + "model_class": lambda item, key, **context: "History", + "size": lambda item, key, **context: int(item.disk_size), + "nice_size": lambda item, key, **context: item.disk_nice_size, + "state": self.serialize_history_state, + "url": lambda item, key, **context: self.url_for("history", id=self.app.security.encode_id(item.id)), + "contents_url": lambda item, key, **context: self.url_for( + "history_contents", history_id=self.app.security.encode_id(item.id) + ), + "empty": lambda item, key, **context: (len(item.datasets) + len(item.dataset_collections)) <= 0, + "count": lambda item, key, **context: len(item.datasets), + "hdas": lambda item, key, **context: [self.app.security.encode_id(hda.id) for hda in item.datasets], + "state_details": self.serialize_state_counts, + "state_ids": self.serialize_state_ids, + "contents": self.serialize_contents, + "non_ready_jobs": lambda item, key, **context: [ + self.app.security.encode_id(job.id) for job in self.manager.non_ready_jobs(item) + ], + "contents_states": self.serialize_contents_states, + "contents_active": self.serialize_contents_active, # TODO: Use base manager's serialize_id for user_id (and others) # after refactoring hierarchy here? - 'user_id': lambda item, key, **context: self.app.security.encode_id(item.user_id) if item.user_id is not None else None + "user_id": lambda item, key, **context: self.app.security.encode_id(item.user_id) + if item.user_id is not None + else None, } self.serializers.update(serializers) @@ -540,21 +582,21 @@ class HistorySerializer(sharable.SharableModelSerializer, deletable.PurgableSeri state = states.ERROR # TODO: history_state and state_counts are classically calc'd at the same time # so this is rel. ineff. - if we keep this... - hda_state_counts = self.serialize_state_counts(history, 'counts', exclude_deleted=True, **context) + hda_state_counts = self.serialize_state_counts(history, "counts", exclude_deleted=True, **context) if history.empty: state = states.NEW else: num_hdas = sum(hda_state_counts.values()) - if (hda_state_counts[states.RUNNING] > 0 - or hda_state_counts[states.SETTING_METADATA] > 0 - or hda_state_counts[states.UPLOAD] > 0): + if ( + hda_state_counts[states.RUNNING] > 0 + or hda_state_counts[states.SETTING_METADATA] > 0 + or hda_state_counts[states.UPLOAD] > 0 + ): state = states.RUNNING # TODO: this method may be more useful if we *also* polled the histories jobs here too - elif (hda_state_counts[states.QUEUED] > 0 - or hda_state_counts[states.NEW] > 0): + elif hda_state_counts[states.QUEUED] > 0 or hda_state_counts[states.NEW] > 0: state = states.QUEUED - elif (hda_state_counts[states.ERROR] > 0 - or hda_state_counts[states.FAILED_METADATA] > 0): + elif hda_state_counts[states.ERROR] > 0 or hda_state_counts[states.FAILED_METADATA] > 0: state = states.ERROR elif hda_state_counts[states.OK] == num_hdas: state = states.OK @@ -565,8 +607,9 @@ class HistorySerializer(sharable.SharableModelSerializer, deletable.PurgableSeri history = item returned = [] for content in self.manager.contents_manager._union_of_contents_query(history).all(): - serialized = self.history_contents_serializer.serialize_to_view(content, - view='summary', trans=trans, user=user) + serialized = self.history_contents_serializer.serialize_to_view( + content, view="summary", trans=trans, user=user + ) returned.append(serialized) return returned @@ -596,6 +639,7 @@ class HistoryDeserializer(sharable.SharableModelDeserializer, deletable.Purgable """ Interface/service object for validating and deserializing dictionaries into histories. """ + model_manager_class = HistoryManager def __init__(self, app: MinimalManagerApp): @@ -606,10 +650,12 @@ class HistoryDeserializer(sharable.SharableModelDeserializer, deletable.Purgable super().add_deserializers() deletable.PurgableDeserializerMixin.add_deserializers(self) - self.deserializers.update({ - 'name': self.deserialize_basestring, - 'genome_build': self.deserialize_genome_build, - }) + self.deserializers.update( + { + "name": self.deserialize_basestring, + "genome_build": self.deserialize_genome_build, + } + ) class HistoryFilters(sharable.SharableModelFilters, deletable.PurgableFiltersMixin): @@ -619,10 +665,12 @@ class HistoryFilters(sharable.SharableModelFilters, deletable.PurgableFiltersMix def _add_parsers(self): super()._add_parsers() deletable.PurgableFiltersMixin._add_parsers(self) - self.orm_filter_parsers.update({ - # history specific - 'name': {'op': ('eq', 'contains', 'like')}, - 'genome_build': {'op': ('eq', 'contains', 'like')}, - 'create_time': {'op': ('le', 'ge', 'gt', 'lt'), 'val': self.parse_date}, - 'update_time': {'op': ('le', 'ge', 'gt', 'lt'), 'val': self.parse_date}, - }) + self.orm_filter_parsers.update( + { + # history specific + "name": {"op": ("eq", "contains", "like")}, + "genome_build": {"op": ("eq", "contains", "like")}, + "create_time": {"op": ("le", "ge", "gt", "lt"), "val": self.parse_date}, + "update_time": {"op": ("le", "ge", "gt", "lt"), "val": self.parse_date}, + } + ) diff --git a/lib/galaxy/managers/history_contents.py b/lib/galaxy/managers/history_contents.py index 5c6e6b8160d..63ff7ae385c 100644 --- a/lib/galaxy/managers/history_contents.py +++ b/lib/galaxy/managers/history_contents.py @@ -3,7 +3,11 @@ Heterogenous lists/contents are difficult to query properly since unions are not easily made. """ import logging -from typing import Any, Dict, List +from typing import ( + Any, + Dict, + List, +) from sqlalchemy import ( asc, @@ -12,17 +16,15 @@ from sqlalchemy import ( func, literal, sql, - true + true, ) from sqlalchemy.orm import ( eagerload, - undefer + undefer, ) -from galaxy import ( - exceptions as glx_exceptions, - model -) +from galaxy import exceptions as glx_exceptions +from galaxy import model from galaxy.managers import ( annotatable, base, @@ -30,7 +32,7 @@ from galaxy.managers import ( hdas, hdcas, taggable, - tools + tools, ) from galaxy.structured_app import MinimalManagerApp from .base import ( @@ -50,11 +52,11 @@ class HistoryContentsManager(base.SortableManager): contained_class = model.HistoryDatasetAssociation contained_class_manager_class = hdas.HDAManager - contained_class_type_name = 'dataset' + contained_class_type_name = "dataset" subcontainer_class = model.HistoryDatasetCollectionAssociation subcontainer_class_manager_class = hdcas.HDCAManager - subcontainer_class_type_name = 'dataset_collection' + subcontainer_class_type_name = "dataset_collection" #: the columns which are common to both subcontainers and non-subcontainers. # (Also the attributes that may be filtered or orderered_by) @@ -76,7 +78,7 @@ class HistoryContentsManager(base.SortableManager): "create_time", "update_time", ) - default_order_by = 'hid' + default_order_by = "hid" def __init__(self, app: MinimalManagerApp): self.app = app @@ -108,38 +110,42 @@ class HistoryContentsManager(base.SortableManager): """ # TODO?: we could branch here based on 'if limit is None and offset is None' - to a simpler (non-union) query # for now, I'm just using this (even for non-limited/offset queries) to reduce code paths - return self._union_of_contents(container, - filters=filters, limit=limit, offset=offset, order_by=order_by, **kwargs) + return self._union_of_contents( + container, filters=filters, limit=limit, offset=offset, order_by=order_by, **kwargs + ) def contents_count(self, container, filters=None, limit=None, offset=None, order_by=None, **kwargs): """ Returns a count of both/all types of contents, based on the given filters. """ - return self.contents_query(container, - filters=filters, limit=limit, offset=offset, order_by=order_by, **kwargs).count() + return self.contents_query( + container, filters=filters, limit=limit, offset=offset, order_by=order_by, **kwargs + ).count() def contents_query(self, container, filters=None, limit=None, offset=None, order_by=None, **kwargs): """ Returns the contents union query for subqueries, etc. """ - return self._union_of_contents_query(container, - filters=filters, limit=limit, offset=offset, order_by=order_by, **kwargs) + return self._union_of_contents_query( + container, filters=filters, limit=limit, offset=offset, order_by=order_by, **kwargs + ) # order_by parsing - similar to FilterParser but not enough yet to warrant a class? def parse_order_by(self, order_by_string, default=None): """Return an ORM compatible order_by using the given string""" - available = ['create_time', 'extension', 'hid', 'history_id', 'name', 'update_time'] + available = ["create_time", "extension", "hid", "history_id", "name", "update_time"] for attribute in available: - attribute_dsc = f'{attribute}-dsc' - attribute_asc = f'{attribute}-asc' + attribute_dsc = f"{attribute}-dsc" + attribute_asc = f"{attribute}-asc" if order_by_string in (attribute, attribute_dsc): return desc(attribute) if order_by_string == attribute_asc: return asc(attribute) if default: return self.parse_order_by(default) - raise glx_exceptions.RequestParameterInvalidException('Unknown order_by', order_by=order_by_string, - available=available) + raise glx_exceptions.RequestParameterInvalidException( + "Unknown order_by", order_by=order_by_string, available=available + ) # history specific methods def state_counts(self, history): @@ -150,13 +156,15 @@ class HistoryContentsManager(base.SortableManager): Note: does not include deleted/hidden contents. """ filters = [ - base.ModelFilterParser.parsed_filter("orm", sql.column('deleted') == false()), - base.ModelFilterParser.parsed_filter("orm", sql.column('visible') == true()) + base.ModelFilterParser.parsed_filter("orm", sql.column("deleted") == false()), + base.ModelFilterParser.parsed_filter("orm", sql.column("visible") == true()), ] contents_subquery = self._union_of_contents_query(history, filters=filters).subquery() - statement = (sql.select([sql.column('state'), func.count('*')]) + statement = ( + sql.select([sql.column("state"), func.count("*")]) .select_from(contents_subquery) - .group_by(sql.column('state'))) + .group_by(sql.column("state")) + ) counts = self.app.model.context.execute(statement).fetchall() return dict(counts) @@ -170,22 +178,18 @@ class HistoryContentsManager(base.SortableManager): """ returned = dict(deleted=0, hidden=0, active=0) contents_subquery = self._union_of_contents_query(history).subquery() - columns = [ - sql.column('deleted'), - sql.column('visible'), - func.count('*') - ] - statement = (sql.select(columns) - .select_from(contents_subquery) - .group_by(sql.column('deleted'), sql.column('visible'))) + columns = [sql.column("deleted"), sql.column("visible"), func.count("*")] + statement = ( + sql.select(columns).select_from(contents_subquery).group_by(sql.column("deleted"), sql.column("visible")) + ) groups = self.app.model.context.execute(statement).fetchall() for deleted, visible, count in groups: if deleted: - returned['deleted'] += count + returned["deleted"] += count if not visible: - returned['hidden'] += count + returned["hidden"] += count if not deleted and visible: - returned['active'] += count + returned["active"] += count return returned def map_datasets(self, history, fn, **kwargs): @@ -229,25 +233,29 @@ class HistoryContentsManager(base.SortableManager): return contents_results # partition ids into a map of { component_class names -> list of ids } from the above union query - id_map: Dict[str, List[int]] = dict([(self.contained_class_type_name, []), (self.subcontainer_class_type_name, [])]) + id_map: Dict[str, List[int]] = dict( + [(self.contained_class_type_name, []), (self.subcontainer_class_type_name, [])] + ) for result in contents_results: result_type = self._get_union_type(result) contents_id = self._get_union_id(result) if result_type in id_map: id_map[result_type].append(contents_id) else: - raise TypeError('Unknown contents type:', result_type) + raise TypeError("Unknown contents type:", result_type) # query 2 & 3: use the ids to query each component_class, returning an id->full component model map contained_ids = id_map[self.contained_class_type_name] id_map[self.contained_class_type_name] = self._contained_id_map(contained_ids) subcontainer_ids = id_map[self.subcontainer_class_type_name] - serialization_params = kwargs.get('serialization_params', None) - id_map[self.subcontainer_class_type_name] = self._subcontainer_id_map(subcontainer_ids, serialization_params=serialization_params) + serialization_params = kwargs.get("serialization_params", None) + id_map[self.subcontainer_class_type_name] = self._subcontainer_id_map( + subcontainer_ids, serialization_params=serialization_params + ) # cycle back over the union query to create an ordered list of the objects returned in queries 2 & 3 above contents = [] - filters = kwargs.get('filters') or [] + filters = kwargs.get("filters") or [] # TODO: or as generator? for result in contents_results: result_type = self._get_union_type(result) @@ -260,25 +268,20 @@ class HistoryContentsManager(base.SortableManager): @staticmethod def passes_filters(content, filters): for filter_fn in filters: - if filter_fn.filter_type == 'function': + if filter_fn.filter_type == "function": if not filter_fn.filter(content): return False return True - def _union_of_contents_query(self, - container, - filters=None, - limit=None, - offset=None, - order_by=None, - user_id=None, - **kwargs): + def _union_of_contents_query( + self, container, filters=None, limit=None, offset=None, order_by=None, user_id=None, **kwargs + ): """ Returns a query for a limited and offset list of both types of contents, filtered and in some order. """ order_by = order_by if order_by is not None else self.default_order_by - order_by = order_by if isinstance(order_by, (tuple, list)) else (order_by, ) + order_by = order_by if isinstance(order_by, (tuple, list)) else (order_by,) # TODO: 3 queries and 3 iterations over results - this is undoubtedly better solved in the actual SQL layer # via one common table for contents, Some Yonder Resplendent and Fanciful Join, or ORM functionality @@ -290,10 +293,12 @@ class HistoryContentsManager(base.SortableManager): # note: I'm trying to keep these private functions as generic as possible in order to move them toward base later # query 1: create a union of common columns for which the component_classes can be filtered/limited - contained_query = self._contents_common_query_for_contained(history_id=container.id if container else None, - user_id=user_id) - subcontainer_query = self._contents_common_query_for_subcontainer(history_id=container.id if container else None, - user_id=user_id) + contained_query = self._contents_common_query_for_contained( + history_id=container.id if container else None, user_id=user_id + ) + subcontainer_query = self._contents_common_query_for_subcontainer( + history_id=container.id if container else None, user_id=user_id + ) filters = filters or [] # Apply filters that are specific to a model @@ -316,8 +321,8 @@ class HistoryContentsManager(base.SortableManager): def _apply_orm_filter(self, qry, orm_filter): if isinstance(orm_filter, sql.elements.BinaryExpression): - for match in filter(lambda col: col['name'] == orm_filter.left.name, qry.column_descriptions): - column = match['expr'] + for match in filter(lambda col: col["name"] == orm_filter.left.name, qry.column_descriptions): + column = match["expr"] new_filter = orm_filter._clone() new_filter.left = column qry = qry.filter(new_filter) @@ -340,11 +345,12 @@ class HistoryContentsManager(base.SortableManager): def _contents_common_query_for_contained(self, history_id, user_id): component_class = self.contained_class # TODO: and now a join with Dataset - this is getting sad - columns = self._contents_common_columns(component_class, - history_content_type=literal('dataset'), + columns = self._contents_common_columns( + component_class, + history_content_type=literal("dataset"), state=model.Dataset.state, # do not have inner collections - collection_id=literal(None) + collection_id=literal(None), ) subquery = self._session().query(*columns) # for the HDA's we need to join the Dataset since it has an actual state column @@ -355,14 +361,16 @@ class HistoryContentsManager(base.SortableManager): # Make sure we only return items that are user-accessible by checking that they are in a history # owned by the current user. # TODO: move into filter mixin, and implement accessible logic as SQL query - subquery = subquery.filter(component_class.table.c.history_id == model.History.table.c.id, - model.History.table.c.user_id == user_id) + subquery = subquery.filter( + component_class.table.c.history_id == model.History.table.c.id, model.History.table.c.user_id == user_id + ) return subquery def _contents_common_query_for_subcontainer(self, history_id, user_id): component_class = self.subcontainer_class - columns = self._contents_common_columns(component_class, - history_content_type=literal('dataset_collection'), + columns = self._contents_common_columns( + component_class, + history_content_type=literal("dataset_collection"), # do not have datasets dataset_id=literal(None), state=model.DatasetCollection.populated_state, @@ -372,13 +380,13 @@ class HistoryContentsManager(base.SortableManager): ) subquery = self._session().query(*columns) # for the HDCA's we need to join the DatasetCollection since it has the populated_state - subquery = subquery.join(model.DatasetCollection, - model.DatasetCollection.id == component_class.collection_id) + subquery = subquery.join(model.DatasetCollection, model.DatasetCollection.id == component_class.collection_id) if history_id: subquery = subquery.filter(component_class.history_id == history_id) else: - subquery = subquery.filter(component_class.history_id == model.History.table.c.id, - model.History.table.c.user_id == user_id) + subquery = subquery.filter( + component_class.history_id == model.History.table.c.id, model.History.table.c.user_id == user_id + ) return subquery def _get_union_type(self, union): @@ -394,12 +402,15 @@ class HistoryContentsManager(base.SortableManager): if not id_list: return [] component_class = self.contained_class - query = (self._session().query(component_class) + query = ( + self._session() + .query(component_class) .filter(component_class.id.in_(id_list)) - .options(undefer('_metadata')) - .options(eagerload('dataset.actions')) - .options(eagerload('tags')) - .options(eagerload('annotations'))) + .options(undefer("_metadata")) + .options(eagerload("dataset.actions")) + .options(eagerload("tags")) + .options(eagerload("annotations")) + ) return {row.id: row for row in query.all()} def _subcontainer_id_map(self, id_list, serialization_params=None): @@ -407,18 +418,21 @@ class HistoryContentsManager(base.SortableManager): if not id_list: return [] component_class = self.subcontainer_class - query = (self._session().query(component_class) + query = ( + self._session() + .query(component_class) .filter(component_class.id.in_(id_list)) - .options(eagerload('collection')) - .options(eagerload('tags')) - .options(eagerload('annotations'))) + .options(eagerload("collection")) + .options(eagerload("tags")) + .options(eagerload("annotations")) + ) # This will conditionally join a potentially costly job_state summary # All the paranoia if-checking makes me wonder if serialization_params # should really be a property of the manager class instance if serialization_params and serialization_params.keys: - if 'job_state_summary' in serialization_params.keys: - query = query.options(eagerload('job_state_summary')) + if "job_state_summary" in serialization_params.keys: + query = query.options(eagerload("job_state_summary")) return {row.id: row for row in query.all()} @@ -427,53 +441,59 @@ class HistoryContentsSerializer(base.ModelSerializer, deletable.PurgableSerializ """ Interface/service object for serializing histories into dictionaries. """ + model_manager_class = HistoryContentsManager def __init__(self, app: MinimalManagerApp, **kwargs): super().__init__(app, **kwargs) - self.default_view = 'summary' - self.add_view('summary', [ - "id", - "type_id", - "history_id", - "hid", - "history_content_type", - "visible", - "dataset_id", - "collection_id", - "name", - "state", - "deleted", - "purged", - "create_time", - "update_time", - ]) + self.default_view = "summary" + self.add_view( + "summary", + [ + "id", + "type_id", + "history_id", + "hid", + "history_content_type", + "visible", + "dataset_id", + "collection_id", + "name", + "state", + "deleted", + "purged", + "create_time", + "update_time", + ], + ) # assumes: outgoing to json.dumps and sanitized def add_serializers(self): super().add_serializers() deletable.PurgableSerializerMixin.add_serializers(self) serializers: Dict[str, Serializer] = { - 'type_id': self.serialize_type_id, - 'history_id': self.serialize_id, - 'dataset_id': self.serialize_id_or_skip, - 'collection_id': self.serialize_id_or_skip, + "type_id": self.serialize_type_id, + "history_id": self.serialize_id, + "dataset_id": self.serialize_id_or_skip, + "collection_id": self.serialize_id_or_skip, } self.serializers.update(serializers) def serialize_id_or_skip(self, item: Any, key: str, **context): """Serialize id or skip if attribute with `key` is not present.""" if not hasattr(item, key): - raise base.SkipAttribute('no such attribute') + raise base.SkipAttribute("no such attribute") return self.serialize_id(item, key, **context) -class HistoryContentsFilters(base.ModelFilterParser, - annotatable.AnnotatableFilterMixin, - deletable.PurgableFiltersMixin, - taggable.TaggableFilterMixin, - tools.ToolFilterMixin): +class HistoryContentsFilters( + base.ModelFilterParser, + annotatable.AnnotatableFilterMixin, + deletable.PurgableFiltersMixin, + taggable.TaggableFilterMixin, + tools.ToolFilterMixin, +): # surprisingly (but ominously), this works for both content classes in the union that's filtered model_class = model.HistoryDatasetAssociation @@ -485,54 +505,54 @@ class HistoryContentsFilters(base.ModelFilterParser, # TODO: genericize these - can probably extract a _get_column( attr, ... ) or something # special cases...special cases everywhere def get_filter(attr, op, val): - if attr == 'history_content_type' and op == 'eq': - if val in ('dataset', 'dataset_collection'): - return sql.column('history_content_type') == val - raise_filter_err(attr, op, val, 'bad op in filter') + if attr == "history_content_type" and op == "eq": + if val in ("dataset", "dataset_collection"): + return sql.column("history_content_type") == val + raise_filter_err(attr, op, val, "bad op in filter") - if attr == 'type_id': - if op == 'eq': - return sql.column('type_id') == val - if op == 'in': - return sql.column('type_id').in_(self.parse_type_id_list(val)) - raise_filter_err(attr, op, val, 'bad op in filter') + if attr == "type_id": + if op == "eq": + return sql.column("type_id") == val + if op == "in": + return sql.column("type_id").in_(self.parse_type_id_list(val)) + raise_filter_err(attr, op, val, "bad op in filter") - if attr in ('update_time', 'create_time'): - if op == 'ge': + if attr in ("update_time", "create_time"): + if op == "ge": return sql.column(attr) >= self.parse_date(val) - if op == 'le': + if op == "le": return sql.column(attr) <= self.parse_date(val) - if op == 'gt': + if op == "gt": return sql.column(attr) > self.parse_date(val) - if op == 'lt': + if op == "lt": return sql.column(attr) < self.parse_date(val) - raise_filter_err(attr, op, val, 'bad op in filter') + raise_filter_err(attr, op, val, "bad op in filter") - if attr == 'state': + if attr == "state": valid_states = model.Dataset.states.values() - if op == 'eq': + if op == "eq": if val not in valid_states: - raise_filter_err(attr, op, val, 'invalid state in filter') - return sql.column('state') == val - if op == 'in': - states = [s for s in val.split(',') if s] + raise_filter_err(attr, op, val, "invalid state in filter") + return sql.column("state") == val + if op == "in": + states = [s for s in val.split(",") if s] for state in states: if state not in valid_states: - raise_filter_err(attr, op, state, 'invalid state in filter') - return sql.column('state').in_(states) - raise_filter_err(attr, op, val, 'bad op in filter') + raise_filter_err(attr, op, state, "invalid state in filter") + return sql.column("state").in_(states) + raise_filter_err(attr, op, val, "bad op in filter") column_filter = get_filter(attr, op, val) if column_filter is not None: - return self.parsed_filter(filter_type='orm', filter=column_filter) + return self.parsed_filter(filter_type="orm", filter=column_filter) return super()._parse_orm_filter(attr, op, val) def decode_type_id(self, type_id): - TYPE_ID_SEP = '-' + TYPE_ID_SEP = "-" split = type_id.split(TYPE_ID_SEP, 1) return TYPE_ID_SEP.join((split[0], str(self.app.security.decode_id(split[1])))) - def parse_type_id_list(self, type_id_list_string, sep=','): + def parse_type_id_list(self, type_id_list_string, sep=","): """ Split `type_id_list_string` at `sep`. """ @@ -544,15 +564,17 @@ class HistoryContentsFilters(base.ModelFilterParser, deletable.PurgableFiltersMixin._add_parsers(self) taggable.TaggableFilterMixin._add_parsers(self) tools.ToolFilterMixin._add_parsers(self) - self.orm_filter_parsers.update({ - 'history_content_type': {'op': ('eq')}, - 'type_id': {'op': ('eq', 'in'), 'val': self.parse_type_id_list}, - 'hid': {'op': ('eq', 'ge', 'le', 'gt', 'lt'), 'val': int}, - # TODO: needs a different val parser - but no way to add to the above - # 'hid-in' : { 'op': ( 'in' ), 'val': self.parse_int_list }, - 'name': {'op': ('eq', 'contains', 'like')}, - 'state': {'op': ('eq', 'in')}, - 'visible': {'op': ('eq'), 'val': parse_bool}, - 'create_time': {'op': ('le', 'ge', 'lt', 'gt'), 'val': self.parse_date}, - 'update_time': {'op': ('le', 'ge', 'lt', 'gt'), 'val': self.parse_date}, - }) + self.orm_filter_parsers.update( + { + "history_content_type": {"op": ("eq")}, + "type_id": {"op": ("eq", "in"), "val": self.parse_type_id_list}, + "hid": {"op": ("eq", "ge", "le", "gt", "lt"), "val": int}, + # TODO: needs a different val parser - but no way to add to the above + # 'hid-in' : { 'op': ( 'in' ), 'val': self.parse_int_list }, + "name": {"op": ("eq", "contains", "like")}, + "state": {"op": ("eq", "in")}, + "visible": {"op": ("eq"), "val": parse_bool}, + "create_time": {"op": ("le", "ge", "lt", "gt"), "val": self.parse_date}, + "update_time": {"op": ("le", "ge", "lt", "gt"), "val": self.parse_date}, + } + ) diff --git a/lib/galaxy/managers/interactivetool.py b/lib/galaxy/managers/interactivetool.py index df3b83ab97c..35c1b51c451 100644 --- a/lib/galaxy/managers/interactivetool.py +++ b/lib/galaxy/managers/interactivetool.py @@ -3,21 +3,18 @@ import sqlite3 from sqlalchemy import or_ - from galaxy import ( exceptions, - model + model, ) from galaxy.util.filelock import FileLock - log = logging.getLogger(__name__) -DATABASE_TABLE_NAME = 'gxitproxy' +DATABASE_TABLE_NAME = "gxitproxy" class InteractiveToolSqlite: - def __init__(self, sqlite_filename, encode_id): self.sqlite_filename = sqlite_filename self.encode_id = encode_id @@ -27,22 +24,22 @@ class InteractiveToolSqlite: conn = sqlite3.connect(self.sqlite_filename) try: c = conn.cursor() - select = f'''SELECT token, host, port, info + select = f"""SELECT token, host, port, info FROM {DATABASE_TABLE_NAME} - WHERE key=? and key_type=?''' - c.execute(select, (key, key_type,)) + WHERE key=? and key_type=?""" + c.execute( + select, + ( + key, + key_type, + ), + ) try: token, host, port, info = c.fetchone() except TypeError: - log.warning('get(): invalid key: %s key_type %s', key, key_type) + log.warning("get(): invalid key: %s key_type %s", key, key_type) return None - return dict( - key=key, - key_type=key_type, - token=token, - host=host, - port=port, - info=info) + return dict(key=key, key_type=key_type, token=token, host=host, port=port, info=info) finally: conn.close() @@ -60,7 +57,8 @@ class InteractiveToolSqlite: c = conn.cursor() try: # Create table - c.execute('''CREATE TABLE %s + c.execute( + """CREATE TABLE %s (key text, key_type text, token text, @@ -68,22 +66,35 @@ class InteractiveToolSqlite: port integer, info text, PRIMARY KEY (key, key_type) - )''' % (DATABASE_TABLE_NAME)) + )""" + % (DATABASE_TABLE_NAME) + ) except Exception: pass - delete = f'''DELETE FROM {DATABASE_TABLE_NAME} WHERE key=? and key_type=?''' - c.execute(delete, (key, key_type,)) - insert = '''INSERT INTO %s + delete = f"""DELETE FROM {DATABASE_TABLE_NAME} WHERE key=? and key_type=?""" + c.execute( + delete, + ( + key, + key_type, + ), + ) + insert = """INSERT INTO %s (key, key_type, token, host, port, info) - VALUES (?, ?, ?, ?, ?, ?)''' % (DATABASE_TABLE_NAME) - c.execute(insert, - (key, - key_type, - token, - host, - port, - info, - )) + VALUES (?, ?, ?, ?, ?, ?)""" % ( + DATABASE_TABLE_NAME + ) + c.execute( + insert, + ( + key, + key_type, + token, + host, + port, + info, + ), + ) conn.commit() finally: conn.close() @@ -94,12 +105,12 @@ class InteractiveToolSqlite: with external resources. Remove entries that match all provided key=values """ assert kwd, ValueError("You must provide some values to key upon") - delete = f'DELETE FROM {DATABASE_TABLE_NAME} WHERE' + delete = f"DELETE FROM {DATABASE_TABLE_NAME} WHERE" value_list = [] for i, (key, value) in enumerate(kwd.items()): if i != 0: - delete += ' and' - delete += f' {key}=?' + delete += " and" + delete += f" {key}=?" value_list.append(value) with FileLock(self.sqlite_filename): conn = sqlite3.connect(self.sqlite_filename) @@ -110,19 +121,24 @@ class InteractiveToolSqlite: # NB: This does not invalidate in-memory caches used by uwsgi (if any) c.execute(delete, tuple(value_list)) except Exception as e: - log.debug('Error removing entry (%s): %s', delete, e) + log.debug("Error removing entry (%s): %s", delete, e) conn.commit() finally: conn.close() def save_entry_point(self, entry_point): - """Convenience method to easily save an entry_point. - """ - return self.save(self.encode_id(entry_point.id), entry_point.__class__.__name__.lower(), entry_point.token, entry_point.host, entry_point.port, None) + """Convenience method to easily save an entry_point.""" + return self.save( + self.encode_id(entry_point.id), + entry_point.__class__.__name__.lower(), + entry_point.token, + entry_point.host, + entry_point.port, + None, + ) def remove_entry_point(self, entry_point): - """Convenience method to easily remove an entry_point. - """ + """Convenience method to easily remove an entry_point.""" return self.remove(key=self.encode_id(entry_point.id), key_type=entry_point.__class__.__name__.lower()) @@ -142,13 +158,21 @@ class InteractiveToolManager: def create_entry_points(self, job, tool, entry_points=None, flush=True): entry_points = entry_points or tool.ports for entry in entry_points: - ep = self.model.InteractiveToolEntryPoint(job=job, tool_port=entry['port'], entry_url=entry['url'], name=entry['name'], requires_domain=entry['requires_domain']) + ep = self.model.InteractiveToolEntryPoint( + job=job, + tool_port=entry["port"], + entry_url=entry["url"], + name=entry["name"], + requires_domain=entry["requires_domain"], + ) self.sa_session.add(ep) if flush: self.sa_session.flush() def configure_entry_point(self, job, tool_port=None, host=None, port=None, protocol=None): - return self.configure_entry_points(job, {tool_port: dict(tool_port=tool_port, host=host, port=port, protocol=protocol)}) + return self.configure_entry_points( + job, {tool_port: dict(tool_port=tool_port, host=host, port=port, protocol=protocol)} + ) def configure_entry_points(self, job, ports_dict): # There can be multiple entry points that reference the same tool port (could have different entry URLs) @@ -160,9 +184,9 @@ class InteractiveToolManager: log.error("Did not find port to assign to InteractiveToolEntryPoint by tool port: %s.", ep.tool_port) not_configured.append(ep) else: - ep.host = port_dict['host'] - ep.port = port_dict['port'] - ep.protocol = port_dict['protocol'] + ep.host = port_dict["host"] + ep.port = port_dict["port"] + ep.protocol = port_dict["protocol"] ep.configured = True self.sa_session.add(ep) self.save_entry_point(ep) @@ -182,13 +206,17 @@ class InteractiveToolManager: if job and tool: self.create_entry_points(job, tool, entry_points) else: - log.warning('Called InteractiveToolManager.create_interactivetool, but job (%s) or tool (%s) is None', job, tool) + log.warning( + "Called InteractiveToolManager.create_interactivetool, but job (%s) or tool (%s) is None", job, tool + ) def get_nonterminal_for_user_by_trans(self, trans): if trans.user: jobs = trans.sa_session.query(trans.app.model.Job).filter(trans.app.model.Job.user == trans.user) else: - jobs = trans.sa_session.query(trans.app.model.Job).filter(trans.app.model.Job.session_id == trans.get_galaxy_session().id) + jobs = trans.sa_session.query(trans.app.model.Job).filter( + trans.app.model.Job.session_id == trans.get_galaxy_session().id + ) def build_and_apply_filters(query, objects, filter_func): if objects is not None: @@ -200,8 +228,13 @@ class InteractiveToolManager: t.append(filter_func(obj)) query = query.filter(or_(*t)) return query - jobs = build_and_apply_filters(jobs, trans.app.model.Job.non_ready_states, lambda s: trans.app.model.Job.state == s) - return trans.sa_session.query(trans.app.model.InteractiveToolEntryPoint).filter(trans.app.model.InteractiveToolEntryPoint.job_id.in_([job.id for job in jobs])) + + jobs = build_and_apply_filters( + jobs, trans.app.model.Job.non_ready_states, lambda s: trans.app.model.Job.state == s + ) + return trans.sa_session.query(trans.app.model.InteractiveToolEntryPoint).filter( + trans.app.model.InteractiveToolEntryPoint.job_id.in_([job.id for job in jobs]) + ) def can_access_job(self, trans, job): if job: @@ -230,7 +263,7 @@ class InteractiveToolManager: self.remove_entry_point(entry_point) job = entry_point.job if not job.finished: - log.debug('Stopping Job: %s for InteractiveToolEntryPoint: %s', job, entry_point) + log.debug("Stopping Job: %s for InteractiveToolEntryPoint: %s", job, entry_point) job.mark_stopped(trans.app.config.track_jobs_in_database) # This self.job_manager.stop(job) does nothing without changing job.state, manually or e.g. with .mark_deleted() self.job_manager.stop(job) @@ -253,11 +286,11 @@ class InteractiveToolManager: def target_if_active(self, trans, entry_point): if entry_point.active and not entry_point.deleted: request_host = trans.request.host - protocol = trans.request.host_url.split('//', 1)[0] + protocol = trans.request.host_url.split("//", 1)[0] if entry_point.requires_domain: - rval = f'{protocol}//{self.get_entry_point_subdomain(trans, entry_point)}.{request_host}/' + rval = f"{protocol}//{self.get_entry_point_subdomain(trans, entry_point)}.{request_host}/" if entry_point.entry_url: - rval = '{}/{}'.format(rval.rstrip('/'), entry_point.entry_url.lstrip('/')) + rval = "{}/{}".format(rval.rstrip("/"), entry_point.entry_url.lstrip("/")) else: rval = self.get_entry_point_path(trans, entry_point) @@ -269,8 +302,8 @@ class InteractiveToolManager: entry_point_prefix = self.app.config.interactivetools_prefix entry_point_token = entry_point.token if self.app.config.interactivetools_shorten_url: - return f'{entry_point_encoded_id}-{entry_point_token[:10]}.{entry_point_prefix}' - return f'{entry_point_encoded_id}-{entry_point_token}.{entry_point_class}.{entry_point_prefix}' + return f"{entry_point_encoded_id}-{entry_point_token[:10]}.{entry_point_prefix}" + return f"{entry_point_encoded_id}-{entry_point_token}.{entry_point_class}.{entry_point_prefix}" def get_entry_point_path(self, trans, entry_point): entry_point_encoded_id = trans.security.encode_id(entry_point.id) @@ -280,13 +313,13 @@ class InteractiveToolManager: if not entry_point.requires_domain: rval = str(self.app.config.interactivetools_base_path).rstrip("/").lstrip("/") if self.app.config.interactivetools_shorten_url: - rval = f'/{rval}/{entry_point_prefix}/{entry_point_encoded_id}/{entry_point.token[:10]}/' + rval = f"/{rval}/{entry_point_prefix}/{entry_point_encoded_id}/{entry_point.token[:10]}/" else: - rval = f'/{rval}/{entry_point_prefix}/access/{entry_point_class}/{entry_point_encoded_id}/{entry_point.token}/' + rval = f"/{rval}/{entry_point_prefix}/access/{entry_point_class}/{entry_point_encoded_id}/{entry_point.token}/" if entry_point.entry_url: rval = f"{rval.rstrip('/')}/{entry_point.entry_url.lstrip('/')}" if rval[0] != "/": - rval = f'/{rval}' + rval = f"/{rval}" return rval def access_entry_point_target(self, trans, entry_point_id): @@ -295,7 +328,9 @@ class InteractiveToolManager: if entry_point.active: return self.target_if_active(trans, entry_point) elif entry_point.deleted: - raise exceptions.MessageException('InteractiveTool has ended. You will have to start a new one.') + raise exceptions.MessageException("InteractiveTool has ended. You will have to start a new one.") else: - raise exceptions.MessageException('InteractiveTool is not active. If you recently launched this tool it may not be ready yet, please wait a moment and refresh this page.') + raise exceptions.MessageException( + "InteractiveTool is not active. If you recently launched this tool it may not be ready yet, please wait a moment and refresh this page." + ) raise exceptions.ItemAccessibilityException("You do not have access to this InteractiveTool entry point.") diff --git a/lib/galaxy/managers/jobs.py b/lib/galaxy/managers/jobs.py index 37cd15debbb..4c7cca0ed4c 100644 --- a/lib/galaxy/managers/jobs.py +++ b/lib/galaxy/managers/jobs.py @@ -7,7 +7,12 @@ from pydantic import ( BaseModel, Field, ) -from sqlalchemy import and_, false, func, or_ +from sqlalchemy import ( + and_, + false, + func, + or_, +) from sqlalchemy.orm import aliased from sqlalchemy.sql import select @@ -42,10 +47,10 @@ def get_path_key(path_tuple): tuple_elements = len(path_tuple) for i, p in enumerate(path_tuple): if isinstance(p, int): - sep = '_' + sep = "_" else: - sep = '|' - if i == (tuple_elements - 2) and p == 'values': + sep = "|" + if i == (tuple_elements - 2) and p == "values": # dataset inputs are always wrapped in lists. To avoid 'rep_factorName_0|rep_factorLevel_2|countsFile|values_0', # we remove the last 2 items of the path tuple (values and list index) return path_key @@ -57,7 +62,6 @@ def get_path_key(path_tuple): class JobManager: - def __init__(self, app: StructuredApp): self.app = app self.dataset_manager = DatasetManager(app) @@ -66,14 +70,20 @@ class JobManager: return JobLock(active=self.app.job_manager.job_lock) def update_job_lock(self, job_lock: JobLock): - self.app.queue_worker.send_control_task('admin_job_lock', kwargs={'job_lock': job_lock.active}, get_response=True) + self.app.queue_worker.send_control_task( + "admin_job_lock", kwargs={"job_lock": job_lock.active}, get_response=True + ) return self.job_lock() def get_accessible_job(self, trans, decoded_job_id): job = trans.sa_session.query(trans.app.model.Job).filter(trans.app.model.Job.id == decoded_job_id).first() if job is None: raise ObjectNotFound() - belongs_to_user = (job.user_id == trans.user.id) if job.user_id and trans.user else (job.session_id == trans.get_galaxy_session().id) + belongs_to_user = ( + (job.user_id == trans.user.id) + if job.user_id and trans.user + else (job.session_id == trans.get_galaxy_session().id) + ) if not trans.user_is_admin and not belongs_to_user: # Check access granted via output datasets. if not job.output_datasets: @@ -96,6 +106,7 @@ class JobManager: class JobSearch: """Search for jobs using tool inputs or other jobs""" + def __init__( self, sa_session: galaxy_scoped_session, @@ -110,72 +121,85 @@ class JobSearch: self.ldda_manager = ldda_manager self.decode_id = id_encoding_helper.decode_id - def by_tool_input(self, trans, tool_id, tool_version, param=None, param_dump=None, job_state='ok'): + def by_tool_input(self, trans, tool_id, tool_version, param=None, param_dump=None, job_state="ok"): """Search for jobs producing same results using the 'inputs' part of a tool POST.""" user = trans.user input_data = defaultdict(list) def populate_input_data_input_id(path, key, value): """Traverses expanded incoming using remap and collects input_ids and input_data.""" - if key == 'id': + if key == "id": path_key = get_path_key(path[:-2]) current_case = param_dump for p in path: current_case = current_case[p] - src = current_case['src'] + src = current_case["src"] current_case = param for i, p in enumerate(path): - if p == 'values' and i == len(path) - 2: + if p == "values" and i == len(path) - 2: continue if isinstance(current_case, (list, dict)): current_case = current_case[p] identifier = getattr(current_case, "element_identifier", None) - input_data[path_key].append({'src': src, - 'id': value, - 'identifier': identifier, - }) + input_data[path_key].append( + { + "src": src, + "id": value, + "identifier": identifier, + } + ) return key, "__id_wildcard__" return key, value wildcard_param_dump = remap(param_dump, visit=populate_input_data_input_id) - return self.__search(tool_id=tool_id, - tool_version=tool_version, - user=user, - input_data=input_data, - job_state=job_state, - param_dump=param_dump, - wildcard_param_dump=wildcard_param_dump) + return self.__search( + tool_id=tool_id, + tool_version=tool_version, + user=user, + input_data=input_data, + job_state=job_state, + param_dump=param_dump, + wildcard_param_dump=wildcard_param_dump, + ) - def __search(self, tool_id, tool_version, user, input_data, job_state=None, param_dump=None, wildcard_param_dump=None): + def __search( + self, tool_id, tool_version, user, input_data, job_state=None, param_dump=None, wildcard_param_dump=None + ): search_timer = ExecutionTimer() def replace_dataset_ids(path, key, value): """Exchanges dataset_ids (HDA, LDA, HDCA, not Dataset) in param_dump with dataset ids used in job.""" - if key == 'id': + if key == "id": current_case = param_dump for p in path: current_case = current_case[p] - src = current_case['src'] + src = current_case["src"] value = job_input_ids[src][value] return key, value return key, value - job_conditions = [and_( - model.Job.tool_id == tool_id, - model.Job.user == user, - model.Job.copied_from_job_id.is_(None) # Always pick original job - )] + job_conditions = [ + and_( + model.Job.tool_id == tool_id, + model.Job.user == user, + model.Job.copied_from_job_id.is_(None), # Always pick original job + ) + ] if tool_version: job_conditions.append(model.Job.tool_version == str(tool_version)) if job_state is None: job_conditions.append( - model.Job.state.in_([model.Job.states.NEW, - model.Job.states.QUEUED, - model.Job.states.WAITING, - model.Job.states.RUNNING, - model.Job.states.OK]) + model.Job.state.in_( + [ + model.Job.states.NEW, + model.Job.states.QUEUED, + model.Job.states.WAITING, + model.Job.states.RUNNING, + model.Job.states.OK, + ] + ) ) else: if isinstance(job_state, str): @@ -184,19 +208,17 @@ class JobSearch: o = [] for s in job_state: o.append(model.Job.state == s) - job_conditions.append( - or_(*o) - ) + job_conditions.append(or_(*o)) for k, v in wildcard_param_dump.items(): wildcard_value = None - if v == {'__class__': 'RuntimeValue'}: + if v == {"__class__": "RuntimeValue"}: # TODO: verify this is always None. e.g. run with runtime input input v = None - elif k.endswith('|__identifier__'): + elif k.endswith("|__identifier__"): # We've taken care of this while constructing the conditions based on ``input_data`` above continue - elif k == 'chromInfo' and '?.len' in v: + elif k == "chromInfo" and "?.len" in v: continue wildcard_value = '"%?.len"' if not wildcard_value: @@ -204,22 +226,22 @@ class JobSearch: wildcard_value = value_dump.replace('"id": "__id_wildcard__"', '"id": %') a = aliased(model.JobParameter) if value_dump == wildcard_value: - job_conditions.append(and_( - model.Job.id == a.job_id, - a.name == k, - a.value == value_dump, - )) + job_conditions.append( + and_( + model.Job.id == a.job_id, + a.name == k, + a.value == value_dump, + ) + ) else: - job_conditions.append(and_( - model.Job.id == a.job_id, - a.name == k, - a.value.like(wildcard_value) - )) + job_conditions.append(and_(model.Job.id == a.job_id, a.name == k, a.value.like(wildcard_value))) - job_conditions.append(and_( - model.Job.any_output_dataset_collection_instances_deleted == false(), - model.Job.any_output_dataset_deleted == false() - )) + job_conditions.append( + and_( + model.Job.any_output_dataset_collection_instances_deleted == false(), + model.Job.any_output_dataset_deleted == false(), + ) + ) subq = self.sa_session.query(model.Job.id).filter(*job_conditions).subquery() data_conditions = [] @@ -234,14 +256,14 @@ class JobSearch: for k, input_list in input_data.items(): # k will be matched against the JobParameter.name column. This can be prefixed depending on whethter # the input is in a repeat, or not (section and conditional) - k = {k, k.split('|')[-1]} + k = {k, k.split("|")[-1]} for type_values in input_list: - t = type_values['src'] - v = type_values['id'] + t = type_values["src"] + v = type_values["id"] requested_ids.append(v) data_types.append(t) - identifier = type_values['identifier'] - if t == 'hda': + identifier = type_values["identifier"] + if t == "hda": a = aliased(model.JobToInputDatasetAssociation) b = aliased(model.HistoryDatasetAssociation) c = aliased(model.HistoryDatasetAssociation) @@ -252,86 +274,105 @@ class JobSearch: ) name_condition = [] if identifier: - data_conditions.append(and_(model.Job.id == d.job_id, - d.name.in_({f"{_}|__identifier__" for _ in k}), - d.value == json.dumps(identifier))) + data_conditions.append( + and_( + model.Job.id == d.job_id, + d.name.in_({f"{_}|__identifier__" for _ in k}), + d.value == json.dumps(identifier), + ) + ) else: stmt = stmt.where(e.name == c.name) name_condition.append(b.name == c.name) - stmt = stmt.where( - e.extension == c.extension, - ).where( - a.dataset_version == e.version, - ).where( - e._metadata == c._metadata, + stmt = ( + stmt.where( + e.extension == c.extension, + ) + .where( + a.dataset_version == e.version, + ) + .where( + e._metadata == c._metadata, + ) + ) + data_conditions.append( + and_( + a.name.in_(k), + a.dataset_id == b.id, # b is the HDA used for the job + c.dataset_id == b.dataset_id, + c.id == v, # c is the requested job input HDA + # We need to make sure that the job we are looking for has been run with identical inputs. + # Here we deal with 3 requirements: + # - the jobs' input dataset (=b) version is 0, meaning the job's input dataset is not yet ready + # - b's update_time is older than the job create time, meaning no changes occurred + # - the job has a dataset_version recorded, and that versions' metadata matches c's metadata. + or_( + and_( + or_(a.dataset_version.in_([0, b.version]), b.update_time < model.Job.create_time), + b.extension == c.extension, + b.metadata == c.metadata, + *name_condition, + ), + b.id.in_(stmt), + ), + or_(b.deleted == false(), c.deleted == false()), + ) ) - data_conditions.append(and_( - a.name.in_(k), - a.dataset_id == b.id, # b is the HDA used for the job - c.dataset_id == b.dataset_id, - c.id == v, # c is the requested job input HDA - # We need to make sure that the job we are looking for has been run with identical inputs. - # Here we deal with 3 requirements: - # - the jobs' input dataset (=b) version is 0, meaning the job's input dataset is not yet ready - # - b's update_time is older than the job create time, meaning no changes occurred - # - the job has a dataset_version recorded, and that versions' metadata matches c's metadata. - or_( - and_(or_(a.dataset_version.in_([0, b.version]), - b.update_time < model.Job.create_time), - b.extension == c.extension, - b.metadata == c.metadata, - *name_condition, - ), - b.id.in_(stmt) - ), - or_(b.deleted == false(), c.deleted == false()) - - )) used_ids.append(a.dataset_id) - elif t == 'ldda': + elif t == "ldda": a = aliased(model.JobToInputLibraryDatasetAssociation) - data_conditions.append(and_( - model.Job.id == a.job_id, - a.name.in_(k), - a.ldda_id == v - )) + data_conditions.append(and_(model.Job.id == a.job_id, a.name.in_(k), a.ldda_id == v)) used_ids.append(a.ldda_id) - elif t == 'hdca': + elif t == "hdca": a = aliased(model.JobToInputDatasetCollectionAssociation) b = aliased(model.HistoryDatasetCollectionAssociation) c = aliased(model.HistoryDatasetCollectionAssociation) - data_conditions.append(and_( - model.Job.id == a.job_id, - a.name.in_(k), - b.id == a.dataset_collection_id, - c.id == v, - b.name == c.name, - or_(and_(b.deleted == false(), b.id == v), - and_(or_(c.copied_from_history_dataset_collection_association_id == b.id, - b.copied_from_history_dataset_collection_association_id == c.id), - c.deleted == false(), - ) - ) - )) + data_conditions.append( + and_( + model.Job.id == a.job_id, + a.name.in_(k), + b.id == a.dataset_collection_id, + c.id == v, + b.name == c.name, + or_( + and_(b.deleted == false(), b.id == v), + and_( + or_( + c.copied_from_history_dataset_collection_association_id == b.id, + b.copied_from_history_dataset_collection_association_id == c.id, + ), + c.deleted == false(), + ), + ), + ) + ) used_ids.append(a.dataset_collection_id) - elif t == 'dce': + elif t == "dce": a = aliased(model.JobToInputDatasetCollectionElementAssociation) b = aliased(model.DatasetCollectionElement) c = aliased(model.DatasetCollectionElement) - data_conditions.append(and_( - model.Job.id == a.job_id, - a.name.in_(k), - a.dataset_collection_element_id == b.id, - b.element_identifier == c.element_identifier, - c.child_collection_id == b.child_collection_id, - c.id == v, - )) + data_conditions.append( + and_( + model.Job.id == a.job_id, + a.name.in_(k), + a.dataset_collection_element_id == b.id, + b.element_identifier == c.element_identifier, + c.child_collection_id == b.child_collection_id, + c.id == v, + ) + ) used_ids.append(a.dataset_collection_element_id) else: return [] - query = self.sa_session.query(model.Job.id, *used_ids).join(subq, model.Job.id == subq.c.id).filter(*data_conditions).group_by(model.Job.id, *used_ids).order_by(model.Job.id.desc()) + query = ( + self.sa_session.query(model.Job.id, *used_ids) + .join(subq, model.Job.id == subq.c.id) + .filter(*data_conditions) + .group_by(model.Job.id, *used_ids) + .order_by(model.Job.id.desc()) + ) for job in query: # We found a job that is equal in terms of tool_id, user, state and input datasets, # but to be able to verify that the parameters match we need to modify all instances of @@ -351,23 +392,21 @@ class JobSearch: # new_param_dump has its dataset ids remapped to those used by the job. # We now ask if the remapped job parameters match the current job. for k, v in new_param_dump.items(): - if v == {'__class__': 'RuntimeValue'}: + if v == {"__class__": "RuntimeValue"}: # TODO: verify this is always None. e.g. run with runtime input input v = None - elif k.endswith('|__identifier__'): + elif k.endswith("|__identifier__"): # We've taken care of this while constructing the conditions based on ``input_data`` above continue - elif k == 'chromInfo' and '?.len' in v: + elif k == "chromInfo" and "?.len" in v: continue wildcard_value = '"%?.len"' if not wildcard_value: wildcard_value = json.dumps(v, sort_keys=True).replace('"id": "__id_wildcard__"', '"id": %') a = aliased(model.JobParameter) - job_parameter_conditions.append(and_( - model.Job.id == a.job_id, - a.name == k, - a.value == json.dumps(v, sort_keys=True) - )) + job_parameter_conditions.append( + and_(model.Job.id == a.job_id, a.name == k, a.value == json.dumps(v, sort_keys=True)) + ) else: job_parameter_conditions = [model.Job.id == job] query = self.sa_session.query(model.Job).filter(*job_parameter_conditions) @@ -382,10 +421,14 @@ class JobSearch: for parameter in job.parameters: if parameter.name.startswith("__"): continue - if parameter.name in {'chromInfo', 'dbkey'} or parameter.name.endswith('|__identifier__'): + if parameter.name in {"chromInfo", "dbkey"} or parameter.name.endswith("|__identifier__"): continue n_parameters += 1 - if not n_parameters == sum(1 for k in param_dump if not k.startswith('__') and not k.endswith('|__identifier__') and k not in {'chromInfo', 'dbkey'}): + if not n_parameters == sum( + 1 + for k in param_dump + if not k.startswith("__") and not k.endswith("|__identifier__") and k not in {"chromInfo", "dbkey"} + ): continue log.info("Found equivalent job %s", search_timer) return job @@ -395,44 +438,48 @@ class JobSearch: def view_show_job(trans, job, full: bool) -> typing.Dict: is_admin = trans.user_is_admin - job_dict = trans.app.security.encode_all_ids(job.to_dict('element', system_details=is_admin), True) - if trans.app.config.expose_dataset_path and 'command_line' not in job_dict: - job_dict['command_line'] = job.command_line + job_dict = trans.app.security.encode_all_ids(job.to_dict("element", system_details=is_admin), True) + if trans.app.config.expose_dataset_path and "command_line" not in job_dict: + job_dict["command_line"] = job.command_line if full: - job_dict.update(dict( - tool_stdout=job.tool_stdout, - tool_stderr=job.tool_stderr, - job_stdout=job.job_stdout, - job_stderr=job.job_stderr, - stderr=job.stderr, - stdout=job.stdout, - job_messages=job.job_messages, - dependencies=job.dependencies - )) + job_dict.update( + dict( + tool_stdout=job.tool_stdout, + tool_stderr=job.tool_stderr, + job_stdout=job.job_stdout, + job_stderr=job.job_stderr, + stderr=job.stderr, + stdout=job.stdout, + job_messages=job.job_messages, + dependencies=job.dependencies, + ) + ) if is_admin: - job_dict['user_email'] = job.get_user_email() - job_dict['job_metrics'] = summarize_job_metrics(trans, job) + job_dict["user_email"] = job.get_user_email() + job_dict["job_metrics"] = summarize_job_metrics(trans, job) return job_dict def invocation_job_source_iter(sa_session, invocation_id): # TODO: Handle subworkflows. - join = model.WorkflowInvocationStep.table.join( - model.WorkflowInvocation - ) - statement = select( - [model.WorkflowInvocationStep.job_id, model.WorkflowInvocationStep.implicit_collection_jobs_id, model.WorkflowInvocationStep.state] - ).select_from( - join - ).where( - model.WorkflowInvocation.id == invocation_id + join = model.WorkflowInvocationStep.table.join(model.WorkflowInvocation) + statement = ( + select( + [ + model.WorkflowInvocationStep.job_id, + model.WorkflowInvocationStep.implicit_collection_jobs_id, + model.WorkflowInvocationStep.state, + ] + ) + .select_from(join) + .where(model.WorkflowInvocation.id == invocation_id) ) for row in sa_session.execute(statement): if row[0]: - yield ('Job', row[0], row[2]) + yield ("Job", row[0], row[2]) if row[1]: - yield ('ImplicitCollectionJobs', row[1], row[2]) + yield ("ImplicitCollectionJobs", row[1], row[2]) def fetch_job_states(sa_session, job_source_ids, job_source_types): @@ -440,7 +487,9 @@ def fetch_job_states(sa_session, job_source_ids, job_source_types): job_ids = set() implicit_collection_job_ids = set() workflow_invocations_job_sources = {} - workflow_invocation_states = {} # should be set before we walk step states to be conservative on whether things are done expanding yet + workflow_invocation_states = ( + {} + ) # should be set before we walk step states to be conservative on whether things are done expanding yet for job_source_id, job_source_type in zip(job_source_ids, job_source_types): if job_source_type == "Job": @@ -451,8 +500,14 @@ def fetch_job_states(sa_session, job_source_ids, job_source_types): invocation_state = sa_session.query(model.WorkflowInvocation).get(job_source_id).state workflow_invocation_states[job_source_id] = invocation_state workflow_invocation_job_sources = [] - for (invocation_step_source_type, invocation_step_source_id, invocation_step_state) in invocation_job_source_iter(sa_session, job_source_id): - workflow_invocation_job_sources.append((invocation_step_source_type, invocation_step_source_id, invocation_step_state)) + for ( + invocation_step_source_type, + invocation_step_source_id, + invocation_step_state, + ) in invocation_job_source_iter(sa_session, job_source_id): + workflow_invocation_job_sources.append( + (invocation_step_source_type, invocation_step_source_id, invocation_step_state) + ) if invocation_step_source_type == "Job": job_ids.add(invocation_step_source_id) elif invocation_step_source_type == "ImplicitCollectionJobs": @@ -467,7 +522,9 @@ def fetch_job_states(sa_session, job_source_ids, job_source_types): for job_id in job_ids: job_summaries[job_id] = summarize_jobs_to_dict(sa_session, sa_session.query(model.Job).get(job_id)) for implicit_collection_jobs_id in implicit_collection_job_ids: - implicit_collection_jobs_summaries[implicit_collection_jobs_id] = summarize_jobs_to_dict(sa_session, sa_session.query(model.ImplicitCollectionJobs).get(implicit_collection_jobs_id)) + implicit_collection_jobs_summaries[implicit_collection_jobs_id] = summarize_jobs_to_dict( + sa_session, sa_session.query(model.ImplicitCollectionJobs).get(implicit_collection_jobs_id) + ) rval = [] for job_source_id, job_source_type in zip(job_source_ids, job_source_types): @@ -480,18 +537,34 @@ def fetch_job_states(sa_session, job_source_ids, job_source_types): invocation_job_summaries = [] invocation_implicit_collection_job_summaries = [] invocation_step_states = [] - for (invocation_step_source_type, invocation_step_source_id, invocation_step_state) in workflow_invocations_job_sources[job_source_id]: + for ( + invocation_step_source_type, + invocation_step_source_id, + invocation_step_state, + ) in workflow_invocations_job_sources[job_source_id]: invocation_step_states.append(invocation_step_state) if invocation_step_source_type == "Job": invocation_job_summaries.append(job_summaries[invocation_step_source_id]) else: - invocation_implicit_collection_job_summaries.append(implicit_collection_jobs_summaries[invocation_step_source_id]) - rval.append(summarize_invocation_jobs(job_source_id, invocation_job_summaries, invocation_implicit_collection_job_summaries, invocation_state, invocation_step_states)) + invocation_implicit_collection_job_summaries.append( + implicit_collection_jobs_summaries[invocation_step_source_id] + ) + rval.append( + summarize_invocation_jobs( + job_source_id, + invocation_job_summaries, + invocation_implicit_collection_job_summaries, + invocation_state, + invocation_step_states, + ) + ) return rval -def summarize_invocation_jobs(invocation_id, job_summaries, implicit_collection_job_summaries, invocation_state, invocation_step_states): +def summarize_invocation_jobs( + invocation_id, job_summaries, implicit_collection_job_summaries, invocation_state, invocation_step_states +): states = {} if invocation_state == "scheduled": all_scheduled = True @@ -567,14 +640,11 @@ def summarize_jobs_to_dict(sa_session, jobs_source): join = model.ImplicitCollectionJobs.table.join( model.ImplicitCollectionJobsJobAssociation.table.join(model.Job) ) - statement = select( - [model.Job.state, func.count("*")] - ).select_from( - join - ).where( - model.ImplicitCollectionJobs.id == jobs_source.id - ).group_by( - model.Job.state + statement = ( + select([model.Job.state, func.count("*")]) + .select_from(join) + .where(model.ImplicitCollectionJobs.id == jobs_source.id) + .group_by(model.Job.state) ) for row in sa_session.execute(statement): states[row[0]] = row[1] @@ -604,7 +674,7 @@ def summarize_job_metrics(trans, job): raw_value=str(metric_value), ) - metrics = [m for m in job.metrics if m.plugin != 'env' or trans.user_is_admin] + metrics = [m for m in job.metrics if m.plugin != "env" or trans.user_is_admin] return list(map(metric_to_dict, metrics)) @@ -615,9 +685,11 @@ def summarize_destination_params(trans, job): represented by the trans parameter. """ - destination_params = {'Runner': job.job_runner_name, - 'Runner Job ID': job.job_runner_external_id, - 'Handler': job.handler} + destination_params = { + "Runner": job.job_runner_name, + "Runner Job ID": job.job_runner_external_id, + "Handler": job.handler, + } job_destination_params = job.destination_params if job_destination_params: destination_params.update(job_destination_params) @@ -630,6 +702,7 @@ def summarize_job_parameters(trans, job): Precondition: the caller has verified the job is accessible to the user represented by the trans parameter. """ + def inputs_recursive(input_params, param_values, depth=1, upgrade_messages=None): if upgrade_messages is None: upgrade_messages = {} @@ -644,21 +717,50 @@ def summarize_job_parameters(trans, job): elif input.type == "section": # Get the value of the current Section parameter rval.append(dict(text=input.name, depth=depth)) - rval.extend(inputs_recursive(input.inputs, param_values[input.name], depth=depth + 1, upgrade_messages=upgrade_messages.get(input.name))) + rval.extend( + inputs_recursive( + input.inputs, + param_values[input.name], + depth=depth + 1, + upgrade_messages=upgrade_messages.get(input.name), + ) + ) elif input.type == "conditional": try: - current_case = param_values[input.name]['__current_case__'] + current_case = param_values[input.name]["__current_case__"] is_valid = True except Exception: current_case = None is_valid = False if is_valid: - rval.append(dict(text=input.test_param.label, depth=depth, value=input.cases[current_case].value)) - rval.extend(inputs_recursive(input.cases[current_case].inputs, param_values[input.name], depth=depth + 1, upgrade_messages=upgrade_messages.get(input.name))) + rval.append( + dict(text=input.test_param.label, depth=depth, value=input.cases[current_case].value) + ) + rval.extend( + inputs_recursive( + input.cases[current_case].inputs, + param_values[input.name], + depth=depth + 1, + upgrade_messages=upgrade_messages.get(input.name), + ) + ) else: - rval.append(dict(text=input.name, depth=depth, notes="The previously used value is no longer valid.", error=True)) + rval.append( + dict( + text=input.name, + depth=depth, + notes="The previously used value is no longer valid.", + error=True, + ) + ) elif input.type == "upload_dataset": - rval.append(dict(text=input.group_title(param_values), depth=depth, value=f"{len(param_values[input.name])} uploaded datasets")) + rval.append( + dict( + text=input.group_title(param_values), + depth=depth, + value=f"{len(param_values[input.name])} uploaded datasets", + ) + ) elif input.type == "data": value = [] for element in listify(param_values[input.name]): @@ -667,11 +769,13 @@ def summarize_job_parameters(trans, job): hda = element value.append({"src": "hda", "id": encoded_id, "hid": hda.hid, "name": hda.name}) elif isinstance(element, model.DatasetCollectionElement): - value.append({'src': "dce", "id": encoded_id, "name": element.element_identifier}) + value.append({"src": "dce", "id": encoded_id, "name": element.element_identifier}) elif isinstance(element, model.HistoryDatasetCollectionAssociation): value.append({"src": "hdca", "id": encoded_id, "hid": element.hid, "name": element.name}) else: - raise Exception(f"Unhandled data input parameter type encountered {element.__class__.__name__}") + raise Exception( + f"Unhandled data input parameter type encountered {element.__class__.__name__}" + ) rval.append(dict(text=input.label, depth=depth, value=value)) elif input.visible: if hasattr(input, "label") and input.label: @@ -679,7 +783,14 @@ def summarize_job_parameters(trans, job): else: # value for label not required, fallback to input name (same as tool panel) label = input.name - rval.append(dict(text=label, depth=depth, value=input.value_to_display_text(param_values[input.name]), notes=upgrade_messages.get(input.name, ''))) + rval.append( + dict( + text=label, + depth=depth, + value=input.value_to_display_text(param_values[input.name]), + notes=upgrade_messages.get(input.name, ""), + ) + ) else: # Parameter does not have a stored value. # Get parameter label. @@ -689,7 +800,9 @@ def summarize_job_parameters(trans, job): label = input.label() else: label = input.label or input.name - rval.append(dict(text=label, depth=depth, notes="not used (parameter was added after this job was run)")) + rval.append( + dict(text=label, depth=depth, notes="not used (parameter was added after this job was run)") + ) return rval @@ -710,15 +823,19 @@ def summarize_job_parameters(trans, job): except Exception: params_objects = job.get_param_values(app, ignore_errors=True) # use different param_objects in the following line, since we want to display original values as much as possible - upgrade_messages = tool.check_and_update_param_values(job.get_param_values(app, ignore_errors=True), - trans, - update_values=False) + upgrade_messages = tool.check_and_update_param_values( + job.get_param_values(app, ignore_errors=True), trans, update_values=False + ) has_parameter_errors = True parameters = inputs_recursive(tool.inputs, params_objects, depth=1, upgrade_messages=upgrade_messages) else: has_parameter_errors = True - return {"parameters": parameters, "has_parameter_errors": has_parameter_errors, 'outputs': summarize_job_outputs(job=job, tool=tool, params=params_objects, security=trans.security)} + return { + "parameters": parameters, + "has_parameter_errors": has_parameter_errors, + "outputs": summarize_job_outputs(job=job, tool=tool, params=params_objects, security=trans.security), + } def get_output_name(tool, output, params): @@ -736,16 +853,23 @@ def summarize_job_outputs(job: model.Job, tool, params, security): outputs = defaultdict(list) output_labels = {} possible_outputs = ( - ('hda', 'dataset_id', job.output_datasets), - ('ldda', 'ldda_id', job.output_library_datasets), - ('hdca', 'dataset_collection_id', job.output_dataset_collection_instances), + ("hda", "dataset_id", job.output_datasets), + ("ldda", "ldda_id", job.output_library_datasets), + ("hdca", "dataset_collection_id", job.output_dataset_collection_instances), ) for src, attribute, output_associations in possible_outputs: for output_association in output_associations: output_name = output_association.name if output_name not in output_labels and tool: - tool_output = tool.output_collections if src == 'hdca' else tool.outputs - output_labels[output_name] = get_output_name(tool=tool, output=tool_output.get(output_name), params=params) + tool_output = tool.output_collections if src == "hdca" else tool.outputs + output_labels[output_name] = get_output_name( + tool=tool, output=tool_output.get(output_name), params=params + ) label = output_labels.get(output_name) - outputs[output_name].append({'label': label, 'value': {'src': src, 'id': security.encode_id(getattr(output_association, attribute))}}) + outputs[output_name].append( + { + "label": label, + "value": {"src": src, "id": security.encode_id(getattr(output_association, attribute))}, + } + ) return outputs diff --git a/lib/galaxy/managers/lddas.py b/lib/galaxy/managers/lddas.py index 7de07fd38a3..8d08e67c87d 100644 --- a/lib/galaxy/managers/lddas.py +++ b/lib/galaxy/managers/lddas.py @@ -12,6 +12,7 @@ class LDDAManager(DatasetAssociationManager): """ A fairly sparse manager for LDDAs. """ + model_class = model.LibraryDatasetDatasetAssociation def __init__(self, app: MinimalManagerApp): @@ -21,10 +22,9 @@ class LDDAManager(DatasetAssociationManager): super().__init__(app) def get(self, trans, id, check_accessible=True): - return manager_base.get_object(trans, id, - 'LibraryDatasetDatasetAssociation', - check_ownership=False, - check_accessible=check_accessible) + return manager_base.get_object( + trans, id, "LibraryDatasetDatasetAssociation", check_ownership=False, check_accessible=check_accessible + ) def _set_permissions(self, trans, library_dataset, role_ids_dict): # Check Git history for an older broken implementation, but it was broken diff --git a/lib/galaxy/managers/libraries.py b/lib/galaxy/managers/libraries.py index 1aedd9e0fb5..71eba2d361c 100644 --- a/lib/galaxy/managers/libraries.py +++ b/lib/galaxy/managers/libraries.py @@ -2,17 +2,21 @@ Manager and Serializer for libraries. """ import logging -from typing import ( - Optional, +from typing import Optional + +from sqlalchemy import ( + and_, + false, + not_, + or_, + true, +) +from sqlalchemy.orm.exc import ( + MultipleResultsFound, + NoResultFound, ) -from sqlalchemy import and_, false, not_, or_, true -from sqlalchemy.orm.exc import MultipleResultsFound -from sqlalchemy.orm.exc import NoResultFound - -from galaxy import ( - exceptions, -) +from galaxy import exceptions from galaxy.managers.folders import FolderManager from galaxy.util import ( pretty_print_time_interval, @@ -41,25 +45,29 @@ class LibraryManager: :rtype: galaxy.model.Library """ try: - library = trans.sa_session.query(trans.app.model.Library).filter(trans.app.model.Library.table.c.id == decoded_library_id).one() + library = ( + trans.sa_session.query(trans.app.model.Library) + .filter(trans.app.model.Library.table.c.id == decoded_library_id) + .one() + ) except MultipleResultsFound: - raise exceptions.InconsistentDatabase('Multiple libraries found with the same id.') + raise exceptions.InconsistentDatabase("Multiple libraries found with the same id.") except NoResultFound: - raise exceptions.RequestParameterInvalidException('No library found with the id provided.') + raise exceptions.RequestParameterInvalidException("No library found with the id provided.") except Exception as e: raise exceptions.InternalServerError(f"Error loading from the database.{unicodify(e)}") library = self.secure(trans, library, check_accessible) return library - def create(self, trans, name, description='', synopsis=''): + def create(self, trans, name, description="", synopsis=""): """ Create a new library. """ if not trans.user_is_admin: - raise exceptions.ItemAccessibilityException('Only administrators can create libraries.') + raise exceptions.ItemAccessibilityException("Only administrators can create libraries.") else: library = trans.app.model.Library(name=name, description=description, synopsis=synopsis) - root_folder = trans.app.model.LibraryFolder(name=name, description='') + root_folder = trans.app.model.LibraryFolder(name=name, description="") library.root_folder = root_folder trans.sa_session.add_all((library, root_folder)) trans.sa_session.flush() @@ -77,7 +85,7 @@ class LibraryManager: if not user_can_modify: raise exceptions.ItemAccessibilityException("You don't have permission update libraries.") if library.deleted: - raise exceptions.RequestParameterInvalidException('You cannot modify a deleted library. Undelete it first.') + raise exceptions.RequestParameterInvalidException("You cannot modify a deleted library. Undelete it first.") if name is not None: library.name = name changed = True @@ -100,7 +108,7 @@ class LibraryManager: Mark given library deleted/undeleted based on the flag. """ if not trans.user_is_admin: - raise exceptions.ItemAccessibilityException('Only administrators can delete and undelete libraries.') + raise exceptions.ItemAccessibilityException("Only administrators can delete and undelete libraries.") if undelete: library.deleted = False else: @@ -127,11 +135,15 @@ class LibraryManager: is_admin = trans.user_is_admin query = trans.sa_session.query(trans.app.model.Library) library_access_action = trans.app.security_agent.permitted_actions.LIBRARY_ACCESS.action - restricted_library_ids = {lp.library_id for lp in ( - trans.sa_session.query(trans.model.LibraryPermissions).filter( - trans.model.LibraryPermissions.table.c.action == library_access_action - ).distinct())} - prefetched_ids = {'restricted_library_ids': restricted_library_ids} + restricted_library_ids = { + lp.library_id + for lp in ( + trans.sa_session.query(trans.model.LibraryPermissions) + .filter(trans.model.LibraryPermissions.table.c.action == library_access_action) + .distinct() + ) + } + prefetched_ids = {"restricted_library_ids": restricted_library_ids} if is_admin: if deleted is None: # Flag is not specified, do not filter on it. @@ -147,7 +159,9 @@ class LibraryManager: else: query = query.filter(trans.app.model.Library.table.c.deleted == false()) current_user_role_ids = [role.id for role in trans.get_current_user_roles()] - all_actions = trans.sa_session.query(trans.model.LibraryPermissions).filter(trans.model.LibraryPermissions.table.c.role_id.in_(current_user_role_ids)) + all_actions = trans.sa_session.query(trans.model.LibraryPermissions).filter( + trans.model.LibraryPermissions.table.c.role_id.in_(current_user_role_ids) + ) library_add_action = trans.app.security_agent.permitted_actions.LIBRARY_ADD.action library_modify_action = trans.app.security_agent.permitted_actions.LIBRARY_MODIFY.action library_manage_action = trans.app.security_agent.permitted_actions.LIBRARY_MANAGE.action @@ -164,13 +178,15 @@ class LibraryManager: allowed_library_modify_ids.add(action.library_id) if action.action == library_manage_action: allowed_library_manage_ids.add(action.library_id) - query = query.filter(or_( - not_(trans.model.Library.table.c.id.in_(restricted_library_ids)), - trans.model.Library.table.c.id.in_(accessible_restricted_library_ids) - )) - prefetched_ids['allowed_library_add_ids'] = allowed_library_add_ids - prefetched_ids['allowed_library_modify_ids'] = allowed_library_modify_ids - prefetched_ids['allowed_library_manage_ids'] = allowed_library_manage_ids + query = query.filter( + or_( + not_(trans.model.Library.table.c.id.in_(restricted_library_ids)), + trans.model.Library.table.c.id.in_(accessible_restricted_library_ids), + ) + ) + prefetched_ids["allowed_library_add_ids"] = allowed_library_add_ids + prefetched_ids["allowed_library_modify_ids"] = allowed_library_modify_ids + prefetched_ids["allowed_library_manage_ids"] = allowed_library_manage_ids return query, prefetched_ids def secure(self, trans, library, check_accessible=True): @@ -197,9 +213,9 @@ class LibraryManager: Check whether the library is accessible to current user. """ if not trans.app.security_agent.can_access_library(trans.get_current_user_roles(), library): - raise exceptions.ObjectNotFound('Library with the id provided was not found.') + raise exceptions.ObjectNotFound("Library with the id provided was not found.") elif library.deleted: - raise exceptions.ObjectNotFound('Library with the id provided is deleted.') + raise exceptions.ObjectNotFound("Library with the id provided is deleted.") else: return library @@ -218,27 +234,41 @@ class LibraryManager: :returns: dict with data about the library :rtype: dictionary """ - restricted_library_ids = prefetched_ids.get('restricted_library_ids', None) if prefetched_ids else None - allowed_library_add_ids = prefetched_ids.get('allowed_library_add_ids', None) if prefetched_ids else None - allowed_library_modify_ids = prefetched_ids.get('allowed_library_modify_ids', None) if prefetched_ids else None - allowed_library_manage_ids = prefetched_ids.get('allowed_library_manage_ids', None) if prefetched_ids else None - library_dict = library.to_dict(view='element', value_mapper={'id': trans.security.encode_id, 'root_folder_id': trans.security.encode_id}) - library_dict['public'] = False if (restricted_library_ids and library.id in restricted_library_ids) else True - library_dict['create_time_pretty'] = pretty_print_time_interval(library.create_time, precise=True) + restricted_library_ids = prefetched_ids.get("restricted_library_ids", None) if prefetched_ids else None + allowed_library_add_ids = prefetched_ids.get("allowed_library_add_ids", None) if prefetched_ids else None + allowed_library_modify_ids = prefetched_ids.get("allowed_library_modify_ids", None) if prefetched_ids else None + allowed_library_manage_ids = prefetched_ids.get("allowed_library_manage_ids", None) if prefetched_ids else None + library_dict = library.to_dict( + view="element", value_mapper={"id": trans.security.encode_id, "root_folder_id": trans.security.encode_id} + ) + library_dict["public"] = False if (restricted_library_ids and library.id in restricted_library_ids) else True + library_dict["create_time_pretty"] = pretty_print_time_interval(library.create_time, precise=True) if not trans.user_is_admin: if prefetched_ids: - library_dict['can_user_add'] = True if (allowed_library_add_ids and library.id in allowed_library_add_ids) else False - library_dict['can_user_modify'] = True if (allowed_library_modify_ids and library.id in allowed_library_modify_ids) else False - library_dict['can_user_manage'] = True if (allowed_library_manage_ids and library.id in allowed_library_manage_ids) else False + library_dict["can_user_add"] = ( + True if (allowed_library_add_ids and library.id in allowed_library_add_ids) else False + ) + library_dict["can_user_modify"] = ( + True if (allowed_library_modify_ids and library.id in allowed_library_modify_ids) else False + ) + library_dict["can_user_manage"] = ( + True if (allowed_library_manage_ids and library.id in allowed_library_manage_ids) else False + ) else: current_user_roles = trans.get_current_user_roles() - library_dict['can_user_add'] = trans.app.security_agent.can_add_library_item(current_user_roles, library) - library_dict['can_user_modify'] = trans.app.security_agent.can_modify_library_item(current_user_roles, library) - library_dict['can_user_manage'] = trans.app.security_agent.can_manage_library_item(current_user_roles, library) + library_dict["can_user_add"] = trans.app.security_agent.can_add_library_item( + current_user_roles, library + ) + library_dict["can_user_modify"] = trans.app.security_agent.can_modify_library_item( + current_user_roles, library + ) + library_dict["can_user_manage"] = trans.app.security_agent.can_manage_library_item( + current_user_roles, library + ) else: - library_dict['can_user_add'] = True - library_dict['can_user_modify'] = True - library_dict['can_user_manage'] = True + library_dict["can_user_add"] = True + library_dict["can_user_modify"] = True + library_dict["can_user_manage"] = True return library_dict def get_current_roles(self, trans, library): @@ -251,14 +281,27 @@ class LibraryManager: :rtype: dictionary :returns: dict of current roles for all available permission types """ - access_library_role_list = [(access_role.name, trans.security.encode_id(access_role.id)) for access_role in self.get_access_roles(trans, library)] - modify_library_role_list = [(modify_role.name, trans.security.encode_id(modify_role.id)) for modify_role in self.get_modify_roles(trans, library)] - manage_library_role_list = [(manage_role.name, trans.security.encode_id(manage_role.id)) for manage_role in self.get_manage_roles(trans, library)] - add_library_item_role_list = [(add_role.name, trans.security.encode_id(add_role.id)) for add_role in self.get_add_roles(trans, library)] - return dict(access_library_role_list=access_library_role_list, - modify_library_role_list=modify_library_role_list, - manage_library_role_list=manage_library_role_list, - add_library_item_role_list=add_library_item_role_list) + access_library_role_list = [ + (access_role.name, trans.security.encode_id(access_role.id)) + for access_role in self.get_access_roles(trans, library) + ] + modify_library_role_list = [ + (modify_role.name, trans.security.encode_id(modify_role.id)) + for modify_role in self.get_modify_roles(trans, library) + ] + manage_library_role_list = [ + (manage_role.name, trans.security.encode_id(manage_role.id)) + for manage_role in self.get_manage_roles(trans, library) + ] + add_library_item_role_list = [ + (add_role.name, trans.security.encode_id(add_role.id)) for add_role in self.get_add_roles(trans, library) + ] + return dict( + access_library_role_list=access_library_role_list, + modify_library_role_list=modify_library_role_list, + manage_library_role_list=manage_library_role_list, + add_library_item_role_list=add_library_item_role_list, + ) def get_access_roles(self, trans, library): """ @@ -270,19 +313,31 @@ class LibraryManager: """ Load modify roles for all library permissions """ - return set(trans.app.security_agent.get_roles_for_action(library, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY)) + return set( + trans.app.security_agent.get_roles_for_action( + library, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY + ) + ) def get_manage_roles(self, trans, library): """ Load manage roles for all library permissions """ - return set(trans.app.security_agent.get_roles_for_action(library, trans.app.security_agent.permitted_actions.LIBRARY_MANAGE)) + return set( + trans.app.security_agent.get_roles_for_action( + library, trans.app.security_agent.permitted_actions.LIBRARY_MANAGE + ) + ) def get_add_roles(self, trans, library): """ Load add roles for all library permissions """ - return set(trans.app.security_agent.get_roles_for_action(library, trans.app.security_agent.permitted_actions.LIBRARY_ADD)) + return set( + trans.app.security_agent.get_roles_for_action( + library, trans.app.security_agent.permitted_actions.LIBRARY_ADD + ) + ) def set_permission_roles(self, trans, library, access_roles, modify_roles, manage_roles, add_roles): """ @@ -310,8 +365,8 @@ def get_containing_library_from_library_dataset(trans, library_dataset): folder = folder.parent # We have folder set to the library's root folder, which has the same name as the library for library in trans.sa_session.query(trans.model.Library).filter( - and_(trans.model.Library.table.c.deleted == false(), - trans.model.Library.table.c.name == folder.name)): + and_(trans.model.Library.table.c.deleted == false(), trans.model.Library.table.c.name == folder.name) + ): # Just to double-check if library.root_folder == folder: return library diff --git a/lib/galaxy/managers/library_datasets.py b/lib/galaxy/managers/library_datasets.py index b26314c0d7e..f2a2eab8973 100644 --- a/lib/galaxy/managers/library_datasets.py +++ b/lib/galaxy/managers/library_datasets.py @@ -3,17 +3,15 @@ import logging from galaxy import ( model, - util + util, ) from galaxy.exceptions import ( InsufficientPermissionsException, InternalServerError, ObjectNotFound, - RequestParameterInvalidException -) -from galaxy.managers import ( - datasets, + RequestParameterInvalidException, ) +from galaxy.managers import datasets from galaxy.model import tags from galaxy.structured_app import MinimalManagerApp from galaxy.util import validation @@ -23,6 +21,7 @@ log = logging.getLogger(__name__) class LibraryDatasetsManager(datasets.DatasetAssociationManager): """Interface/service object for interacting with library datasets.""" + model_class = model.LibraryDatasetDatasetAssociation def __init__(self, app: MinimalManagerApp): @@ -42,7 +41,11 @@ class LibraryDatasetsManager(datasets.DatasetAssociationManager): :rtype: galaxy.model.LibraryDataset """ try: - ld = trans.sa_session.query(trans.app.model.LibraryDataset).filter(trans.app.model.LibraryDataset.table.c.id == decoded_library_dataset_id).one() + ld = ( + trans.sa_session.query(trans.app.model.LibraryDataset) + .filter(trans.app.model.LibraryDataset.table.c.id == decoded_library_dataset_id) + .one() + ) except Exception as e: raise InternalServerError(f"Error loading from the database.{util.unicodify(e)}") ld = self.secure(trans, ld, check_accessible) @@ -80,30 +83,30 @@ class LibraryDatasetsManager(datasets.DatasetAssociationManager): def _set_from_dict(self, trans, ldda, new_data): changed = False - new_name = new_data.get('name', None) + new_name = new_data.get("name", None) if new_name is not None and new_name != ldda.name: ldda.name = new_name changed = True - new_misc_info = new_data.get('misc_info', None) + new_misc_info = new_data.get("misc_info", None) if new_misc_info is not None and new_misc_info != ldda.info: ldda.info = new_misc_info changed = True - new_message = new_data.get('message', None) + new_message = new_data.get("message", None) if new_message is not None and new_message != ldda.message: ldda.message = new_message changed = True - new_file_ext = new_data.get('file_ext', None) - if new_file_ext == 'auto': + new_file_ext = new_data.get("file_ext", None) + if new_file_ext == "auto": self.detect_datatype(trans, ldda) elif new_file_ext is not None and new_file_ext != ldda.extension: ldda.extension = new_file_ext self.set_metadata(trans, ldda) changed = True - new_genome_build = new_data.get('genome_build', None) + new_genome_build = new_data.get("genome_build", None) if new_genome_build is not None and new_genome_build != ldda.dbkey: ldda.dbkey = new_genome_build changed = True - new_tags = new_data.get('tags', None) + new_tags = new_data.get("tags", None) if new_tags is not None and new_tags != ldda.tags: self.tag_handler.delete_item_tags(item=ldda, user=trans.user) tag_list = self.tag_handler.parse_tags_list(new_tags) @@ -122,25 +125,29 @@ class LibraryDatasetsManager(datasets.DatasetAssociationManager): for key, val in payload.items(): if val is None: continue - if key in ('name'): + if key in ("name"): if len(val) < MINIMUM_STRING_LENGTH: - raise RequestParameterInvalidException(f'{key} must have at least length of {MINIMUM_STRING_LENGTH}') + raise RequestParameterInvalidException( + f"{key} must have at least length of {MINIMUM_STRING_LENGTH}" + ) val = validation.validate_and_sanitize_basestring(key, val) validated_payload[key] = val - if key in ('misc_info', 'message'): + if key in ("misc_info", "message"): val = validation.validate_and_sanitize_basestring(key, val) validated_payload[key] = val - if key in ('file_ext'): + if key in ("file_ext"): datatype = self.app.datatypes_registry.get_datatype_by_extension(val) if datatype is None and val not in ("auto",): - raise RequestParameterInvalidException(f'This Galaxy does not recognize the datatype of: {val}') + raise RequestParameterInvalidException(f"This Galaxy does not recognize the datatype of: {val}") validated_payload[key] = val - if key in ('genome_build'): + if key in ("genome_build"): if len(val) < MINIMUM_STRING_LENGTH: - raise RequestParameterInvalidException(f'{key} must have at least length of {MINIMUM_STRING_LENGTH}') + raise RequestParameterInvalidException( + f"{key} must have at least length of {MINIMUM_STRING_LENGTH}" + ) val = validation.validate_and_sanitize_basestring(key, val) validated_payload[key] = val - if key in ('tags'): + if key in ("tags"): val = validation.validate_and_sanitize_basestring_list(key, util.listify(val)) validated_payload[key] = val return validated_payload @@ -177,9 +184,9 @@ class LibraryDatasetsManager(datasets.DatasetAssociationManager): :raises: ObjectNotFound """ if not trans.app.security_agent.can_access_library_item(trans.get_current_user_roles(), ld, trans.user): - raise ObjectNotFound('Library dataset with the id provided was not found.') + raise ObjectNotFound("Library dataset with the id provided was not found.") elif ld.deleted: - raise ObjectNotFound('Library dataset with the id provided is deleted.') + raise ObjectNotFound("Library dataset with the id provided is deleted.") else: return ld @@ -196,11 +203,11 @@ class LibraryDatasetsManager(datasets.DatasetAssociationManager): :raises: ObjectNotFound """ if ld.deleted: - raise ObjectNotFound('Library dataset with the id provided is deleted.') + raise ObjectNotFound("Library dataset with the id provided is deleted.") elif trans.user_is_admin: return ld if not trans.app.security_agent.can_modify_library_item(trans.get_current_user_roles(), ld): - raise InsufficientPermissionsException('You do not have proper permission to modify this library dataset.') + raise InsufficientPermissionsException("You do not have proper permission to modify this library dataset.") else: return ld @@ -221,29 +228,33 @@ class LibraryDatasetsManager(datasets.DatasetAssociationManager): rval = trans.security.encode_all_ids(ld.to_dict()) if len(expired_ldda_versions) > 0: - rval['has_versions'] = True - rval['expired_versions'] = expired_ldda_versions + rval["has_versions"] = True + rval["expired_versions"] = expired_ldda_versions ldda = ld.library_dataset_dataset_association if ldda.creating_job_associations: if ldda.creating_job_associations[0].job.stdout: - rval['job_stdout'] = ldda.creating_job_associations[0].job.stdout.strip() + rval["job_stdout"] = ldda.creating_job_associations[0].job.stdout.strip() if ldda.creating_job_associations[0].job.stderr: - rval['job_stderr'] = ldda.creating_job_associations[0].job.stderr.strip() + rval["job_stderr"] = ldda.creating_job_associations[0].job.stderr.strip() if ldda.dataset.uuid: - rval['uuid'] = str(ldda.dataset.uuid) - rval['deleted'] = ld.deleted - rval['folder_id'] = f"F{rval['folder_id']}" - rval['full_path'] = full_path - rval['file_size'] = util.nice_size(int(ldda.get_size())) - rval['date_uploaded'] = ldda.create_time.strftime("%Y-%m-%d %I:%M %p") - rval['update_time'] = ldda.update_time.strftime("%Y-%m-%d %I:%M %p") - rval['can_user_modify'] = trans.user_is_admin or trans.app.security_agent.can_modify_library_item(current_user_roles, ld) - rval['is_unrestricted'] = trans.app.security_agent.dataset_is_public(ldda.dataset) - rval['tags'] = self.tag_handler.get_tags_str(ldda.tags) + rval["uuid"] = str(ldda.dataset.uuid) + rval["deleted"] = ld.deleted + rval["folder_id"] = f"F{rval['folder_id']}" + rval["full_path"] = full_path + rval["file_size"] = util.nice_size(int(ldda.get_size())) + rval["date_uploaded"] = ldda.create_time.strftime("%Y-%m-%d %I:%M %p") + rval["update_time"] = ldda.update_time.strftime("%Y-%m-%d %I:%M %p") + rval["can_user_modify"] = trans.user_is_admin or trans.app.security_agent.can_modify_library_item( + current_user_roles, ld + ) + rval["is_unrestricted"] = trans.app.security_agent.dataset_is_public(ldda.dataset) + rval["tags"] = self.tag_handler.get_tags_str(ldda.tags) # Manage dataset permission is always attached to the dataset itself, not the the ld or ldda to maintain consistency - rval['can_user_manage'] = trans.user_is_admin or trans.app.security_agent.can_manage_dataset(current_user_roles, ldda.dataset) + rval["can_user_manage"] = trans.user_is_admin or trans.app.security_agent.can_manage_dataset( + current_user_roles, ldda.dataset + ) return rval def _build_path(self, trans, folder): diff --git a/lib/galaxy/managers/licenses.py b/lib/galaxy/managers/licenses.py index 8af0d02e39d..dae5c996287 100644 --- a/lib/galaxy/managers/licenses.py +++ b/lib/galaxy/managers/licenses.py @@ -6,7 +6,7 @@ from pkg_resources import resource_string from pydantic import ( BaseModel, Field, - HttpUrl + HttpUrl, ) from galaxy import exceptions @@ -16,57 +16,33 @@ log = logging.getLogger(__name__) # https://github.com/spdx/license-list-data/blob/master/accessingLicenses.md#license-list-table-of-contents class LicenseMetadataModel(BaseModel): - licenseId: str = Field( - title="Identifier", - description="SPDX Identifier", - example="Apache-2.0" - ) - name: str = Field( - title="Name", - description="Full name of the license", - example="Apache License 2.0" - ) + licenseId: str = Field(title="Identifier", description="SPDX Identifier", example="Apache-2.0") + name: str = Field(title="Name", description="Full name of the license", example="Apache License 2.0") reference: str = Field( - title="Reference", - description="Reference to the HTML format for the license file", - example="./Apache-2.0.html" + title="Reference", description="Reference to the HTML format for the license file", example="./Apache-2.0.html" ) referenceNumber: int = Field( - title="Reference number", - description="*Deprecated* - this field is generated and is no longer in use" + title="Reference number", description="*Deprecated* - this field is generated and is no longer in use" ) isDeprecatedLicenseId: bool = Field( - title="Deprecated License", - description="True if the entire license is deprecated", - example=False + title="Deprecated License", description="True if the entire license is deprecated", example=False ) isOsiApproved: bool = Field( title="OSI approved", description="Indicates if the [OSI](https://opensource.org/) has approved the license", - example=True + example=True, ) seeAlso: List[HttpUrl] = Field( - title="Reference URLs", - description="Cross reference URL pointing to additional copies of the license" + title="Reference URLs", description="Cross reference URL pointing to additional copies of the license" ) detailsUrl: HttpUrl = Field( title="Details URL", description="URL to the SPDX json details for this license", - example="http://spdx.org/licenses/Apache-2.0.json" - ) - recommended: bool = Field( - title="Recommended", - description="True if this license is recommended to be used" - ) - url: HttpUrl = Field( - title="URL", - description="License URL", - example="http://www.apache.org/licenses/LICENSE-2.0" - ) - spdxUrl: HttpUrl = Field( - title="SPDX URL", - example="https://spdx.org/licenses/Apache-2.0.html" + example="http://spdx.org/licenses/Apache-2.0.json", ) + recommended: bool = Field(title="Recommended", description="True if this license is recommended to be used") + url: HttpUrl = Field(title="URL", description="License URL", example="http://www.apache.org/licenses/LICENSE-2.0") + spdxUrl: HttpUrl = Field(title="SPDX URL", example="https://spdx.org/licenses/Apache-2.0.html") # https://docs.google.com/document/d/16vnRtDjrx5eHSl4jXs2vMaDTI6luyyLzU6xMvRHsnbI/edit#heading=h.1pihjj16olz2 @@ -85,7 +61,7 @@ RECOMMENDED_LICENSES = [ "MPL-2.0", "PDDL-1.0", ] -SPDX_LICENSES_STRING = resource_string(__name__, 'licenses.json').decode("UTF-8") +SPDX_LICENSES_STRING = resource_string(__name__, "licenses.json").decode("UTF-8") SPDX_LICENSES = json.loads(SPDX_LICENSES_STRING) for license in SPDX_LICENSES["licenses"]: license["recommended"] = license["licenseId"] in RECOMMENDED_LICENSES @@ -99,7 +75,6 @@ for license in SPDX_LICENSES["licenses"]: class LicensesManager: - def __init__(self): by_index = {} for spdx_license in self.index(): @@ -118,9 +93,7 @@ class LicensesManager: return self._by_index[uri] else: log.warning(f"Unknown license URI encountered [{uri}]") - return { - "url": uri - } + return {"url": uri} def get_licenses(self) -> List[LicenseMetadataModel]: return SPDX_LICENSES["licenses"] diff --git a/lib/galaxy/managers/markdown_parse.py b/lib/galaxy/managers/markdown_parse.py index 1408dd50954..e5ee92258e5 100644 --- a/lib/galaxy/managers/markdown_parse.py +++ b/lib/galaxy/managers/markdown_parse.py @@ -6,14 +6,16 @@ Galaxy Markdown. Keeping things isolated to allow re-use of these utilities in o projects (e.g. gxformat2). """ import re -from typing import cast, Dict, List, Union - - -BLOCK_FENCE_START = re.compile(r'```.*') -BLOCK_FENCE_END = re.compile(r'```[\s]*') -GALAXY_FLAVORED_MARKDOWN_CONTAINER_LINE_PATTERN = re.compile( - r"```\s*galaxy\s*" +from typing import ( + cast, + Dict, + List, + Union, ) + +BLOCK_FENCE_START = re.compile(r"```.*") +BLOCK_FENCE_END = re.compile(r"```[\s]*") +GALAXY_FLAVORED_MARKDOWN_CONTAINER_LINE_PATTERN = re.compile(r"```\s*galaxy\s*") VALID_CONTAINER_END_PATTERN = re.compile(r"^```\s*$") @@ -48,12 +50,12 @@ VALID_ARGUMENTS: Dict[str, Union[List[str], DynamicArguments]] = { "invocation_inputs": [], } GALAXY_FLAVORED_MARKDOWN_CONTAINERS = list(VALID_ARGUMENTS.keys()) -GALAXY_FLAVORED_MARKDOWN_CONTAINER_REGEX = r'(?P%s)' % "|".join(GALAXY_FLAVORED_MARKDOWN_CONTAINERS) +GALAXY_FLAVORED_MARKDOWN_CONTAINER_REGEX = r"(?P%s)" % "|".join(GALAXY_FLAVORED_MARKDOWN_CONTAINERS) -ARG_VAL_REGEX = r'''[\w_\-]+|\"[^\"]+\"|\'[^\']+\'''' -FUNCTION_ARG = r'\s*[\w\|]+\s*=\s*(?:%s)\s*' % ARG_VAL_REGEX +ARG_VAL_REGEX = r"""[\w_\-]+|\"[^\"]+\"|\'[^\']+\'""" +FUNCTION_ARG = r"\s*[\w\|]+\s*=\s*(?:%s)\s*" % ARG_VAL_REGEX # embed commas between arguments -FUNCTION_MULTIPLE_ARGS = fr'(?P{FUNCTION_ARG})(?P(?:,{FUNCTION_ARG})*)' +FUNCTION_MULTIPLE_ARGS = rf"(?P{FUNCTION_ARG})(?P(?:,{FUNCTION_ARG})*)" FUNCTION_MULTIPLE_ARGS_PATTERN = re.compile(FUNCTION_MULTIPLE_ARGS) FUNCTION_CALL_LINE_TEMPLATE = f"\\s*%s\\s*\\((?:{FUNCTION_MULTIPLE_ARGS})?\\)\\s*" GALAXY_MARKDOWN_FUNCTION_CALL_LINE = re.compile(FUNCTION_CALL_LINE_TEMPLATE % GALAXY_FLAVORED_MARKDOWN_CONTAINER_REGEX) @@ -75,7 +77,11 @@ def validate_galaxy_markdown(galaxy_markdown, internal=True): expecting_container_close = expecting_container_close_for is not None if not fenced and expecting_container_close: - invalid_line("[{line}] is not expected close line for [{expected_for}]", line=line, expected_for=expecting_container_close_for) + invalid_line( + "[{line}] is not expected close line for [{expected_for}]", + line=line, + expected_for=expecting_container_close_for, + ) continue elif not fenced: continue @@ -86,7 +92,11 @@ def validate_galaxy_markdown(galaxy_markdown, internal=True): elif open_fence and GALAXY_FLAVORED_MARKDOWN_CONTAINER_LINE_PATTERN.match(line): if expecting_container_close: if not VALID_CONTAINER_END_PATTERN.match(line): - invalid_line("Invalid command close line [{line}] for [{expected_for}]", line=line, expected_for=expecting_container_close_for) + invalid_line( + "Invalid command close line [{line}] for [{expected_for}]", + line=line, + expected_for=expecting_container_close_for, + ) # else closing container and we're done expecting_container_close_for = None function_calls = 0 @@ -159,6 +169,6 @@ def _split_markdown_lines(markdown): __all__ = ( - 'validate_galaxy_markdown', - 'GALAXY_MARKDOWN_FUNCTION_CALL_LINE', + "validate_galaxy_markdown", + "GALAXY_MARKDOWN_FUNCTION_CALL_LINE", ) diff --git a/lib/galaxy/managers/markdown_util.py b/lib/galaxy/managers/markdown_util.py index 26787fc452d..4e816663c26 100644 --- a/lib/galaxy/managers/markdown_util.py +++ b/lib/galaxy/managers/markdown_util.py @@ -28,6 +28,7 @@ from typing import ( import markdown import pkg_resources + try: import weasyprint except Exception: @@ -48,22 +49,27 @@ from galaxy.model.item_attrs import get_item_annotation_str from galaxy.model.orm.now import now from galaxy.schema import PdfDocumentType from galaxy.util.sanitize_html import sanitize_html -from .markdown_parse import GALAXY_MARKDOWN_FUNCTION_CALL_LINE, validate_galaxy_markdown +from .markdown_parse import ( + GALAXY_MARKDOWN_FUNCTION_CALL_LINE, + validate_galaxy_markdown, +) log = logging.getLogger(__name__) -ARG_VAL_CAPTURED_REGEX = r'''(?:([\w_\-\|]+)|\"([^\"]+)\"|\'([^\']+)\')''' -OUTPUT_LABEL_PATTERN = re.compile(r'output=\s*%s\s*' % ARG_VAL_CAPTURED_REGEX) -INPUT_LABEL_PATTERN = re.compile(r'input=\s*%s\s*' % ARG_VAL_CAPTURED_REGEX) -STEP_LABEL_PATTERN = re.compile(r'step=\s*%s\s*' % ARG_VAL_CAPTURED_REGEX) -PATH_LABEL_PATTERN = re.compile(r'path=\s*%s\s*' % ARG_VAL_CAPTURED_REGEX) +ARG_VAL_CAPTURED_REGEX = r"""(?:([\w_\-\|]+)|\"([^\"]+)\"|\'([^\']+)\')""" +OUTPUT_LABEL_PATTERN = re.compile(r"output=\s*%s\s*" % ARG_VAL_CAPTURED_REGEX) +INPUT_LABEL_PATTERN = re.compile(r"input=\s*%s\s*" % ARG_VAL_CAPTURED_REGEX) +STEP_LABEL_PATTERN = re.compile(r"step=\s*%s\s*" % ARG_VAL_CAPTURED_REGEX) +PATH_LABEL_PATTERN = re.compile(r"path=\s*%s\s*" % ARG_VAL_CAPTURED_REGEX) # STEP_OUTPUT_LABEL_PATTERN = re.compile(r'step_output=([\w_\-]+)/([\w_\-]+)') -UNENCODED_ID_PATTERN = re.compile(r'(history_id|workflow_id|history_dataset_id|history_dataset_collection_id|job_id|invocation_id)=([\d]+)') -ENCODED_ID_PATTERN = re.compile(r'(history_id|workflow_id|history_dataset_id|history_dataset_collection_id|job_id|invocation_id)=([a-z0-9]+)') -INVOCATION_SECTION_MARKDOWN_CONTAINER_LINE_PATTERN = re.compile( - r"```\s*galaxy\s*" +UNENCODED_ID_PATTERN = re.compile( + r"(history_id|workflow_id|history_dataset_id|history_dataset_collection_id|job_id|invocation_id)=([\d]+)" ) -GALAXY_FENCED_BLOCK = re.compile(r'^```\s*galaxy\s*(.*?)^```', re.MULTILINE ^ re.DOTALL) +ENCODED_ID_PATTERN = re.compile( + r"(history_id|workflow_id|history_dataset_id|history_dataset_collection_id|job_id|invocation_id)=([a-z0-9]+)" +) +INVOCATION_SECTION_MARKDOWN_CONTAINER_LINE_PATTERN = re.compile(r"```\s*galaxy\s*") +GALAXY_FENCED_BLOCK = re.compile(r"^```\s*galaxy\s*(.*?)^```", re.MULTILINE ^ re.DOTALL) VALID_CONTAINER_START_PATTERN = re.compile(r"^```\s+[\w]+.*$") @@ -86,7 +92,6 @@ def ready_galaxy_markdown_for_import(trans, external_galaxy_markdown): class GalaxyInternalMarkdownDirectiveHandler(metaclass=abc.ABCMeta): - def walk(self, trans, internal_galaxy_markdown): hda_manager = trans.app.hda_manager history_manager = trans.app.history_manager @@ -264,7 +269,6 @@ class GalaxyInternalMarkdownDirectiveHandler(metaclass=abc.ABCMeta): class ReadyForExportMarkdownDirectiveHandler(GalaxyInternalMarkdownDirectiveHandler): - def __init__(self, trans, extra_rendering_data=None): extra_rendering_data = extra_rendering_data or {} self.trans = trans @@ -300,9 +304,7 @@ class ReadyForExportMarkdownDirectiveHandler(GalaxyInternalMarkdownDirectiveHand def handle_dataset_collection_display(self, line, hdca): hdca_serializer = HDCASerializer(self.trans.app) - hdca_view = hdca_serializer.serialize_to_view( - hdca, user=self.trans.user, trans=self.trans, view="summary" - ) + hdca_view = hdca_serializer.serialize_to_view(hdca, user=self.trans.user, trans=self.trans, view="summary") self.ensure_rendering_data_for("history_dataset_collections", hdca).update(hdca_view) def handle_tool_stdout(self, line, job): @@ -345,11 +347,13 @@ class ReadyForExportMarkdownDirectiveHandler(GalaxyInternalMarkdownDirectiveHand def handle_error(self, container, line, error): if "errors" not in self.extra_rendering_data: self.extra_rendering_data["errors"] = [] - self.extra_rendering_data["errors"].append({ - "error": error, - "line": line, - "container": container, - }) + self.extra_rendering_data["errors"].append( + { + "error": error, + "line": line, + "container": container, + } + ) return (line, False) @@ -373,17 +377,16 @@ def ready_galaxy_markdown_for_export(trans, internal_galaxy_markdown): class ToBasicMarkdownDirectiveHandler(GalaxyInternalMarkdownDirectiveHandler): - def __init__(self, trans, markdown_formatting_helpers): self.trans = trans self.markdown_formatting_helpers = markdown_formatting_helpers def handle_dataset_display(self, line, hda): name = hda.name or "" - markdown = '---\n' + markdown = "---\n" markdown += f"**Dataset:** {name}\n\n" markdown += self._display_dataset_content(hda) - markdown += '\n---\n' + markdown += "\n---\n" return (markdown, True) def handle_dataset_embedded(self, line, hda): @@ -408,7 +411,7 @@ class ToBasicMarkdownDirectiveHandler(GalaxyInternalMarkdownDirectiveHandler): def handle_dataset_as_image(self, line, hda): dataset = hda.dataset - name = hda.name or '' + name = hda.name or "" path_match = re.search(PATH_LABEL_PATTERN, line) if path_match: @@ -445,7 +448,7 @@ class ToBasicMarkdownDirectiveHandler(GalaxyInternalMarkdownDirectiveHandler): def handle_workflow_display(self, line, stored_workflow): # workflows/display.mako as markdown... meh... - markdown = '---\n' + markdown = "---\n" markdown += f"**Workflow:** {stored_workflow.name}\n\n" markdown += "**Steps:**\n\n" markdown += "|Step|Annotation|\n" @@ -453,7 +456,7 @@ class ToBasicMarkdownDirectiveHandler(GalaxyInternalMarkdownDirectiveHandler): # Pass two should add tool information, labels, etc.. but # it requires module_injector and such. for order_index, step in enumerate(stored_workflow.latest_workflow.steps): - annotation = get_item_annotation_str(self.trans.sa_session, self.trans.user, step) or '' + annotation = get_item_annotation_str(self.trans.sa_session, self.trans.user, step) or "" markdown += "|{}|{}|\n".format(step.label or "Step %d" % (order_index + 1), annotation) markdown += "\n---\n" return (markdown, True) @@ -471,8 +474,9 @@ class ToBasicMarkdownDirectiveHandler(GalaxyInternalMarkdownDirectiveHandler): for element in collection.elements: markdown_wrapper[0] += f"**Element:** {element_prefix}{element.element_identifier}\n\n" markdown_wrapper[0] += self._display_dataset_content(element.hda, header="Element Contents") + walk_elements(hdca.collection) - markdown = f'---\n{markdown_wrapper[0]}\n---\n' + markdown = f"---\n{markdown_wrapper[0]}\n---\n" return (markdown, True) def handle_tool_stdout(self, line, job): @@ -571,8 +575,7 @@ class MarkdownFormatHelpers: def to_basic_markdown(trans, internal_galaxy_markdown: str) -> str: - """Replace Galaxy Markdown extensions with plain Markdown for PDF/HTML export. - """ + """Replace Galaxy Markdown extensions with plain Markdown for PDF/HTML export.""" markdown_formatting_helpers = MarkdownFormatHelpers() directive_handler = ToBasicMarkdownDirectiveHandler(trans, markdown_formatting_helpers) plain_markdown = directive_handler.walk(trans, internal_galaxy_markdown) @@ -589,14 +592,14 @@ def to_pdf_raw(basic_markdown: str, css_paths: Optional[List[str]] = None) -> by """Convert RAW markdown with specified CSS paths into bytes of a PDF.""" css_paths = css_paths or [] as_html = to_html(basic_markdown) - directory = tempfile.mkdtemp('gxmarkdown') + directory = tempfile.mkdtemp("gxmarkdown") index = os.path.join(directory, "index.html") try: output_file = codecs.open(index, "w", encoding="utf-8", errors="xmlcharrefreplace") output_file.write(as_html) output_file.close() html = weasyprint.HTML(filename=index) - stylesheets = [weasyprint.CSS(string=pkg_resources.resource_string(__name__, 'markdown_export_base.css'))] + stylesheets = [weasyprint.CSS(string=pkg_resources.resource_string(__name__, "markdown_export_base.css"))] for css_path in css_paths: with open(css_path) as f: css_content = f.read() @@ -625,10 +628,10 @@ def internal_galaxy_markdown_to_pdf(trans, internal_galaxy_markdown: str, docume def to_branded_pdf(basic_markdown: str, document_type: PdfDocumentType, config: GalaxyAppConfiguration) -> bytes: - document_type_prologue = getattr(config, f"markdown_export_prologue_{document_type}s", '') or '' - document_type_epilogue = getattr(config, f"markdown_export_epilogue_{document_type}s", '') or '' - general_prologue = config.markdown_export_prologue or '' - general_epilogue = config.markdown_export_epilogue or '' + document_type_prologue = getattr(config, f"markdown_export_prologue_{document_type}s", "") or "" + document_type_epilogue = getattr(config, f"markdown_export_epilogue_{document_type}s", "") or "" + general_prologue = config.markdown_export_prologue or "" + general_epilogue = config.markdown_export_epilogue or "" effective_prologue = document_type_prologue or general_prologue effective_epilogue = document_type_epilogue or general_epilogue branded_markdown = effective_prologue + basic_markdown + effective_epilogue @@ -671,13 +674,17 @@ def resolve_invocation_markdown(trans, invocation, workflow_markdown): ```galaxy history_dataset_display(output="{}") ``` -""".format(output_assoc.workflow_output.label, output_assoc.workflow_output.label) +""".format( + output_assoc.workflow_output.label, output_assoc.workflow_output.label + ) else: section_markdown += """#### Output Dataset Collection: {} ```galaxy history_dataset_collection_display(output="{}") ``` -""".format(output_assoc.workflow_output.label, output_assoc.workflow_output.label) +""".format( + output_assoc.workflow_output.label, output_assoc.workflow_output.label + ) elif container == "invocation_inputs": for input_assoc in invocation.input_associations: if not input_assoc.workflow_step.label: @@ -688,13 +695,17 @@ history_dataset_collection_display(output="{}") ```galaxy history_dataset_display(input="{}") ``` -""".format(input_assoc.workflow_step.label, input_assoc.workflow_step.label) +""".format( + input_assoc.workflow_step.label, input_assoc.workflow_step.label + ) else: section_markdown += """#### Input Dataset Collection: {} ```galaxy history_dataset_collection_display(input={}) ``` -""".format(input_assoc.workflow_step.label, input_assoc.workflow_step.label) +""".format( + input_assoc.workflow_step.label, input_assoc.workflow_step.label + ) else: return line, False return section_markdown, True @@ -781,7 +792,6 @@ def _remap_galaxy_markdown_containers(func, markdown): def _remap_galaxy_markdown_calls(func, markdown): - def _remap_container(container): matching_line = None for line in container.splitlines(): @@ -808,8 +818,8 @@ def _validate(*args, **kwds): __all__ = ( - 'internal_galaxy_markdown_to_pdf', - 'ready_galaxy_markdown_for_export', - 'ready_galaxy_markdown_for_import', - 'resolve_invocation_markdown', + "internal_galaxy_markdown_to_pdf", + "ready_galaxy_markdown_for_export", + "ready_galaxy_markdown_for_import", + "resolve_invocation_markdown", ) diff --git a/lib/galaxy/managers/metrics.py b/lib/galaxy/managers/metrics.py index 6ff09336451..5e49dfd241b 100644 --- a/lib/galaxy/managers/metrics.py +++ b/lib/galaxy/managers/metrics.py @@ -36,7 +36,7 @@ class Metric(BaseModel): ..., # Required title="Timestamp", description="The timestamp in ISO format.", - example=datetime_to_iso8601(datetime.utcnow()) + example=datetime_to_iso8601(datetime.utcnow()), ) level: int = Field( ..., # Required @@ -53,15 +53,12 @@ class Metric(BaseModel): class CreateMetricsPayload(BaseModel): metrics: List[Metric] = Field( default=[], - title='List of metrics to be recorded.', + title="List of metrics to be recorded.", example=[ Metric( - namespace="test-source", - time=datetime_to_iso8601(datetime.utcnow()), - level=0, - args='{"test":"value"}' + namespace="test-source", time=datetime_to_iso8601(datetime.utcnow()), level=0, args='{"test":"value"}' ) - ] + ], ) @@ -103,10 +100,7 @@ class MetricsManager: return response def _parse_metrics( - self, - metrics: Optional[List[Metric]] = None, - user_id=None, - session_id=None + self, metrics: Optional[List[Metric]] = None, user_id=None, session_id=None ) -> TimeSeriesTupleGenerator: """ Return a generator yielding the each given metric as a tuple: @@ -122,12 +116,7 @@ class MetricsManager: for metric in metrics: label = metric.namespace time = self._deserialize_isoformat_date(metric.time) - kwargs = { - 'level': metric.level, - 'args': metric.args, - 'user': user_id, - 'session': session_id - } + kwargs = {"level": metric.level, "args": metric.args, "user": user_id, "session": session_id} yield (label, time, kwargs) def _send_metrics(self, trans, metrics: TimeSeriesTupleGenerator) -> None: @@ -142,7 +131,7 @@ class MetricsManager: trans.app.trace_logger.log(label, event_time=int(time.timestamp()), **kwargs) elif self.debugging: for label, time, kwargs in metrics: - log.debug(f'{label} {time} {kwargs}') + log.debug(f"{label} {time} {kwargs}") def _get_server_pong(self, trans) -> Any: """ diff --git a/lib/galaxy/managers/pages.py b/lib/galaxy/managers/pages.py index 0d599be6782..57a575771d2 100644 --- a/lib/galaxy/managers/pages.py +++ b/lib/galaxy/managers/pages.py @@ -9,12 +9,16 @@ import logging import re from html.entities import name2codepoint from html.parser import HTMLParser -from typing import ( - Callable, -) +from typing import Callable -from galaxy import exceptions, model -from galaxy.managers import base, sharable +from galaxy import ( + exceptions, + model, +) +from galaxy.managers import ( + base, + sharable, +) from galaxy.managers.context import ProvidesHistoryContext from galaxy.managers.markdown_util import ( ready_galaxy_markdown_for_export, @@ -30,33 +34,33 @@ log = logging.getLogger(__name__) # Copied from https://github.com/kurtmckee/feedparser _cp1252 = { - 128: '\u20ac', # euro sign - 130: '\u201a', # single low-9 quotation mark - 131: '\u0192', # latin small letter f with hook - 132: '\u201e', # double low-9 quotation mark - 133: '\u2026', # horizontal ellipsis - 134: '\u2020', # dagger - 135: '\u2021', # double dagger - 136: '\u02c6', # modifier letter circumflex accent - 137: '\u2030', # per mille sign - 138: '\u0160', # latin capital letter s with caron - 139: '\u2039', # single left-pointing angle quotation mark - 140: '\u0152', # latin capital ligature oe - 142: '\u017d', # latin capital letter z with caron - 145: '\u2018', # left single quotation mark - 146: '\u2019', # right single quotation mark - 147: '\u201c', # left double quotation mark - 148: '\u201d', # right double quotation mark - 149: '\u2022', # bullet - 150: '\u2013', # en dash - 151: '\u2014', # em dash - 152: '\u02dc', # small tilde - 153: '\u2122', # trade mark sign - 154: '\u0161', # latin small letter s with caron - 155: '\u203a', # single right-pointing angle quotation mark - 156: '\u0153', # latin small ligature oe - 158: '\u017e', # latin small letter z with caron - 159: '\u0178', # latin capital letter y with diaeresis + 128: "\u20ac", # euro sign + 130: "\u201a", # single low-9 quotation mark + 131: "\u0192", # latin small letter f with hook + 132: "\u201e", # double low-9 quotation mark + 133: "\u2026", # horizontal ellipsis + 134: "\u2020", # dagger + 135: "\u2021", # double dagger + 136: "\u02c6", # modifier letter circumflex accent + 137: "\u2030", # per mille sign + 138: "\u0160", # latin capital letter s with caron + 139: "\u2039", # single left-pointing angle quotation mark + 140: "\u0152", # latin capital ligature oe + 142: "\u017d", # latin capital letter z with caron + 145: "\u2018", # left single quotation mark + 146: "\u2019", # right single quotation mark + 147: "\u201c", # left double quotation mark + 148: "\u201d", # right double quotation mark + 149: "\u2022", # bullet + 150: "\u2013", # en dash + 151: "\u2014", # em dash + 152: "\u02dc", # small tilde + 153: "\u2122", # trade mark sign + 154: "\u0161", # latin small letter s with caron + 155: "\u203a", # single right-pointing angle quotation mark + 156: "\u0153", # latin small ligature oe + 158: "\u017e", # latin small letter z with caron + 159: "\u0178", # latin capital letter y with diaeresis } @@ -64,7 +68,7 @@ class PageManager(sharable.SharableModelManager, UsesAnnotations): """Provides operations for managing a Page.""" model_class = model.Page - foreign_key_name = 'page' + foreign_key_name = "page" user_share_model = model.PageUserShareAssociation tag_assoc = model.PageTagAssociation @@ -72,8 +76,7 @@ class PageManager(sharable.SharableModelManager, UsesAnnotations): rating_assoc = model.PageRatingAssociation def __init__(self, app: MinimalManagerApp): - """ - """ + """ """ super().__init__(app) self.workflow_manager = app.workflow_manager @@ -85,8 +88,14 @@ class PageManager(sharable.SharableModelManager, UsesAnnotations): elif not payload.get("slug"): raise exceptions.ObjectAttributeMissingException("Page id is required") elif not base.is_valid_slug(payload["slug"]): - raise exceptions.ObjectAttributeInvalidException("Page identifier must consist of only lowercase letters, numbers, and the '-' character") - elif trans.sa_session.query(trans.app.model.Page).filter_by(user=user, slug=payload["slug"], deleted=False).first(): + raise exceptions.ObjectAttributeInvalidException( + "Page identifier must consist of only lowercase letters, numbers, and the '-' character" + ) + elif ( + trans.sa_session.query(trans.app.model.Page) + .filter_by(user=user, slug=payload["slug"], deleted=False) + .first() + ): raise exceptions.DuplicatedSlugException("Page identifier must be unique") if payload.get("invocation_id"): @@ -101,8 +110,8 @@ class PageManager(sharable.SharableModelManager, UsesAnnotations): # Create the new stored page page = trans.app.model.Page() - page.title = payload['title'] - page.slug = payload['slug'] + page.title = payload["title"] + page.slug = payload["slug"] page_annotation = payload.get("annotation", None) if page_annotation is not None: page_annotation = sanitize_html(page_annotation) @@ -111,7 +120,7 @@ class PageManager(sharable.SharableModelManager, UsesAnnotations): page.user = user # And the first (empty) page revision page_revision = trans.app.model.PageRevision() - page_revision.title = payload['title'] + page_revision.title = payload["title"] page_revision.page = page page.latest_revision = page_revision page_revision.content = content @@ -129,10 +138,12 @@ class PageManager(sharable.SharableModelManager, UsesAnnotations): if not content: raise exceptions.ObjectAttributeMissingException("content undefined or empty") if content_format not in [None, PageContentFormat.html.value, PageContentFormat.markdown.value]: - raise exceptions.RequestParameterInvalidException(f"content_format [{content_format}], if specified, must be either html or markdown") + raise exceptions.RequestParameterInvalidException( + f"content_format [{content_format}], if specified, must be either html or markdown" + ) - if 'title' in payload: - title = payload['title'] + if "title" in payload: + title = payload["title"] else: title = page.title @@ -159,7 +170,7 @@ class PageManager(sharable.SharableModelManager, UsesAnnotations): processor = PageContentProcessor(trans, placeholderRenderForSave) processor.feed(content) # Output is string, so convert to unicode for saving. - content = unicodify(processor.output(), 'utf-8') + content = unicodify(processor.output(), "utf-8") except exceptions.MessageException: raise except Exception: @@ -167,7 +178,9 @@ class PageManager(sharable.SharableModelManager, UsesAnnotations): elif content_format == PageContentFormat.markdown.value: content = ready_galaxy_markdown_for_import(trans, content) else: - raise exceptions.RequestParameterInvalidException(f"content_format [{content_format}] must be either html or markdown") + raise exceptions.RequestParameterInvalidException( + f"content_format [{content_format}] must be either html or markdown" + ) return content def rewrite_content_for_export(self, trans, as_dict): @@ -176,14 +189,16 @@ class PageManager(sharable.SharableModelManager, UsesAnnotations): if content_format == PageContentFormat.html.value: processor = PageContentProcessor(trans, placeholderRenderForEdit) processor.feed(content) - content = unicodify(processor.output(), 'utf-8') + content = unicodify(processor.output(), "utf-8") as_dict["content"] = content elif content_format == PageContentFormat.markdown.value: content, extra_attributes = ready_galaxy_markdown_for_export(trans, content) as_dict["content"] = content as_dict.update(extra_attributes) else: - raise exceptions.RequestParameterInvalidException(f"content_format [{content_format}] must be either html or markdown") + raise exceptions.RequestParameterInvalidException( + f"content_format [{content_format}] must be either html or markdown" + ) return as_dict @@ -191,21 +206,21 @@ class PageSerializer(sharable.SharableModelSerializer): """ Interface/service object for serializing pages into dictionaries. """ + model_manager_class = PageManager - SINGLE_CHAR_ABBR = 'p' + SINGLE_CHAR_ABBR = "p" def __init__(self, app: MinimalManagerApp): super().__init__(app) self.page_manager = PageManager(app) - self.default_view = 'summary' - self.add_view('summary', []) - self.add_view('detailed', []) + self.default_view = "summary" + self.add_view("summary", []) + self.add_view("detailed", []) def add_serializers(self): super().add_serializers() - self.serializers.update({ - }) + self.serializers.update({}) class PageDeserializer(sharable.SharableModelDeserializer): @@ -213,6 +228,7 @@ class PageDeserializer(sharable.SharableModelDeserializer): Interface/service object for validating and deserializing dictionaries into pages. """ + model_manager_class = PageManager def __init__(self, app: MinimalManagerApp): @@ -221,8 +237,7 @@ class PageDeserializer(sharable.SharableModelDeserializer): def add_deserializers(self): super().add_deserializers() - self.deserializers.update({ - }) + self.deserializers.update({}) self.deserializable_keyset.update(self.deserializers.keys()) @@ -231,11 +246,28 @@ class PageContentProcessor(HTMLParser): Processes page content to produce HTML that is suitable for display. For now, processor renders embedded objects. """ + bare_ampersand = re.compile(r"&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)") elements_no_end_tag = { - 'area', 'base', 'basefont', 'br', 'col', 'command', 'embed', 'frame', - 'hr', 'img', 'input', 'isindex', 'keygen', 'link', 'meta', 'param', - 'source', 'track', 'wbr' + "area", + "base", + "basefont", + "br", + "col", + "command", + "embed", + "frame", + "hr", + "img", + "input", + "isindex", + "keygen", + "link", + "meta", + "param", + "source", + "track", + "wbr", } def __init__(self, trans, render_embed_html_fn: Callable): @@ -257,10 +289,10 @@ class PageContentProcessor(HTMLParser): return f"<{tag}>" def feed(self, data): - data = re.compile(r'\s]+?)\s*/>', self._shorttag_replace, data) - data = data.replace(''', "'") - data = data.replace('"', '"') + data = re.compile(r"\s]+?)\s*/>", self._shorttag_replace, data) + data = data.replace("'", "'") + data = data.replace(""", '"') HTMLParser.feed(self, data) HTMLParser.close(self) @@ -301,17 +333,17 @@ class PageContentProcessor(HTMLParser): # Default behavior: not ignoring and no embedded content. uattrs = [] - strattrs = '' + strattrs = "" if attrs: for key, value in attrs: - value = value.replace('>', '>').replace('<', '<').replace('"', '"') + value = value.replace(">", ">").replace("<", "<").replace('"', """) value = self.bare_ampersand.sub("&", value) uattrs.append((key, value)) - strattrs = ''.join(f' {k}="{v}"' for k, v in uattrs) + strattrs = "".join(f' {k}="{v}"' for k, v in uattrs) if tag in self.elements_no_end_tag: - self.pieces.append(f'<{tag}{strattrs} />') + self.pieces.append(f"<{tag}{strattrs} />") else: - self.pieces.append(f'<{tag}{strattrs}>') + self.pieces.append(f"<{tag}{strattrs}>") def handle_endtag(self, tag): """ @@ -335,23 +367,23 @@ class PageContentProcessor(HTMLParser): # called for each character reference, e.g. for ' ', ref will be '160' # Reconstruct the original character reference. ref = ref.lower() - if ref.startswith('x'): + if ref.startswith("x"): value = int(ref[1:], 16) else: value = int(ref) if value in _cp1252: - self.pieces.append(f'&#{hex(ord(_cp1252[value]))[1:]};') + self.pieces.append(f"&#{hex(ord(_cp1252[value]))[1:]};") else: - self.pieces.append(f'&#{ref};') + self.pieces.append(f"&#{ref};") def handle_entityref(self, ref): # called for each entity reference, e.g. for '©', ref will be 'copy' # Reconstruct the original entity reference. - if ref in name2codepoint or ref == 'apos': - self.pieces.append(f'&{ref};') + if ref in name2codepoint or ref == "apos": + self.pieces.append(f"&{ref};") else: - self.pieces.append(f'&{ref}') + self.pieces.append(f"&{ref}") def handle_data(self, text): """ @@ -367,23 +399,23 @@ class PageContentProcessor(HTMLParser): def handle_comment(self, text): # called for each HTML comment, e.g. # Reconstruct the original comment. - self.pieces.append(f'') + self.pieces.append(f"") def handle_decl(self, text): # called for the DOCTYPE, if present, e.g. # # Reconstruct original DOCTYPE - self.pieces.append(f'') + self.pieces.append(f"") def handle_pi(self, text): # called for each processing instruction, e.g. # Reconstruct original processing instruction. - self.pieces.append(f'') + self.pieces.append(f"") def output(self): - '''Return processed HTML as a single string''' - return ''.join(self.pieces) + """Return processed HTML as a single string""" + return "".join(self.pieces) PAGE_MAXRAW = 10**15 @@ -404,14 +436,14 @@ def get_page_identifiers(item_id, app): # Utilities for encoding/decoding HTML content. -PLACEHOLDER_TEMPLATE = '''

                Embedded Galaxy {class_shorthand} - '{item_name}'

                [Do not edit this block; Galaxy will fill it in with the annotated {class_shorthand} when it is displayed]

                ''' +PLACEHOLDER_TEMPLATE = """

                Embedded Galaxy {class_shorthand} - '{item_name}'

                [Do not edit this block; Galaxy will fill it in with the annotated {class_shorthand} when it is displayed]

                """ # This is a mapping of the id portion of page contents to the cssclass/shortname. PAGE_CLASS_MAPPING = { - 'History': 'History', - 'HistoryDatasetAssociation': 'Dataset', - 'StoredWorkflow': 'Workflow', - 'Visualization': 'Visualization' + "History": "History", + "HistoryDatasetAssociation": "Dataset", + "StoredWorkflow": "Workflow", + "Visualization": "Visualization", } @@ -421,21 +453,21 @@ def placeholderRenderForEdit(trans: ProvidesHistoryContext, item_class, item_id) def placeholderRenderForSave(trans: ProvidesHistoryContext, item_class, item_id, encode=False): encoded_item_id, decoded_item_id = get_page_identifiers(item_id, trans.app) - item_name = '' - if item_class == 'History': + item_name = "" + if item_class == "History": history = trans.sa_session.query(model.History).get(decoded_item_id) history = base.security_check(trans, history, False, True) item_name = history.name - elif item_class == 'HistoryDatasetAssociation': + elif item_class == "HistoryDatasetAssociation": hda = trans.sa_session.query(model.HistoryDatasetAssociation).get(decoded_item_id) hda_manager = trans.app.hda_manager hda = hda_manager.get_accessible(decoded_item_id, trans.user) item_name = hda.name - elif item_class == 'StoredWorkflow': + elif item_class == "StoredWorkflow": wf = trans.sa_session.query(model.StoredWorkflow).get(decoded_item_id) wf = base.security_check(trans, wf, False, True) item_name = wf.name - elif item_class == 'Visualization': + elif item_class == "Visualization": visualization = trans.sa_session.query(model.Visualization).get(decoded_item_id) visualization = base.security_check(trans, visualization, False, True) item_name = visualization.title @@ -449,5 +481,5 @@ def placeholderRenderForSave(trans: ProvidesHistoryContext, item_class, item_id, class_shorthand=class_shorthand, class_shorthand_lower=class_shorthand.lower(), item_id=item_id, - item_name=item_name + item_name=item_name, ) diff --git a/lib/galaxy/managers/quotas.py b/lib/galaxy/managers/quotas.py index 90194d862a8..e0a71eb5c5c 100644 --- a/lib/galaxy/managers/quotas.py +++ b/lib/galaxy/managers/quotas.py @@ -11,7 +11,10 @@ from typing import ( Union, ) -from galaxy import model, util +from galaxy import ( + model, + util, +) from galaxy.app import StructuredApp from galaxy.exceptions import ActionInputError from galaxy.managers import base @@ -44,7 +47,9 @@ class QuotaManager: params = CreateQuotaParams.parse_obj(payload) create_amount = self._parse_amount(params.amount) if self.sa_session.query(model.Quota).filter(model.Quota.name == params.name).first(): - raise ActionInputError("Quota names must be unique and a quota with that name already exists, please choose another name.") + raise ActionInputError( + "Quota names must be unique and a quota with that name already exists, please choose another name." + ) elif create_amount is False: raise ActionInputError("Unable to parse the provided amount.") elif params.operation not in model.Quota.valid_operations: @@ -54,7 +59,9 @@ class QuotaManager: elif create_amount is None and params.operation != QuotaOperation.EXACT: raise ActionInputError("Operation for an unlimited quota must be '='.") # Create the quota - quota = model.Quota(name=params.name, description=params.description, amount=create_amount, operation=params.operation) + quota = model.Quota( + name=params.name, description=params.description, amount=create_amount, operation=params.operation + ) self.sa_session.add(quota) # If this is a default quota, create the DefaultQuotaAssociation if params.default != DefaultQuotaValues.NO: @@ -62,8 +69,14 @@ class QuotaManager: message = f"Default quota '{quota.name}' has been created." else: # Create the UserQuotaAssociations - in_users = [self.sa_session.query(model.User).get(decode_id(x) if decode_id else x) for x in util.listify(params.in_users)] - in_groups = [self.sa_session.query(model.Group).get(decode_id(x) if decode_id else x) for x in util.listify(params.in_groups)] + in_users = [ + self.sa_session.query(model.User).get(decode_id(x) if decode_id else x) + for x in util.listify(params.in_users) + ] + in_groups = [ + self.sa_session.query(model.Group).get(decode_id(x) if decode_id else x) + for x in util.listify(params.in_groups) + ] if None in in_users: raise ActionInputError("One or more invalid user id has been provided.") for user in in_users: @@ -80,7 +93,7 @@ class QuotaManager: return quota, message def _parse_amount(self, amount: str) -> Optional[Union[int, bool]]: - if amount.lower() in ('unlimited', 'none', 'no limit'): + if amount.lower() in ("unlimited", "none", "no limit"): return None try: return util.size_to_bytes(amount) @@ -89,9 +102,12 @@ class QuotaManager: def rename_quota(self, quota, params) -> str: if not params.name: - raise ActionInputError('Enter a valid name.') - elif params.name != quota.name and self.sa_session.query(model.Quota).filter(model.Quota.name == params.name).first(): - raise ActionInputError('A quota with that name already exists.') + raise ActionInputError("Enter a valid name.") + elif ( + params.name != quota.name + and self.sa_session.query(model.Quota).filter(model.Quota.name == params.name).first() + ): + raise ActionInputError("A quota with that name already exists.") else: old_name = quota.name quota.name = params.name @@ -104,12 +120,18 @@ class QuotaManager: def manage_users_and_groups_for_quota(self, quota, params, decode_id=None) -> str: if quota.default: - raise ActionInputError('Default quotas cannot be associated with specific users and groups.') + raise ActionInputError("Default quotas cannot be associated with specific users and groups.") else: - in_users = [self.sa_session.query(model.User).get(decode_id(x) if decode_id else x) for x in util.listify(params.in_users)] + in_users = [ + self.sa_session.query(model.User).get(decode_id(x) if decode_id else x) + for x in util.listify(params.in_users) + ] if None in in_users: raise ActionInputError("One or more invalid user id has been provided.") - in_groups = [self.sa_session.query(model.Group).get(decode_id(x) if decode_id else x) for x in util.listify(params.in_groups)] + in_groups = [ + self.sa_session.query(model.Group).get(decode_id(x) if decode_id else x) + for x in util.listify(params.in_groups) + ] if None in in_groups: raise ActionInputError("One or more invalid group id has been provided.") self.quota_agent.set_entity_quota_associations(quotas=[quota], users=in_users, groups=in_groups) @@ -118,7 +140,7 @@ class QuotaManager: return message def edit_quota(self, quota, params) -> str: - if params.amount.lower() in ('unlimited', 'none', 'no limit'): + if params.amount.lower() in ("unlimited", "none", "no limit"): new_amount = None else: try: @@ -126,11 +148,11 @@ class QuotaManager: except (AssertionError, ValueError): new_amount = False if not params.amount: - raise ActionInputError('Enter a valid amount.') + raise ActionInputError("Enter a valid amount.") elif new_amount is False: - raise ActionInputError('Unable to parse the provided amount.') + raise ActionInputError("Unable to parse the provided amount.") elif params.operation not in model.Quota.valid_operations: - raise ActionInputError('Enter a valid operation.') + raise ActionInputError("Enter a valid operation.") else: quota.amount = new_amount quota.operation = params.operation @@ -140,10 +162,10 @@ class QuotaManager: return message def set_quota_default(self, quota, params) -> str: - if params.default != 'no' and params.default not in model.DefaultQuotaAssociation.types.__members__.values(): - raise ActionInputError('Enter a valid default type.') + if params.default != "no" and params.default not in model.DefaultQuotaAssociation.types.__members__.values(): + raise ActionInputError("Enter a valid default type.") else: - if params.default != 'no': + if params.default != "no": self.quota_agent.set_default_quota(params.default, quota) message = f"Quota '{quota.name}' is now the default for {params.default} users." else: @@ -175,14 +197,16 @@ class QuotaManager: if len(names) == 1: raise ActionInputError(f"Quota '{names[0]}' is a default, please unset it as a default before deleting it.") elif len(names) > 1: - raise ActionInputError(f"Quotas are defaults, please unset them as defaults before deleting them: {', '.join(names)}") + raise ActionInputError( + f"Quotas are defaults, please unset them as defaults before deleting them: {', '.join(names)}" + ) message = f"Deleted {len(quotas)} quotas: " for q in quotas: q.deleted = True self.sa_session.add(q) names.append(q.name) self.sa_session.flush() - message += ', '.join(names) + message += ", ".join(names) return message def undelete_quota(self, quota, params=None) -> str: @@ -201,7 +225,7 @@ class QuotaManager: self.sa_session.add(q) names.append(q.name) self.sa_session.flush() - message += ', '.join(names) + message += ", ".join(names) return message def purge_quota(self, quota, params=None): @@ -230,8 +254,8 @@ class QuotaManager: self.sa_session.delete(gqa) names.append(q.name) self.sa_session.flush() - message += ', '.join(names) + message += ", ".join(names) return message def get_quota(self, trans, id: EncodedDatabaseIdField, deleted: Optional[bool] = None) -> model.Quota: - return base.get_object(trans, id, 'Quota', check_ownership=False, check_accessible=False, deleted=deleted) + return base.get_object(trans, id, "Quota", check_ownership=False, check_accessible=False, deleted=deleted) diff --git a/lib/galaxy/managers/ratable.py b/lib/galaxy/managers/ratable.py index b86c5505e3d..e2bb1f08fea 100644 --- a/lib/galaxy/managers/ratable.py +++ b/lib/galaxy/managers/ratable.py @@ -62,16 +62,16 @@ class RatableManagerMixin: class RatableSerializerMixin: - def add_serializers(self): - self.serializers['user_rating'] = self.serialize_user_rating - self.serializers['community_rating'] = self.serialize_community_rating + self.serializers["user_rating"] = self.serialize_user_rating + self.serializers["community_rating"] = self.serialize_community_rating def serialize_user_rating(self, item, key, user=None, **context): """Returns the integer rating given to this item by the user.""" if not user: - raise base.ModelSerializingError('user_rating requires a user', - model_class=self.manager.model_class, id=self.serialize_id(item, 'id')) + raise base.ModelSerializingError( + "user_rating requires a user", model_class=self.manager.model_class, id=self.serialize_id(item, "id") + ) return self.manager.rating(item, user) def serialize_community_rating(self, item, key, **context): @@ -84,26 +84,25 @@ class RatableSerializerMixin: # than getting the rows and calc'ing both here with one query manager = self.manager return { - 'average': manager.ratings_avg(item), - 'count': manager.ratings_count(item), + "average": manager.ratings_avg(item), + "count": manager.ratings_count(item), } class RatableDeserializerMixin: - def add_deserializers(self): - self.deserializers['user_rating'] = self.deserialize_rating + self.deserializers["user_rating"] = self.deserialize_rating def deserialize_rating(self, item, key, val, user=None, **context): if not user: - raise base.ModelDeserializingError('user_rating requires a user', - model_class=self.manager.model_class, id=self.serialize_id(item, 'id')) + raise base.ModelDeserializingError( + "user_rating requires a user", model_class=self.manager.model_class, id=self.serialize_id(item, "id") + ) val = self.validate.int_range(key, val, 0, 5) return self.manager.rate(item, user, val, flush=False) class RatableFilterMixin: - def _ratings_avg_accessor(self, item): return self.manager.ratings_avg(item) @@ -112,14 +111,16 @@ class RatableFilterMixin: Adds the following filters: `community_rating`: filter """ - self.fn_filter_parsers.update({ - 'community_rating': { - 'op': { - 'eq': lambda i, v: self._ratings_avg_accessor(i) == v, - # TODO: default to greater than (currently 'eq' due to base/controller.py) - 'ge': lambda i, v: self._ratings_avg_accessor(i) >= v, - 'le': lambda i, v: self._ratings_avg_accessor(i) <= v, - }, - 'val': float + self.fn_filter_parsers.update( + { + "community_rating": { + "op": { + "eq": lambda i, v: self._ratings_avg_accessor(i) == v, + # TODO: default to greater than (currently 'eq' due to base/controller.py) + "ge": lambda i, v: self._ratings_avg_accessor(i) >= v, + "le": lambda i, v: self._ratings_avg_accessor(i) <= v, + }, + "val": float, + } } - }) + ) diff --git a/lib/galaxy/managers/rbac_secured.py b/lib/galaxy/managers/rbac_secured.py index 3587089b6b4..ccbc81d0264 100644 --- a/lib/galaxy/managers/rbac_secured.py +++ b/lib/galaxy/managers/rbac_secured.py @@ -3,7 +3,7 @@ import logging import galaxy.exceptions from galaxy import ( model, - security + security, ) from galaxy.managers import users @@ -38,7 +38,7 @@ class RBACPermission: def error_unless_permitted(self, item, user, trans=None): if not self.is_permitted(item, user, trans=trans): - error_info = dict(model_class=item.__class__, id=getattr(item, 'id', None)) + error_info = dict(model_class=item.__class__, id=getattr(item, "id", None)) raise self.permission_failed_error_class(**error_info) def grant(self, item, user, flush=True): @@ -52,7 +52,7 @@ class RBACPermission: def _error_unless_role_permitted(self, item, role): if not self._role_is_permitted(item, role): - error_info = dict(model_class=item.__class__, id=getattr(item, 'id', None)) + error_info = dict(model_class=item.__class__, id=getattr(item, "id", None)) raise self.permission_failed_error_class(**error_info) def _grant_role(self, item, role, flush=True): @@ -75,6 +75,7 @@ class DatasetRBACPermission(RBACPermission): - manage permissions : can a role manage the permissions on a dataset - access : can a role read/look at/copy a dataset """ + permissions_class = model.DatasetPermissions action_name = None @@ -167,8 +168,9 @@ class ManageDatasetRBACPermission(DatasetRBACPermission): When checking permissions for a user, if any of the user's roles have permission on the dataset """ + # TODO: We may also be able to infer/record the dataset 'owner' as well. - action_name = security.RBACAgent.permitted_actions.get('DATASET_MANAGE_PERMISSIONS').action + action_name = security.RBACAgent.permitted_actions.get("DATASET_MANAGE_PERMISSIONS").action permission_failed_error_class = DatasetManagePermissionFailedException # ---- interface @@ -227,7 +229,8 @@ class AccessDatasetRBACPermission(DatasetRBACPermission): An user must have all the Roles of all the access permissions associated with a dataset in order to access it. """ - action_name = security.RBACAgent.permitted_actions.get('DATASET_ACCESS').action + + action_name = security.RBACAgent.permitted_actions.get("DATASET_ACCESS").action permission_failed_error_class = DatasetAccessPermissionFailedException # ---- interface @@ -238,9 +241,11 @@ class AccessDatasetRBACPermission(DatasetRBACPermission): current_roles = self._roles(dataset) # NOTE: that because of short circuiting this allows # anonymous access to public datasets - return (self._is_public_based_on_roles(current_roles) - or self.user_manager.is_admin(user) # admin is always permitted - or self._user_has_all_roles(user, current_roles)) + return ( + self._is_public_based_on_roles(current_roles) + or self.user_manager.is_admin(user) # admin is always permitted + or self._user_has_all_roles(user, current_roles) + ) def grant(self, item, user): pass @@ -273,6 +278,8 @@ class AccessDatasetRBACPermission(DatasetRBACPermission): def _role_is_permitted(self, dataset, role): current_roles = self._roles(dataset) - return (self._is_public_based_on_roles(current_roles) - # if there's only one role and this is it, let em in - or ((len(current_roles) == 1) and (role == current_roles[0]))) + return ( + self._is_public_based_on_roles(current_roles) + # if there's only one role and this is it, let em in + or ((len(current_roles) == 1) and (role == current_roles[0])) + ) diff --git a/lib/galaxy/managers/remote_files.py b/lib/galaxy/managers/remote_files.py index 2e9bfbd96db..6fb6830e966 100644 --- a/lib/galaxy/managers/remote_files.py +++ b/lib/galaxy/managers/remote_files.py @@ -58,11 +58,11 @@ class RemoteFilesManager: default_format = RemoteFilesFormat.flat default_recursive = True elif target == RemoteFilesTarget.importdir: - uri = 'gximport://' + uri = "gximport://" default_format = RemoteFilesFormat.flat default_recursive = True - elif target in [RemoteFilesTarget.ftpdir, 'ftp']: # legacy, allow both - uri = 'gxftp://' + elif target in [RemoteFilesTarget.ftpdir, "ftp"]: # legacy, allow both + uri = "gxftp://" default_format = RemoteFilesFormat.flat default_recursive = True else: @@ -100,13 +100,19 @@ class RemoteFilesManager: path = ent["path"] path_hash = hashlib.sha1(smart_str(path)).hexdigest() if ent["class"] == "Directory": - path_type = 'folder' + path_type = "folder" disabled = True if disable == RemoteFilesDisableMode.folders else False else: - path_type = 'file' + path_type = "file" disabled = True if disable == RemoteFilesDisableMode.files else False - jstree_paths.append(jstree.Path(path, path_hash, {'type': path_type, 'state': {'disabled': disabled}, 'li_attr': {'full_path': path}})) + jstree_paths.append( + jstree.Path( + path, + path_hash, + {"type": path_type, "state": {"disabled": disabled}, "li_attr": {"full_path": path}}, + ) + ) userdir_jstree = jstree.JSTree(jstree_paths) index = userdir_jstree.jsonData() diff --git a/lib/galaxy/managers/roles.py b/lib/galaxy/managers/roles.py index 9eec5893248..42a3b491679 100644 --- a/lib/galaxy/managers/roles.py +++ b/lib/galaxy/managers/roles.py @@ -21,8 +21,9 @@ class RoleManager(base.ModelManager): """ Business logic for roles. """ + model_class = model.Role - foreign_key_name = 'role' + foreign_key_name = "role" user_assoc = model.UserRoleAssociation group_assoc = model.GroupRoleAssociation @@ -40,17 +41,16 @@ class RoleManager(base.ModelManager): :raises: InconsistentDatabase, RequestParameterInvalidException, InternalServerError """ try: - role = (self.session().query(self.model_class) - .filter(self.model_class.id == decoded_role_id).one()) + role = self.session().query(self.model_class).filter(self.model_class.id == decoded_role_id).one() except sqlalchemy_exceptions.MultipleResultsFound: - raise galaxy.exceptions.InconsistentDatabase('Multiple roles found with the same id.') + raise galaxy.exceptions.InconsistentDatabase("Multiple roles found with the same id.") except sqlalchemy_exceptions.NoResultFound: - raise galaxy.exceptions.RequestParameterInvalidException('No accessible role found with the id provided.') + raise galaxy.exceptions.RequestParameterInvalidException("No accessible role found with the id provided.") except Exception as e: raise galaxy.exceptions.InternalServerError(f"Error loading from the database.{unicodify(e)}") if not (trans.user_is_admin or trans.app.security_agent.ok_to_display(trans.user, role)): - raise galaxy.exceptions.RequestParameterInvalidException('No accessible role found with the id provided.') + raise galaxy.exceptions.RequestParameterInvalidException("No accessible role found with the id provided.") return role diff --git a/lib/galaxy/managers/secured.py b/lib/galaxy/managers/secured.py index 70f37c6f22b..7103c435a18 100644 --- a/lib/galaxy/managers/secured.py +++ b/lib/galaxy/managers/secured.py @@ -5,7 +5,10 @@ Owned models can be modified and deleted. """ from typing import Type -from galaxy import exceptions, model +from galaxy import ( + exceptions, + model, +) class AccessibleManagerMixin: @@ -81,6 +84,7 @@ class OwnableManagerMixin: This can also be thought of as write/edit privileges. """ + # declare what we are using from base ModelManager model_class: Type[model._HasTable] diff --git a/lib/galaxy/managers/session.py b/lib/galaxy/managers/session.py index 8ec49cd23ad..5d9fc238af8 100644 --- a/lib/galaxy/managers/session.py +++ b/lib/galaxy/managers/session.py @@ -4,9 +4,7 @@ from sqlalchemy import ( and_, true, ) -from sqlalchemy.orm import ( - joinedload, -) +from sqlalchemy.orm import joinedload from galaxy.model.base import SharedModelMapping @@ -24,9 +22,15 @@ class GalaxySessionManager: """Returns GalaxySession if session_key is valid.""" # going through self.model since this can be used by Galaxy or Toolshed despite # type annotations - galaxy_session = self.sa_session.query(self.model.GalaxySession).filter( - and_( - self.model.GalaxySession.table.c.session_key == session_key, - self.model.GalaxySession.table.c.is_valid == true()) - ).options(joinedload("user")).first() + galaxy_session = ( + self.sa_session.query(self.model.GalaxySession) + .filter( + and_( + self.model.GalaxySession.table.c.session_key == session_key, + self.model.GalaxySession.table.c.is_valid == true(), + ) + ) + .options(joinedload("user")) + .first() + ) return galaxy_session diff --git a/lib/galaxy/managers/sharable.py b/lib/galaxy/managers/sharable.py index cf1ddcf2bb3..f013f182be5 100644 --- a/lib/galaxy/managers/sharable.py +++ b/lib/galaxy/managers/sharable.py @@ -29,7 +29,7 @@ from galaxy.managers import ( ratable, secured, taggable, - users + users, ) from galaxy.model import ( User, @@ -46,8 +46,14 @@ from galaxy.util import ready_name_for_url log = logging.getLogger(__name__) -class SharableModelManager(base.ModelManager, secured.OwnableManagerMixin, secured.AccessibleManagerMixin, - taggable.TaggableManagerMixin, annotatable.AnnotatableManagerMixin, ratable.RatableManagerMixin): +class SharableModelManager( + base.ModelManager, + secured.OwnableManagerMixin, + secured.AccessibleManagerMixin, + taggable.TaggableManagerMixin, + annotatable.AnnotatableManagerMixin, + ratable.RatableManagerMixin, +): # e.g. histories, pages, stored workflows, visualizations # base.DeleteableModelMixin? (all four are deletable) @@ -107,7 +113,7 @@ class SharableModelManager(base.ModelManager, secured.OwnableManagerMixin, secur importable, and slug attributes. """ self.create_unique_slug(item, flush=False) - return self._session_setattr(item, 'importable', True, flush=flush) + return self._session_setattr(item, "importable", True, flush=flush) def make_non_importable(self, item, flush=True): """ @@ -118,7 +124,7 @@ class SharableModelManager(base.ModelManager, secured.OwnableManagerMixin, secur # item must be unpublished if non-importable if item.published: self.unpublish(item, flush=False) - return self._session_setattr(item, 'importable', False, flush=flush) + return self._session_setattr(item, "importable", False, flush=flush) # .... published def publish(self, item, flush=True): @@ -128,13 +134,13 @@ class SharableModelManager(base.ModelManager, secured.OwnableManagerMixin, secur # item must be importable to be published if not item.importable: self.make_importable(item, flush=False) - return self._session_setattr(item, 'published', True, flush=flush) + return self._session_setattr(item, "published", True, flush=flush) def unpublish(self, item, flush=True): """ Set the published flag on `item` to False. """ - return self._session_setattr(item, 'published', False, flush=flush) + return self._session_setattr(item, "published", False, flush=flush) def _query_published(self, filters=None, **kwargs): """ @@ -209,7 +215,7 @@ class SharableModelManager(base.ModelManager, secured.OwnableManagerMixin, secur Return a query for this model already filtered to models shared with a particular user. """ - query = self.session().query(self.model_class).join('users_shared_with') + query = self.session().query(self.model_class).join("users_shared_with") if eagerloads is False: query = query.enable_eagerloads(False) # TODO: as filter in FilterParser also @@ -224,13 +230,13 @@ class SharableModelManager(base.ModelManager, secured.OwnableManagerMixin, secur orm_filters, fn_filters = self._split_filters(filters) if not fn_filters: # if no fn_filtering required, we can use the 'all orm' version with limit offset - query = self._query_shared_with(user, filters=orm_filters, - order_by=order_by, limit=limit, offset=offset, **kwargs) + query = self._query_shared_with( + user, filters=orm_filters, order_by=order_by, limit=limit, offset=offset, **kwargs + ) return self._orm_list(query=query, **kwargs) # fn filters will change the number of items returnable by limit/offset - remove them here from the orm query - query = self._query_shared_with(user, filters=orm_filters, - order_by=order_by, limit=None, offset=None, **kwargs) + query = self._query_shared_with(user, filters=orm_filters, order_by=order_by, limit=None, offset=None, **kwargs) # apply limit and offset afterwards items = self._apply_fn_filters_gen(query.all(), fn_filters) return list(self._apply_fn_limit_offset_gen(items, limit, offset)) @@ -245,7 +251,7 @@ class SharableModelManager(base.ModelManager, secured.OwnableManagerMixin, secur return None def make_members_public(self, trans, item): - """ Make potential elements of this item public. + """Make potential elements of this item public. This method must be overridden in managers that need to change permissions of internal elements contained associated with the given item. @@ -301,9 +307,7 @@ class SharableModelManager(base.ModelManager, secured.OwnableManagerMixin, secur return VALID_SLUG_RE.match(slug) def _slug_exists(self, user, slug): - query = (self.session().query(self.model_class) - .filter_by(user_id=user.id, slug=slug) - .with_entities(func.count())) + query = self.session().query(self.model_class).filter_by(user_id=user.id, slug=slug).with_entities(func.count()) return query.scalar() != 0 def _slugify(self, start_with): @@ -312,13 +316,13 @@ class SharableModelManager(base.ModelManager, secured.OwnableManagerMixin, secur # Remove all non-alphanumeric characters. slug_base = re.sub(r"[^a-zA-Z0-9\-]", "", slug_base) # Remove trailing '-'. - if slug_base.endswith('-'): + if slug_base.endswith("-"): slug_base = slug_base[:-1] return slug_base def _default_slug_base(self, item): # override in subclasses - if hasattr(item, 'title'): + if hasattr(item, "title"): return item.title.lower() return item.name.lower() @@ -339,12 +343,12 @@ class SharableModelManager(base.ModelManager, secured.OwnableManagerMixin, secur # add integer to end. new_slug = slug_base count = 1 - while (self.session().query(item.__class__) - .filter_by(user=item.user, slug=new_slug, importable=True) - .count() != 0): + while ( + self.session().query(item.__class__).filter_by(user=item.user, slug=new_slug, importable=True).count() != 0 + ): # Slug taken; choose a new slug based on count. This approach can # handle numerous items with the same name gracefully. - new_slug = '%s-%i' % (slug_base, count) + new_slug = "%s-%i" % (slug_base, count) count += 1 return new_slug @@ -362,37 +366,34 @@ class SharableModelManager(base.ModelManager, secured.OwnableManagerMixin, secur # TODO: def by_slug( self, user, **kwargs ): -class SharableModelSerializer(base.ModelSerializer, - taggable.TaggableSerializerMixin, annotatable.AnnotatableSerializerMixin, ratable.RatableSerializerMixin): +class SharableModelSerializer( + base.ModelSerializer, + taggable.TaggableSerializerMixin, + annotatable.AnnotatableSerializerMixin, + ratable.RatableSerializerMixin, +): # TODO: stub SINGLE_CHAR_ABBR: Optional[str] = None def __init__(self, app, **kwargs): super().__init__(app, **kwargs) - self.add_view('sharing', [ - 'id', - 'title', - 'importable', - 'published', - 'username_and_slug', - 'users_shared_with' - ]) + self.add_view("sharing", ["id", "title", "importable", "published", "username_and_slug", "users_shared_with"]) def add_serializers(self): super().add_serializers() taggable.TaggableSerializerMixin.add_serializers(self) annotatable.AnnotatableSerializerMixin.add_serializers(self) ratable.RatableSerializerMixin.add_serializers(self) - self.serializers.update({ - 'id': self.serialize_id, - 'title': self.serialize_title, - 'username_and_slug': self.serialize_username_and_slug, - 'users_shared_with': self.serialize_users_shared_with - }) + self.serializers.update( + { + "id": self.serialize_id, + "title": self.serialize_title, + "username_and_slug": self.serialize_username_and_slug, + "users_shared_with": self.serialize_users_shared_with, + } + ) # these use the default serializer but must still be white-listed - self.serializable_keyset.update([ - 'importable', 'published', 'slug' - ]) + self.serializable_keyset.update(["importable", "published", "slug"]) def serialize_title(self, item, key, **context): if hasattr(item, "title"): @@ -403,7 +404,7 @@ class SharableModelSerializer(base.ModelSerializer, def serialize_username_and_slug(self, item, key, **context): if not (item.user and item.user.username and item.slug and self.SINGLE_CHAR_ABBR): return None - return ('/').join(('u', item.user.username, self.SINGLE_CHAR_ABBR, item.slug)) + return ("/").join(("u", item.user.username, self.SINGLE_CHAR_ABBR, item.slug)) # the only ones that needs any fns: # user/user_id @@ -421,12 +422,15 @@ class SharableModelSerializer(base.ModelSerializer, self.skip() share_assocs = self.manager.get_share_assocs(item) - return [self.serialize_id(share, 'user_id') for share in share_assocs] + return [self.serialize_id(share, "user_id") for share in share_assocs] -class SharableModelDeserializer(base.ModelDeserializer, - taggable.TaggableDeserializerMixin, annotatable.AnnotatableDeserializerMixin, ratable.RatableDeserializerMixin): - +class SharableModelDeserializer( + base.ModelDeserializer, + taggable.TaggableDeserializerMixin, + annotatable.AnnotatableDeserializerMixin, + ratable.RatableDeserializerMixin, +): def __init__(self, app: MinimalManagerApp, **kwargs): super().__init__(app, **kwargs) self.tag_handler = app.tag_handler @@ -437,15 +441,16 @@ class SharableModelDeserializer(base.ModelDeserializer, annotatable.AnnotatableDeserializerMixin.add_deserializers(self) ratable.RatableDeserializerMixin.add_deserializers(self) - self.deserializers.update({ - 'published': self.deserialize_published, - 'importable': self.deserialize_importable, - 'users_shared_with': self.deserialize_users_shared_with, - }) + self.deserializers.update( + { + "published": self.deserialize_published, + "importable": self.deserialize_importable, + "users_shared_with": self.deserialize_users_shared_with, + } + ) def deserialize_published(self, item, key, val, **context): - """ - """ + """ """ val = self.validate.bool(key, val) if item.published == val: return val @@ -457,8 +462,7 @@ class SharableModelDeserializer(base.ModelDeserializer, return item.published def deserialize_importable(self, item, key, val, **context): - """ - """ + """ """ val = self.validate.bool(key, val) if item.importable == val: return val @@ -484,22 +488,24 @@ class SharableModelDeserializer(base.ModelDeserializer, return current_shares -class SharableModelFilters(base.ModelFilterParser, - taggable.TaggableFilterMixin, annotatable.AnnotatableFilterMixin, ratable.RatableFilterMixin): - +class SharableModelFilters( + base.ModelFilterParser, taggable.TaggableFilterMixin, annotatable.AnnotatableFilterMixin, ratable.RatableFilterMixin +): def _add_parsers(self): super()._add_parsers() taggable.TaggableFilterMixin._add_parsers(self) annotatable.AnnotatableFilterMixin._add_parsers(self) ratable.RatableFilterMixin._add_parsers(self) - self.orm_filter_parsers.update({ - 'importable': {'op': ('eq'), 'val': base.parse_bool}, - 'published': {'op': ('eq'), 'val': base.parse_bool}, - 'slug': {'op': ('eq', 'contains', 'like')}, - # chose by user should prob. only be available for admin? (most often we'll only need trans.user) - # 'user' : { 'op': ( 'eq' ), 'val': self.parse_id_list }, - }) + self.orm_filter_parsers.update( + { + "importable": {"op": ("eq"), "val": base.parse_bool}, + "published": {"op": ("eq"), "val": base.parse_bool}, + "slug": {"op": ("eq", "contains", "like")}, + # chose by user should prob. only be available for admin? (most often we'll only need trans.user) + # 'user' : { 'op': ( 'eq' ), 'val': self.parse_id_list }, + } + ) class SlugBuilder: @@ -519,12 +525,12 @@ class SlugBuilder: cur_slug = item.slug # Setup slug base. - if cur_slug is None or cur_slug == '': + if cur_slug is None or cur_slug == "": # Item can have either a name or a title. - item_name = '' - if hasattr(item, 'name'): + item_name = "" + if hasattr(item, "name"): item_name = item.name - elif hasattr(item, 'title'): + elif hasattr(item, "title"): item_name = item.title slug_base = ready_name_for_url(item_name.lower()) else: @@ -536,10 +542,15 @@ class SlugBuilder: count = 1 # Ensure unique across model class and user and don't include this item # in the check in case it has previously been assigned a valid slug. - while sa_session.query(item.__class__).filter(item.__class__.user == item.user, item.__class__.slug == new_slug, item.__class__.id != item.id).count() != 0: + while ( + sa_session.query(item.__class__) + .filter(item.__class__.user == item.user, item.__class__.slug == new_slug, item.__class__.id != item.id) + .count() + != 0 + ): # Slug taken; choose a new slug based on count. This approach can # handle numerous items with the same name gracefully. - new_slug = f'{slug_base}-{count}' + new_slug = f"{slug_base}-{count}" count += 1 # Set slug and return. diff --git a/lib/galaxy/managers/taggable.py b/lib/galaxy/managers/taggable.py index 233fc417c9d..9a6377ecf2b 100644 --- a/lib/galaxy/managers/taggable.py +++ b/lib/galaxy/managers/taggable.py @@ -12,7 +12,10 @@ from sqlalchemy import sql from galaxy import model from galaxy.model.tags import GalaxyTagHandler from galaxy.util import unicodify -from .base import ModelValidator, raise_filter_err +from .base import ( + ModelValidator, + raise_filter_err, +) log = logging.getLogger(__name__) @@ -28,7 +31,7 @@ def _tag_str_gen(item): def _tags_to_strings(item): - if not hasattr(item, 'tags'): + if not hasattr(item, "tags"): return None return sorted(list(_tag_str_gen(item))) @@ -42,8 +45,8 @@ def _tags_from_strings(item, tag_handler, new_tags_list, user=None): return # TODO: duped from tags manager - de-dupe when moved to taggable mixin tag_handler.delete_item_tags(user, item) - new_tags_str = ','.join(new_tags_list) - tag_handler.apply_item_tags(user, item, unicodify(new_tags_str, 'utf-8')) + new_tags_str = ",".join(new_tags_list) + tag_handler.apply_item_tags(user, item, unicodify(new_tags_str, "utf-8")) # TODO:!! does the creation of new_tags_list mean there are now more and more unused tag rows in the db? @@ -70,9 +73,8 @@ class TaggableManagerMixin: class TaggableSerializerMixin: - def add_serializers(self): - self.serializers['tags'] = self.serialize_tags + self.serializers["tags"] = self.serialize_tags def serialize_tags(self, item, key, **context): """ @@ -86,7 +88,7 @@ class TaggableDeserializerMixin: validate: ModelValidator def add_deserializers(self): - self.deserializers['tags'] = self.deserialize_tags + self.deserializers["tags"] = self.deserialize_tags def deserialize_tags(self, item, key, val, user=None, **context): """ @@ -101,24 +103,23 @@ class TaggableDeserializerMixin: class TaggableFilterMixin: - valid_ops = ('eq', 'contains', 'has') + valid_ops = ("eq", "contains", "has") def create_tag_filter(self, attr, op, val): - def _create_tag_filter(model_class=None): if op not in TaggableFilterMixin.valid_ops: - raise_filter_err(attr, op, val, 'bad op in filter') + raise_filter_err(attr, op, val, "bad op in filter") if model_class is None: return True class_name = model_class.__name__ - if class_name == 'HistoryDatasetCollectionAssociation': + if class_name == "HistoryDatasetCollectionAssociation": # Unfortunately we were a little inconsistent with our naming scheme - class_name = 'HistoryDatasetCollection' + class_name = "HistoryDatasetCollection" target_model = getattr(model, f"{class_name}TagAssociation") id_column = f"{target_model.table.name.rsplit('_tag_association')[0]}_id" column = target_model.table.c.user_tname + ":" + target_model.table.c.user_value - if op == 'eq': - if ':' not in val: + if op == "eq": + if ":" not in val: # We require an exact match and the tag to look for has no user_value, # so we can't just concatenate user_tname, ':' and user_vale cond = target_model.table.c.user_tname == val @@ -126,13 +127,9 @@ class TaggableFilterMixin: cond = column == val else: cond = column.contains(val, autoescape=True) - return sql.expression.and_( - model_class.table.c.id == getattr(target_model.table.c, id_column), - cond - ) + return sql.expression.and_(model_class.table.c.id == getattr(target_model.table.c, id_column), cond) + return _create_tag_filter def _add_parsers(self): - self.orm_filter_parsers.update({ - 'tag': self.create_tag_filter - }) + self.orm_filter_parsers.update({"tag": self.create_tag_filter}) diff --git a/lib/galaxy/managers/tags.py b/lib/galaxy/managers/tags.py index 3afbe9862e0..9fe308f163e 100644 --- a/lib/galaxy/managers/tags.py +++ b/lib/galaxy/managers/tags.py @@ -1,7 +1,5 @@ from enum import Enum -from typing import ( - Optional, -) +from typing import Optional from pydantic import ( BaseModel, @@ -17,7 +15,7 @@ from galaxy.schema.schema import TagCollection taggable_item_names = {item: item for item in ItemTagAssociation.associated_item_names} # This Enum is generated dynamically and mypy can not statically infer it's real type # so it should be ignored. See: https://github.com/python/mypy/issues/4865#issuecomment-592560696 -TaggableItemClass = Enum('TaggableItemClass', taggable_item_names) # type: ignore[misc] +TaggableItemClass = Enum("TaggableItemClass", taggable_item_names) # type: ignore[misc] class ItemTagsPayload(BaseModel): diff --git a/lib/galaxy/managers/tool_data.py b/lib/galaxy/managers/tool_data.py index c03d3bdc39f..d4f50f7eed4 100644 --- a/lib/galaxy/managers/tool_data.py +++ b/lib/galaxy/managers/tool_data.py @@ -37,7 +37,7 @@ class ToolDataManager: def show(self, table_name: str) -> ToolDataDetails: """Get details of a given data table""" data_table = self._data_table(table_name) - element_view = data_table.to_dict(view='element') + element_view = data_table.to_dict(view="element") return ToolDataDetails.parse_obj(element_view) def show_field(self, table_name: str, field_name: str) -> ToolDataField: @@ -69,7 +69,9 @@ class ToolDataManager: split_values = values.split("\t") if len(split_values) != len(data_table.get_column_name_list()): - raise exceptions.RequestParameterInvalidException(f"Invalid data table item ( {values} ) specified. Wrong number of columns ({len(split_values)} given, {len(data_table.get_column_name_list())} required).") + raise exceptions.RequestParameterInvalidException( + f"Invalid data table item ( {values} ) specified. Wrong number of columns ({len(split_values)} given, {len(data_table.get_column_name_list())} required)." + ) data_table.remove_entry(split_values) return self._reload_data_table(table_name) @@ -87,9 +89,5 @@ class ToolDataManager: return out def _reload_data_table(self, name: str) -> ToolDataDetails: - self._app.queue_worker.send_control_task( - 'reload_tool_data_tables', - noop_self=True, - kwargs={'table_name': name} - ) + self._app.queue_worker.send_control_task("reload_tool_data_tables", noop_self=True, kwargs={"table_name": name}) return self.show(name) diff --git a/lib/galaxy/managers/tools.py b/lib/galaxy/managers/tools.py index a9f4d6222ab..1b05ff43b46 100644 --- a/lib/galaxy/managers/tools.py +++ b/lib/galaxy/managers/tools.py @@ -1,14 +1,23 @@ import logging -from typing import Optional, TYPE_CHECKING, Union +from typing import ( + Optional, + TYPE_CHECKING, + Union, +) from uuid import UUID from sqlalchemy import sql -from galaxy import exceptions -from galaxy import model +from galaxy import ( + exceptions, + model, +) from galaxy.exceptions import DuplicatedIdentifierException from galaxy.tool_util.cwl import tool_proxy -from .base import ModelManager, raise_filter_err +from .base import ( + ModelManager, + raise_filter_err, +) from .executables import artifact_class log = logging.getLogger(__name__) @@ -18,31 +27,27 @@ if TYPE_CHECKING: class DynamicToolManager(ModelManager): - """ Manages dynamic tools stored in Galaxy's database. - """ + """Manages dynamic tools stored in Galaxy's database.""" + model_class = model.DynamicTool def get_tool_by_uuid(self, uuid: Optional[Union[UUID, str]]): - dynamic_tool = self._one_or_none( - self.query().filter(self.model_class.uuid == uuid) - ) + dynamic_tool = self._one_or_none(self.query().filter(self.model_class.uuid == uuid)) return dynamic_tool def get_tool_by_tool_id(self, tool_id): - dynamic_tool = self._one_or_none( - self.query().filter(self.model_class.tool_id == tool_id) - ) + dynamic_tool = self._one_or_none(self.query().filter(self.model_class.tool_id == tool_id)) return dynamic_tool def get_tool_by_id(self, object_id): - dynamic_tool = self._one_or_none( - self.query().filter(self.model_class.id == object_id) - ) + dynamic_tool = self._one_or_none(self.query().filter(self.model_class.id == object_id)) return dynamic_tool def create_tool(self, trans, tool_payload, allow_load=True): if not getattr(self.app.config, "enable_beta_tool_formats", False): - raise exceptions.ConfigDoesNotAllowException("Set 'enable_beta_tool_formats' in Galaxy config to create dynamic tools.") + raise exceptions.ConfigDoesNotAllowException( + "Set 'enable_beta_tool_formats' in Galaxy config to create dynamic tools." + ) dynamic_tool = None uuid_str = tool_payload.get("uuid") @@ -66,15 +71,11 @@ class DynamicToolManager(ModelManager): assert src == "representation" representation = tool_payload.get("representation") if not representation: - raise exceptions.ObjectAttributeMissingException( - "A tool 'representation' is required." - ) + raise exceptions.ObjectAttributeMissingException("A tool 'representation' is required.") tool_format = representation.get("class") if not tool_format: - raise exceptions.ObjectAttributeMissingException( - "Current tool representations require 'class'." - ) + raise exceptions.ObjectAttributeMissingException("Current tool representations require 'class'.") tool_path = tool_payload.get("path") tool_directory = tool_payload.get("tool_directory") @@ -122,32 +123,36 @@ class ToolFilterMixin: orm_filter_parsers: "OrmFilterParsersType" def create_tool_filter(self, attr, op, val): - def _create_tool_filter(model_class=None): - if op == 'eq': + if op == "eq": cond = model.Job.table.c.tool_id == val - elif op == 'contains': + elif op == "contains": cond = model.Job.table.c.tool_id.contains(val, autoescape=True) else: - raise_filter_err(attr, op, val, 'bad op in filter') + raise_filter_err(attr, op, val, "bad op in filter") if model_class is model.HistoryDatasetAssociation: return sql.expression.and_( model.Job.table.c.id == model.JobToOutputDatasetAssociation.table.c.job_id, - model.HistoryDatasetAssociation.table.c.id == model.JobToOutputDatasetAssociation.table.c.dataset_id, - cond + model.HistoryDatasetAssociation.table.c.id + == model.JobToOutputDatasetAssociation.table.c.dataset_id, + cond, ) elif model_class is model.HistoryDatasetCollectionAssociation: return sql.expression.and_( model.Job.id == model.JobToOutputDatasetAssociation.job_id, model.JobToOutputDatasetAssociation.dataset_id == model.DatasetCollectionElement.hda_id, - model.DatasetCollectionElement.dataset_collection_id == model.HistoryDatasetCollectionAssociation.collection_id, + model.DatasetCollectionElement.dataset_collection_id + == model.HistoryDatasetCollectionAssociation.collection_id, cond, ) else: return True + return _create_tool_filter def _add_parsers(self): - self.orm_filter_parsers.update({ - 'tool_id': self.create_tool_filter, - }) + self.orm_filter_parsers.update( + { + "tool_id": self.create_tool_filter, + } + ) diff --git a/lib/galaxy/managers/users.py b/lib/galaxy/managers/users.py index 05d7b2bc4ea..09bf8eeb49e 100644 --- a/lib/galaxy/managers/users.py +++ b/lib/galaxy/managers/users.py @@ -9,7 +9,13 @@ import time from datetime import datetime from markupsafe import escape -from sqlalchemy import and_, desc, exc, func, true +from sqlalchemy import ( + and_, + desc, + exc, + func, + true, +) from sqlalchemy.orm.exc import NoResultFound from galaxy import ( @@ -21,15 +27,18 @@ from galaxy import ( from galaxy.managers import ( api_keys, base, - deletable + deletable, ) from galaxy.security.validate_user_input import ( VALID_EMAIL_RE, validate_email, validate_password, - validate_publicname + validate_publicname, +) +from galaxy.structured_app import ( + BasicSharedApp, + MinimalManagerApp, ) -from galaxy.structured_app import BasicSharedApp, MinimalManagerApp from galaxy.util.hash_util import new_secure_hash from galaxy.web import url_for @@ -50,7 +59,7 @@ can also copy and paste it into your browser. class UserManager(base.ModelManager, deletable.PurgableManagerMixin): - foreign_key_name = 'user' + foreign_key_name = "user" # TODO: there is quite a bit of functionality around the user (authentication, permissions, quotas, groups/roles) # most of which it may be unneccessary to have here @@ -73,9 +82,13 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin): return None, message if not email or not username or not password or not confirm: return None, "Please provide email, username and password." - message = "\n".join((validate_email(trans, email), - validate_password(trans, password, confirm), - validate_publicname(trans, username))).rstrip() + message = "\n".join( + ( + validate_email(trans, email), + validate_password(trans, password, confirm), + validate_publicname(trans, username), + ) + ).rstrip() if message: return None, message email = util.restore_text(email) @@ -120,23 +133,29 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin): def delete(self, user, flush=True): """Mark the given user deleted.""" if not self.app.config.allow_user_deletion: - raise exceptions.ConfigDoesNotAllowException('The configuration of this Galaxy instance does not allow admins to delete users.') + raise exceptions.ConfigDoesNotAllowException( + "The configuration of this Galaxy instance does not allow admins to delete users." + ) super().delete(user, flush=flush) def undelete(self, user, flush=True): """Remove the deleted flag for the given user.""" if not self.app.config.allow_user_deletion: - raise exceptions.ConfigDoesNotAllowException('The configuration of this Galaxy instance does not allow admins to undelete users.') + raise exceptions.ConfigDoesNotAllowException( + "The configuration of this Galaxy instance does not allow admins to undelete users." + ) if user.purged: - raise exceptions.ItemDeletionException('Purged user cannot be undeleted.') + raise exceptions.ItemDeletionException("Purged user cannot be undeleted.") super().undelete(user, flush=flush) def purge(self, user, flush=True): """Purge the given user. They must have the deleted flag already.""" if not self.app.config.allow_user_deletion: - raise exceptions.ConfigDoesNotAllowException('The configuration of this Galaxy instance does not allow admins to delete or purge users.') + raise exceptions.ConfigDoesNotAllowException( + "The configuration of this Galaxy instance does not allow admins to delete or purge users." + ) if not user.deleted: - raise exceptions.MessageException('User \'%s\' has not been deleted, so they cannot be purged.' % user.email) + raise exceptions.MessageException("User '%s' has not been deleted, so they cannot be purged." % user.email) private_role = self.app.security_agent.get_private_user_role(user) # Delete History for active_history in user.active_histories: @@ -157,8 +176,8 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin): # Delete UserAddresses for address in user.addresses: self.session().delete(address) - compliance_log = logging.getLogger('COMPLIANCE') - compliance_log.info(f'delete-user-event: {user.username}') + compliance_log = logging.getLogger("COMPLIANCE") + compliance_log.info(f"delete-user-event: {user.username}") # Maybe there is some case in the future where an admin needs # to prove that a user was using a server for some reason (e.g. # a court case.) So we make this painfully hard to recover (and @@ -188,8 +207,12 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin): user.username = uname_hash # Redact user addresses as well if self.app.config.redact_user_address_during_deletion: - user_addresses = self.session().query(self.app.model.UserAddress) \ - .filter(self.app.model.UserAddress.user_id == user.id).all() + user_addresses = ( + self.session() + .query(self.app.model.UserAddress) + .filter(self.app.model.UserAddress.user_id == user.id) + .all() + ) for addr in user_addresses: addr.desc = new_secure_hash(addr.desc + pseudorandom_value) addr.name = new_secure_hash(addr.name + pseudorandom_value) @@ -212,7 +235,7 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin): """ # TODO: remove this check when unique=True is added to the email column if self.by_email(email) is not None: - raise exceptions.Conflict('Email must be unique', email=email) + raise exceptions.Conflict("Email must be unique", email=email) def by_id(self, user_id): return self.app.model.session.query(self.model_class).get(user_id) @@ -239,17 +262,17 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin): try: provided_key = sa_session.query(self.app.model.APIKeys).filter(self.app.model.APIKeys.key == api_key).one() except NoResultFound: - raise exceptions.AuthenticationFailed('Provided API key is not valid.') + raise exceptions.AuthenticationFailed("Provided API key is not valid.") if provided_key.user.deleted: - raise exceptions.AuthenticationFailed('User account is deactivated, please contact an administrator.') + raise exceptions.AuthenticationFailed("User account is deactivated, please contact an administrator.") sa_session.refresh(provided_key.user) newest_key = provided_key.user.api_keys[0] if newest_key.key != provided_key.key: - raise exceptions.AuthenticationFailed('Provided API key has expired.') + raise exceptions.AuthenticationFailed("Provided API key has expired.") return provided_key.user def check_master_api_key(self, api_key): - master_api_key = getattr(self.app.config, 'master_api_key', None) + master_api_key = getattr(self.app.config, "master_api_key", None) if not master_api_key: return False # Hash keys to make them the same size, so we can do safe comparison. @@ -311,15 +334,17 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin): user = None if VALID_EMAIL_RE.match(identity): # VALID_PUBLICNAME and VALID_EMAIL do not overlap, so 'identity' here is an email address - user = self.session().query(self.model_class).filter( - self.model_class.table.c.email == identity).first() + user = self.session().query(self.model_class).filter(self.model_class.table.c.email == identity).first() if not user: # Try a case-insensitive match on the email - user = self.session().query(self.model_class).filter( - func.lower(self.model_class.table.c.email) == identity.lower()).first() + user = ( + self.session() + .query(self.model_class) + .filter(func.lower(self.model_class.table.c.email) == identity.lower()) + .first() + ) else: - user = self.session().query(self.model_class).filter( - self.model_class.table.c.username == identity).first() + user = self.session().query(self.model_class).filter(self.model_class.table.c.username == identity).first() return user # ---- current @@ -351,9 +376,7 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin): """ Return this most recent APIKey for this user or None if none have been created. """ - query = (self.session().query(model.APIKeys) - .filter_by(user=user) - .order_by(desc(model.APIKeys.create_time))) + query = self.session().query(model.APIKeys).filter_by(user=user).order_by(desc(model.APIKeys.create_time)) all = query.all() if len(all): return all[0] @@ -404,8 +427,7 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin): # create a union of subqueries for each for this user - getting only the tname and user_value all_tags_query = None for tag_model in tag_models: - subq = (self.session().query(tag_model.user_tname, tag_model.user_value) - .filter(tag_model.user == user)) + subq = self.session().query(tag_model.user_tname, tag_model.user_value).filter(tag_model.user == user) all_tags_query = subq if all_tags_query is None else all_tags_query.union(subq) # if nothing init'd the query, bail @@ -460,10 +482,13 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin): user.set_password_cleartext(password) # Invalidate all other sessions if trans.galaxy_session: - for other_galaxy_session in trans.sa_session.query(self.app.model.GalaxySession) \ - .filter(and_(self.app.model.GalaxySession.table.c.user_id == user.id, - self.app.model.GalaxySession.table.c.is_valid == true(), - self.app.model.GalaxySession.table.c.id != trans.galaxy_session.id)): + for other_galaxy_session in trans.sa_session.query(self.app.model.GalaxySession).filter( + and_( + self.app.model.GalaxySession.table.c.user_id == user.id, + self.app.model.GalaxySession.table.c.is_valid == true(), + self.app.model.GalaxySession.table.c.id != trans.galaxy_session.id, + ) + ): other_galaxy_session.is_valid = False trans.sa_session.add(other_galaxy_session) trans.sa_session.add(user) @@ -477,43 +502,50 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin): Send the verification email containing the activation link to the user's email. """ activation_token = self.__get_activation_token(trans, email) - activation_link = url_for(controller='user', action='activate', activation_token=activation_token, email=escape(email), qualified=True) + activation_link = url_for( + controller="user", action="activate", activation_token=activation_token, email=escape(email), qualified=True + ) host = self.__get_host(trans) - custom_message = '' + custom_message = "" if self.app.config.custom_activation_email_message: custom_message = f"{self.app.config.custom_activation_email_message}\n\n" - body = ("Hello %s,\n\n" - "In order to complete the activation process for %s begun on %s at %s, please click " - "on the following link to verify your account:\n\n" "%s \n\n" - "By clicking on the above link and opening a Galaxy account you are also confirming " - "that you have read and agreed to Galaxy's Terms and Conditions for use of this " - "service (%s). This includes a quota limit of one account per user. Attempts to " - "subvert this limit by creating multiple accounts or through any other method may " - "result in termination of all associated accounts and data.\n\n" - "Please contact us if you need help with your account at: %s. You can also browse " - "resources available" " at: %s. \n\n" - "More about the Galaxy Project can be found at galaxyproject.org\n\n" - "%s" - "Your Galaxy Team" % ( - escape(username), - escape(email), - datetime.utcnow().strftime("%D"), - trans.request.host, - activation_link, - self.app.config.terms_url, - self.app.config.error_email_to, - self.app.config.instance_resource_url, - custom_message) - ) + body = ( + "Hello %s,\n\n" + "In order to complete the activation process for %s begun on %s at %s, please click " + "on the following link to verify your account:\n\n" + "%s \n\n" + "By clicking on the above link and opening a Galaxy account you are also confirming " + "that you have read and agreed to Galaxy's Terms and Conditions for use of this " + "service (%s). This includes a quota limit of one account per user. Attempts to " + "subvert this limit by creating multiple accounts or through any other method may " + "result in termination of all associated accounts and data.\n\n" + "Please contact us if you need help with your account at: %s. You can also browse " + "resources available" + " at: %s. \n\n" + "More about the Galaxy Project can be found at galaxyproject.org\n\n" + "%s" + "Your Galaxy Team" + % ( + escape(username), + escape(email), + datetime.utcnow().strftime("%D"), + trans.request.host, + activation_link, + self.app.config.terms_url, + self.app.config.error_email_to, + self.app.config.instance_resource_url, + custom_message, + ) + ) to = email frm = self.app.config.email_from or f"galaxy-no-reply@{host}" - subject = 'Galaxy Account Activation' + subject = "Galaxy Account Activation" try: util.send_mail(frm, to, subject, body, self.app.config) return True except Exception: log.debug(body) - log.exception('Unable to send the activation email.') + log.exception("Unable to send the activation email.") return False def __get_activation_token(self, trans, email): @@ -543,16 +575,20 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin): reset_user, prt = self.get_reset_token(trans, email) if prt: host = self.__get_host(trans) - reset_url = url_for(controller='root', action='login', token=prt.token) - body = PASSWORD_RESET_TEMPLATE % (host, prt.expiration_time.strftime(trans.app.config.pretty_datetime_format), - trans.request.host, reset_url) + reset_url = url_for(controller="root", action="login", token=prt.token) + body = PASSWORD_RESET_TEMPLATE % ( + host, + prt.expiration_time.strftime(trans.app.config.pretty_datetime_format), + trans.request.host, + reset_url, + ) frm = trans.app.config.email_from or f"galaxy-no-reply@{host}" - subject = 'Galaxy Password Reset' + subject = "Galaxy Password Reset" try: util.send_mail(frm, email, subject, body, self.app.config) trans.sa_session.add(reset_user) trans.sa_session.flush() - trans.log_event(f'User reset password: {email}') + trans.log_event(f"User reset password: {email}") except Exception as e: log.debug(body) return f"Failed to submit email. Please contact the administrator: {util.unicodify(e)}" @@ -560,9 +596,15 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin): return "Failed to produce password reset token. User not found." def get_reset_token(self, trans, email): - reset_user = trans.sa_session.query(self.app.model.User).filter(self.app.model.User.table.c.email == email).first() + reset_user = ( + trans.sa_session.query(self.app.model.User).filter(self.app.model.User.table.c.email == email).first() + ) if not reset_user and email != email.lower(): - reset_user = trans.sa_session.query(self.app.model.User).filter(func.lower(self.app.model.User.table.c.email) == email.lower()).first() + reset_user = ( + trans.sa_session.query(self.app.model.User) + .filter(func.lower(self.app.model.User.table.c.email) == email.lower()) + .first() + ) if reset_user: prt = self.app.model.PasswordResetToken(reset_user) trans.sa_session.add(prt) @@ -571,8 +613,8 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin): return None, None def __get_host(self, trans): - host = trans.request.host.split(':')[0] - if host in ['localhost', '127.0.0.1', '0.0.0.0']: + host = trans.request.host.split(":")[0] + if host in ["localhost", "127.0.0.1", "0.0.0.0"]: host = socket.getfqdn() return host @@ -580,14 +622,14 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin): if self.app.config.smtp_server is None: return "Subscribing to the mailing list has failed because mail is not configured for this Galaxy instance. Please contact your local Galaxy administrator." else: - body = (self.app.config.mailing_join_body or '') + '\n' + body = (self.app.config.mailing_join_body or "") + "\n" to = self.app.config.mailing_join_addr frm = email - subject = self.app.config.mailing_join_subject or '' + subject = self.app.config.mailing_join_subject or "" try: util.send_mail(frm, to, subject, body, self.app.config) except Exception: - log.exception('Subscribing to the mailing list has failed.') + log.exception("Subscribing to the mailing list has failed.") return "Subscribing to the mailing list has failed." def activate(self, user): @@ -606,47 +648,47 @@ class UserSerializer(base.ModelSerializer, deletable.PurgableSerializerMixin): super().__init__(app) self.user_manager = self.manager - self.default_view = 'summary' - self.add_view('summary', [ - 'id', 'email', 'username' - ]) - self.add_view('detailed', [ - # 'update_time', - # 'create_time', - 'is_admin', - 'total_disk_usage', - 'nice_total_disk_usage', - 'quota_percent', - 'quota', - 'deleted', - 'purged', - # 'active', - - 'preferences', - # all tags - 'tags_used', - # all annotations - # 'annotations' - ], include_keys_from='summary') + self.default_view = "summary" + self.add_view("summary", ["id", "email", "username"]) + self.add_view( + "detailed", + [ + # 'update_time', + # 'create_time', + "is_admin", + "total_disk_usage", + "nice_total_disk_usage", + "quota_percent", + "quota", + "deleted", + "purged", + # 'active', + "preferences", + # all tags + "tags_used", + # all annotations + # 'annotations' + ], + include_keys_from="summary", + ) def add_serializers(self): super().add_serializers() deletable.PurgableSerializerMixin.add_serializers(self) - self.serializers.update({ - 'id': self.serialize_id, - 'create_time': self.serialize_date, - 'update_time': self.serialize_date, - 'is_admin': lambda i, k, **c: self.user_manager.is_admin(i), - - 'preferences': lambda i, k, **c: self.user_manager.preferences(i), - - 'total_disk_usage': lambda i, k, **c: float(i.total_disk_usage), - 'quota_percent': lambda i, k, **c: self.user_manager.quota(i), - 'quota': lambda i, k, **c: self.user_manager.quota(i, total=True), - - 'tags_used': lambda i, k, **c: self.user_manager.tags_used(i), - }) + self.serializers.update( + { + "id": self.serialize_id, + "create_time": self.serialize_date, + "update_time": self.serialize_date, + "is_admin": lambda i, k, **c: self.user_manager.is_admin(i), + "preferences": lambda i, k, **c: self.user_manager.preferences(i), + "total_disk_usage": lambda i, k, **c: float(i.total_disk_usage), + "quota_percent": lambda i, k, **c: self.user_manager.quota(i), + "quota": lambda i, k, **c: self.user_manager.quota(i, total=True), + "tags_used": lambda i, k, **c: self.user_manager.tags_used(i), + } + ) class UserDeserializer(base.ModelDeserializer): @@ -654,13 +696,16 @@ class UserDeserializer(base.ModelDeserializer): Service object for validating and deserializing dictionaries that update/alter users. """ + model_manager_class = UserManager def add_deserializers(self): super().add_deserializers() - self.deserializers.update({ - 'username': self.deserialize_username, - }) + self.deserializers.update( + { + "username": self.deserialize_username, + } + ) def deserialize_username(self, item, key, username, trans=None, **context): # TODO: validate_publicname requires trans and should(?) raise exceptions @@ -678,7 +723,7 @@ class CurrentUserSerializer(UserSerializer): """ Override to return at least some usage info if user is anonymous. """ - kwargs['current_user'] = user + kwargs["current_user"] = user if self.user_manager.is_anonymous(user): return self.serialize_current_anonymous_user(user, keys, **kwargs) return super(UserSerializer, self).serialize(user, keys, **kwargs) @@ -696,10 +741,10 @@ class CurrentUserSerializer(UserSerializer): # a very small subset of keys available values = { - 'id': None, - 'total_disk_usage': float(usage), - 'nice_total_disk_usage': util.nice_size(usage), - 'quota_percent': percent, + "id": None, + "total_disk_usage": float(usage), + "nice_total_disk_usage": util.nice_size(usage), + "quota_percent": percent, } serialized = {} for key in keys: @@ -717,11 +762,13 @@ class AdminUserFilterParser(base.ModelFilterParser, deletable.PurgableFiltersMix deletable.PurgableFiltersMixin._add_parsers(self) # PRECONDITION: user making the query has been verified as an admin - self.orm_filter_parsers.update({ - 'email': {'op': ('eq', 'contains', 'like')}, - 'username': {'op': ('eq', 'contains', 'like')}, - 'active': {'op': ('eq')}, - 'disk_usage': {'op': ('le', 'ge')} - }) + self.orm_filter_parsers.update( + { + "email": {"op": ("eq", "contains", "like")}, + "username": {"op": ("eq", "contains", "like")}, + "active": {"op": ("eq")}, + "disk_usage": {"op": ("le", "ge")}, + } + ) self.fn_filter_parsers.update({}) diff --git a/lib/galaxy/managers/visualizations.py b/lib/galaxy/managers/visualizations.py index 9f89c476f32..59657a90285 100644 --- a/lib/galaxy/managers/visualizations.py +++ b/lib/galaxy/managers/visualizations.py @@ -21,7 +21,7 @@ class VisualizationManager(sharable.SharableModelManager): # TODO: revisions model_class = model.Visualization - foreign_key_name = 'visualization' + foreign_key_name = "visualization" user_share_model = model.VisualizationUserShareAssociation tag_assoc = model.VisualizationTagAssociation @@ -38,21 +38,21 @@ class VisualizationSerializer(sharable.SharableModelSerializer): """ Interface/service object for serializing visualizations into dictionaries. """ + model_manager_class = VisualizationManager - SINGLE_CHAR_ABBR = 'v' + SINGLE_CHAR_ABBR = "v" def __init__(self, app: MinimalManagerApp): super().__init__(app) self.visualization_manager = self.manager - self.default_view = 'summary' - self.add_view('summary', []) - self.add_view('detailed', []) + self.default_view = "summary" + self.add_view("summary", []) + self.add_view("detailed", []) def add_serializers(self): super().add_serializers() - self.serializers.update({ - }) + self.serializers.update({}) class VisualizationDeserializer(sharable.SharableModelDeserializer): @@ -60,6 +60,7 @@ class VisualizationDeserializer(sharable.SharableModelDeserializer): Interface/service object for validating and deserializing dictionaries into visualizations. """ + model_manager_class = VisualizationManager def __init__(self, app): @@ -68,6 +69,5 @@ class VisualizationDeserializer(sharable.SharableModelDeserializer): def add_deserializers(self): super().add_deserializers() - self.deserializers.update({ - }) + self.deserializers.update({}) self.deserializable_keyset.update(self.deserializers.keys()) diff --git a/lib/galaxy/managers/workflows.py b/lib/galaxy/managers/workflows.py index 1d5769e86bb..19fb78ce4bf 100644 --- a/lib/galaxy/managers/workflows.py +++ b/lib/galaxy/managers/workflows.py @@ -19,25 +19,28 @@ from gxformat2 import ( ) from pydantic import BaseModel from sqlalchemy import and_ -from sqlalchemy.orm import joinedload, subqueryload +from sqlalchemy.orm import ( + joinedload, + subqueryload, +) from galaxy import ( exceptions, model, - util + util, ) from galaxy.job_execution.actions.post import ActionBox from galaxy.model.item_attrs import UsesAnnotations from galaxy.structured_app import MinimalManagerApp from galaxy.tools.parameters import ( params_to_incoming, - visit_input_values + visit_input_values, ) from galaxy.tools.parameters.basic import ( DataCollectionToolParameter, DataToolParameter, RuntimeValue, - workflow_building_modes + workflow_building_modes, ) from galaxy.util.json import ( safe_dumps, @@ -49,7 +52,7 @@ from galaxy.workflow.modules import ( is_tool_module_type, module_factory, ToolModule, - WorkflowModuleInjector + WorkflowModuleInjector, ) from galaxy.workflow.refactor.execute import WorkflowRefactorExecutor from galaxy.workflow.refactor.schema import ( @@ -66,7 +69,7 @@ log = logging.getLogger(__name__) class WorkflowsManager: - """ Handle CRUD type operations related to workflows. More interesting + """Handle CRUD type operations related to workflows. More interesting stuff regarding workflow execution, step sorting, etc... can be found in the galaxy.workflow module. """ @@ -75,29 +78,36 @@ class WorkflowsManager: self.app = app def get_stored_workflow(self, trans, workflow_id, by_stored_id=True): - """ Use a supplied ID (UUID or encoded stored workflow ID) to find + """Use a supplied ID (UUID or encoded stored workflow ID) to find a workflow. """ if util.is_uuid(workflow_id): # see if they have passed in the UUID for a workflow that is attached to a stored workflow workflow_uuid = uuid.UUID(workflow_id) - workflow_query = trans.sa_session.query(trans.app.model.StoredWorkflow).filter(and_( - trans.app.model.StoredWorkflow.id == trans.app.model.Workflow.stored_workflow_id, - trans.app.model.Workflow.uuid == workflow_uuid - )) + workflow_query = trans.sa_session.query(trans.app.model.StoredWorkflow).filter( + and_( + trans.app.model.StoredWorkflow.id == trans.app.model.Workflow.stored_workflow_id, + trans.app.model.Workflow.uuid == workflow_uuid, + ) + ) elif by_stored_id: workflow_id = decode_id(self.app, workflow_id) - workflow_query = trans.sa_session.query(trans.app.model.StoredWorkflow).\ - filter(trans.app.model.StoredWorkflow.id == workflow_id) + workflow_query = trans.sa_session.query(trans.app.model.StoredWorkflow).filter( + trans.app.model.StoredWorkflow.id == workflow_id + ) else: workflow_id = decode_id(self.app, workflow_id) - workflow_query = trans.sa_session.query(trans.app.model.StoredWorkflow).filter(and_( - trans.app.model.StoredWorkflow.id == trans.app.model.Workflow.stored_workflow_id, - trans.app.model.Workflow.id == workflow_id - )) - stored_workflow = workflow_query.options(joinedload('annotations'), - joinedload('tags'), - subqueryload('latest_workflow').joinedload('steps').joinedload('*')).first() + workflow_query = trans.sa_session.query(trans.app.model.StoredWorkflow).filter( + and_( + trans.app.model.StoredWorkflow.id == trans.app.model.Workflow.stored_workflow_id, + trans.app.model.Workflow.id == workflow_id, + ) + ) + stored_workflow = workflow_query.options( + joinedload("annotations"), + joinedload("tags"), + subqueryload("latest_workflow").joinedload("steps").joinedload("*"), + ).first() if stored_workflow is None: if not by_stored_id: # May have a subworkflow without attached StoredWorkflow object, this was the default prior to 20.09 release. @@ -109,14 +119,19 @@ class WorkflowsManager: return stored_workflow def get_stored_accessible_workflow(self, trans, workflow_id, by_stored_id=True): - """ Get a stored workflow from a encoded stored workflow id and + """Get a stored workflow from a encoded stored workflow id and make sure it accessible to the user. """ stored_workflow = self.get_stored_workflow(trans, workflow_id, by_stored_id=by_stored_id) # check to see if user has permissions to selected workflow if stored_workflow.user != trans.user and not trans.user_is_admin and not stored_workflow.published: - if trans.sa_session.query(trans.app.model.StoredWorkflowUserShareAssociation).filter_by(user=trans.user, stored_workflow=stored_workflow).count() == 0: + if ( + trans.sa_session.query(trans.app.model.StoredWorkflowUserShareAssociation) + .filter_by(user=trans.user, stored_workflow=stored_workflow) + .count() + == 0 + ): message = "Workflow is not owned by or shared with current user" raise exceptions.ItemAccessibilityException(message) @@ -128,13 +143,15 @@ class WorkflowsManager: # To properly serialize them we do need a StoredWorkflow, so we create and attach one here. # We hide the new StoredWorkflow to avoid cluttering the default workflow view. if workflow and workflow.stored_workflow is None and self.check_security(trans, has_workflow=workflow): - stored_workflow = trans.app.model.StoredWorkflow(user=trans.user, name=workflow.name, workflow=workflow, hidden=True) + stored_workflow = trans.app.model.StoredWorkflow( + user=trans.user, name=workflow.name, workflow=workflow, hidden=True + ) trans.sa_session.add(stored_workflow) trans.sa_session.flush() return stored_workflow def get_owned_workflow(self, trans, encoded_workflow_id): - """ Get a workflow (non-stored) from a encoded workflow id and + """Get a workflow (non-stored) from a encoded workflow id and make sure it accessible to the user. """ workflow_id = decode_id(self.app, encoded_workflow_id) @@ -143,7 +160,7 @@ class WorkflowsManager: return workflow def check_security(self, trans, has_workflow, check_ownership=True, check_accessible=True): - """ check accessibility or ownership of workflows, storedworkflows, and + """check accessibility or ownership of workflows, storedworkflows, and workflowinvocations. Throw an exception or returns True if user has needed level of access. """ @@ -170,21 +187,25 @@ class WorkflowsManager: if check_ownership: raise exceptions.ItemOwnershipException() # else check_accessible... - if trans.sa_session.query(model.StoredWorkflowUserShareAssociation).filter_by(user=trans.user, stored_workflow=stored_workflow).count() == 0: + if ( + trans.sa_session.query(model.StoredWorkflowUserShareAssociation) + .filter_by(user=trans.user, stored_workflow=stored_workflow) + .count() + == 0 + ): raise exceptions.ItemAccessibilityException() return True def get_invocation(self, trans, decoded_invocation_id, eager=False): - q = trans.sa_session.query( - self.app.model.WorkflowInvocation - ) + q = trans.sa_session.query(self.app.model.WorkflowInvocation) if eager: - q = q.options(subqueryload(self.app.model.WorkflowInvocation.steps).joinedload( - 'implicit_collection_jobs').joinedload( - 'jobs').joinedload( - 'job').joinedload( - 'input_datasets') + q = q.options( + subqueryload(self.app.model.WorkflowInvocation.steps) + .joinedload("implicit_collection_jobs") + .joinedload("jobs") + .joinedload("job") + .joinedload("input_datasets") ) workflow_invocation = q.get(decoded_invocation_id) if not workflow_invocation: @@ -204,7 +225,8 @@ class WorkflowsManager: if invocation_markdown: runtime_report_config_json = {"markdown": invocation_markdown} return generate_report( - trans, workflow_invocation, + trans, + workflow_invocation, runtime_report_config_json=runtime_report_config_json, plugin_type=generator_plugin_type, target_format=target_format, @@ -225,22 +247,28 @@ class WorkflowsManager: def get_invocation_step(self, trans, decoded_workflow_invocation_step_id): try: - workflow_invocation_step = trans.sa_session.query( - model.WorkflowInvocationStep - ).get(decoded_workflow_invocation_step_id) + workflow_invocation_step = trans.sa_session.query(model.WorkflowInvocationStep).get( + decoded_workflow_invocation_step_id + ) except Exception: raise exceptions.ObjectNotFound() - self.check_security(trans, workflow_invocation_step.workflow_invocation, check_ownership=True, check_accessible=False) + self.check_security( + trans, workflow_invocation_step.workflow_invocation, check_ownership=True, check_accessible=False + ) return workflow_invocation_step def update_invocation_step(self, trans, decoded_workflow_invocation_step_id, action): if action is None: - raise exceptions.RequestParameterMissingException("Updating workflow invocation step requires an action parameter. ") + raise exceptions.RequestParameterMissingException( + "Updating workflow invocation step requires an action parameter. " + ) workflow_invocation_step = self.get_invocation_step(trans, decoded_workflow_invocation_step_id) workflow_invocation = workflow_invocation_step.workflow_invocation if not workflow_invocation.active: - raise exceptions.RequestParameterInvalidException("Attempting to modify the state of a completed workflow invocation.") + raise exceptions.RequestParameterInvalidException( + "Attempting to modify the state of a completed workflow invocation." + ) step = workflow_invocation_step.workflow_step module = module_factory.from_workflow_step(trans, step) @@ -250,8 +278,19 @@ class WorkflowsManager: trans.sa_session.flush() return workflow_invocation_step - def build_invocations_query(self, trans, stored_workflow_id=None, history_id=None, job_id=None, user_id=None, - include_terminal=True, limit=None, offset=None, sort_by=None, sort_desc=None): + def build_invocations_query( + self, + trans, + stored_workflow_id=None, + history_id=None, + job_id=None, + user_id=None, + include_terminal=True, + limit=None, + offset=None, + sort_by=None, + sort_desc=None, + ): """Get invocations owned by the current user.""" sa_session = trans.sa_session invocations_query = sa_session.query(model.WorkflowInvocation) @@ -259,25 +298,17 @@ class WorkflowsManager: stored_workflow = sa_session.query(model.StoredWorkflow).get(stored_workflow_id) if not stored_workflow: raise exceptions.ObjectNotFound() - invocations_query = invocations_query.join( - model.Workflow - ).filter( + invocations_query = invocations_query.join(model.Workflow).filter( model.Workflow.table.c.stored_workflow_id == stored_workflow_id ) if user_id is not None: - invocations_query = invocations_query.join( - model.History - ).filter( - model.History.table.c.user_id == user_id - ) + invocations_query = invocations_query.join(model.History).filter(model.History.table.c.user_id == user_id) if history_id is not None: - invocations_query = invocations_query.filter( - model.WorkflowInvocation.table.c.history_id == history_id - ) + invocations_query = invocations_query.filter(model.WorkflowInvocation.table.c.history_id == history_id) if job_id is not None: - invocations_query = invocations_query.join( - model.WorkflowInvocationStep - ).filter(model.WorkflowInvocationStep.table.c.job_id == job_id) + invocations_query = invocations_query.join(model.WorkflowInvocationStep).filter( + model.WorkflowInvocationStep.table.c.job_id == job_id + ) if not include_terminal: invocations_query = invocations_query.filter( model.WorkflowInvocation.table.c.state.in_(model.WorkflowInvocation.non_terminal_states) @@ -294,17 +325,18 @@ class WorkflowsManager: invocations_query = invocations_query.limit(limit) if offset is not None: invocations_query = invocations_query.offset(offset) - invocations = [inv for inv in invocations_query if self.check_security(trans, - inv, - check_ownership=True, - check_accessible=False)] + invocations = [ + inv + for inv in invocations_query + if self.check_security(trans, inv, check_ownership=True, check_accessible=False) + ] return invocations, total_matches def serialize_workflow_invocation(self, invocation, **kwd): app = self.app view = kwd.get("view", "element") - step_details = util.string_as_bool(kwd.get('step_details', False)) - legacy_job_state = util.string_as_bool(kwd.get('legacy_job_state', False)) + step_details = util.string_as_bool(kwd.get("step_details", False)) + legacy_job_state = util.string_as_bool(kwd.get("legacy_job_state", False)) as_dict = invocation.to_dict(view, step_details=step_details, legacy_job_state=legacy_job_state) return app.security.encode_all_ids(as_dict, recursive=True) @@ -318,7 +350,6 @@ CreatedWorkflow = namedtuple("CreatedWorkflow", ["stored_workflow", "workflow", class WorkflowContentsManager(UsesAnnotations): - def __init__(self, app: MinimalManagerApp): self.app = app self._resource_mapper_function = get_resource_mapper_function(app) @@ -353,7 +384,9 @@ class WorkflowContentsManager(UsesAnnotations): galaxy_interface = Format2ConverterGalaxyInterface() import_options = ImportOptions() import_options.deduplicate_subworkflows = True - as_dict = python_to_workflow(as_dict, galaxy_interface, workflow_directory=workflow_directory, import_options=import_options) + as_dict = python_to_workflow( + as_dict, galaxy_interface, workflow_directory=workflow_directory, import_options=import_options + ) return RawWorkflowDescription(as_dict, workflow_path) @@ -370,10 +403,10 @@ class WorkflowContentsManager(UsesAnnotations): # Put parameters in workflow mode trans.workflow_building_mode = workflow_building_modes.ENABLED # If there's a source, put it in the workflow name. - if 'name' not in data: + if "name" not in data: raise exceptions.RequestParameterInvalidException(f"Invalid workflow format detected [{data}]") - workflow_input_name = data['name'] + workflow_input_name = data["name"] imported_sufix = f"(imported from {source})" if source and imported_sufix not in workflow_input_name: name = f"{workflow_input_name} {imported_sufix}" @@ -385,8 +418,8 @@ class WorkflowContentsManager(UsesAnnotations): workflow_create_options, name=name, ) - if 'uuid' in data: - workflow.uuid = data['uuid'] + if "uuid" in data: + workflow.uuid = data["uuid"] # Connect up stored = model.StoredWorkflow() @@ -397,10 +430,10 @@ class WorkflowContentsManager(UsesAnnotations): stored.user = trans.user stored.published = workflow_create_options.publish stored.hidden = hidden - if data['annotation']: - annotation = sanitize_html(data['annotation']) + if data["annotation"]: + annotation = sanitize_html(data["annotation"]) self.add_item_annotation(trans.sa_session, stored.user, stored, annotation) - workflow_tags = data.get('tags', []) + workflow_tags = data.get("tags", []) trans.app.tag_handler.set_tags_from_list(user=trans.user, item=stored, new_tags_list=workflow_tags) # Persist @@ -415,13 +448,11 @@ class WorkflowContentsManager(UsesAnnotations): trans.sa_session.flush() - return CreatedWorkflow( - stored_workflow=stored, - workflow=workflow, - missing_tools=missing_tool_tups - ) + return CreatedWorkflow(stored_workflow=stored, workflow=workflow, missing_tools=missing_tool_tups) - def update_workflow_from_raw_description(self, trans, stored_workflow, raw_workflow_description, workflow_update_options): + def update_workflow_from_raw_description( + self, trans, stored_workflow, raw_workflow_description, workflow_update_options + ): raw_workflow_description = self.ensure_raw_description(raw_workflow_description) # Put parameters in workflow mode @@ -452,12 +483,12 @@ class WorkflowContentsManager(UsesAnnotations): if workflow_update_options.update_stored_workflow_attributes: update_dict = raw_workflow_description.as_dict - if 'name' in update_dict: - sanitized_name = sanitize_html(update_dict['name']) + if "name" in update_dict: + sanitized_name = sanitize_html(update_dict["name"]) workflow.name = sanitized_name stored_workflow.name = sanitized_name - if 'annotation' in update_dict: - newAnnotation = sanitize_html(update_dict['annotation']) + if "annotation" in update_dict: + newAnnotation = sanitize_html(update_dict["annotation"]) sa_session = None if dry_run else trans.sa_session self.add_item_annotation(sa_session, stored_workflow.user, stored_workflow, newAnnotation) @@ -474,7 +505,9 @@ class WorkflowContentsManager(UsesAnnotations): errors.append("This workflow contains cycles") return workflow, errors - def _workflow_from_raw_description(self, trans, raw_workflow_description, workflow_state_resolution_options, name, **kwds): + def _workflow_from_raw_description( + self, trans, raw_workflow_description, workflow_state_resolution_options, name, **kwds + ): # don't commit the workflow or attach its part to the sa session - just build a # a transient model to operate on or render. dry_run = kwds.pop("dry_run", False) @@ -487,16 +520,16 @@ class WorkflowContentsManager(UsesAnnotations): workflow = model.Workflow() workflow.name = name - if 'report' in data: - workflow.reports_config = data['report'] - workflow.license = data.get('license') - workflow.creator_metadata = data.get('creator') + if "report" in data: + workflow.reports_config = data["report"] + workflow.license = data.get("license") + workflow.creator_metadata = data.get("creator") - if 'license' in data: - workflow.license = data['license'] + if "license" in data: + workflow.license = data["license"] - if 'creator' in data: - workflow.creator_metadata = data['creator'] + if "creator" in data: + workflow.creator_metadata = data["creator"] # Assume no errors until we find a step that has some workflow.has_errors = False @@ -512,7 +545,9 @@ class WorkflowContentsManager(UsesAnnotations): if subworkflows: subworkflow_id_map = {} for key, subworkflow_dict in subworkflows.items(): - subworkflow = self.__build_embedded_subworkflow(trans, subworkflow_dict, workflow_state_resolution_options) + subworkflow = self.__build_embedded_subworkflow( + trans, subworkflow_dict, workflow_state_resolution_options + ) subworkflow_id_map[key] = subworkflow # Keep track of tools required by the workflow that are not available in @@ -521,7 +556,9 @@ class WorkflowContentsManager(UsesAnnotations): missing_tool_tups = [] for step_dict in self.__walk_step_dicts(data): if not dry_run: - self.__load_subworkflows(trans, step_dict, subworkflow_id_map, workflow_state_resolution_options, dry_run=dry_run) + self.__load_subworkflows( + trans, step_dict, subworkflow_id_map, workflow_state_resolution_options, dry_run=dry_run + ) module_kwds = workflow_state_resolution_options.dict() module_kwds.update(kwds) # TODO: maybe drop this? @@ -529,7 +566,7 @@ class WorkflowContentsManager(UsesAnnotations): module, step = self.__module_from_dict(trans, steps, steps_by_external_id, step_dict, **module_kwds) is_tool = is_tool_module_type(module.type) if is_tool and module.tool is None: - missing_tool_tup = (module.tool_id, module.get_name(), module.tool_version, step_dict['id']) + missing_tool_tup = (module.tool_id, module.get_name(), module.tool_version, step_dict["id"]) if missing_tool_tup not in missing_tool_tups: missing_tool_tups.append(missing_tool_tup) if module.get_errors(): @@ -544,7 +581,7 @@ class WorkflowContentsManager(UsesAnnotations): return workflow, missing_tool_tups def workflow_to_dict(self, trans, stored, style="export", version=None, history=None): - """ Export the workflow contents to a dictionary ready for JSON-ification and to be + """Export the workflow contents to a dictionary ready for JSON-ification and to be sent out via API for instance. There are three styles of export allowed 'export', 'instance', and 'editor'. The Galaxy team will do its best to preserve the backward compatibility of the 'export' style - this is the export method meant to be portable across Galaxy instances and over @@ -552,10 +589,11 @@ class WorkflowContentsManager(UsesAnnotations): option describes the workflow in a context more tied to the current Galaxy instance and includes fields like 'url' and 'url' and actual unencoded step ids instead of 'order_index'. """ + def to_format_2(wf_dict, **kwds): return from_galaxy_native(wf_dict, None, **kwds) - if version == '': + if version == "": version = None if version is not None: version = int(version) @@ -581,11 +619,11 @@ class WorkflowContentsManager(UsesAnnotations): elif style == "ga": wf_dict = self._workflow_to_dict_export(trans, stored, workflow=workflow) else: - raise exceptions.RequestParameterInvalidException(f'Unknown workflow style {style}') + raise exceptions.RequestParameterInvalidException(f"Unknown workflow style {style}") if version is not None: - wf_dict['version'] = version + wf_dict["version"] = version else: - wf_dict['version'] = len(stored.workflows) - 1 + wf_dict["version"] = len(stored.workflows) - 1 return wf_dict def _sync_stored_workflow(self, trans, stored_workflow): @@ -605,9 +643,9 @@ class WorkflowContentsManager(UsesAnnotations): Builds workflow dictionary used by run workflow form """ if len(workflow.steps) == 0: - raise exceptions.MessageException('Workflow cannot be run because it does not have any steps.') + raise exceptions.MessageException("Workflow cannot be run because it does not have any steps.") if attach_ordered_steps(workflow, workflow.steps): - raise exceptions.MessageException('Workflow cannot be run because it contains cycles.') + raise exceptions.MessageException("Workflow cannot be run because it contains cycles.") trans.workflow_building_mode = workflow_building_modes.USE_HISTORY module_injector = WorkflowModuleInjector(trans) has_upgrade_messages = False @@ -624,7 +662,7 @@ class WorkflowContentsManager(UsesAnnotations): continue if step.upgrade_messages: has_upgrade_messages = True - if step.type in ('tool', 'subworkflow', None): + if step.type in ("tool", "subworkflow", None): if step.module.version_changes: step_version_changes.extend(step.module.version_changes) step_errors = step.module.get_errors() @@ -640,47 +678,55 @@ class WorkflowContentsManager(UsesAnnotations): step_models = [] for step in workflow.steps: step_model = None - if step.type == 'tool': + if step.type == "tool": incoming: Dict[str, Any] = {} - tool = trans.app.toolbox.get_tool(step.tool_id, tool_version=step.tool_version, tool_uuid=step.tool_uuid) + tool = trans.app.toolbox.get_tool( + step.tool_id, tool_version=step.tool_version, tool_uuid=step.tool_uuid + ) params_to_incoming(incoming, tool.inputs, step.state.inputs, trans.app) - step_model = tool.to_json(trans, incoming, workflow_building_mode=workflow_building_modes.USE_HISTORY, history=history) - step_model['post_job_actions'] = [{ - 'short_str': ActionBox.get_short_str(pja), - 'action_type': pja.action_type, - 'output_name': pja.output_name, - 'action_arguments': pja.action_arguments - } for pja in step.post_job_actions] + step_model = tool.to_json( + trans, incoming, workflow_building_mode=workflow_building_modes.USE_HISTORY, history=history + ) + step_model["post_job_actions"] = [ + { + "short_str": ActionBox.get_short_str(pja), + "action_type": pja.action_type, + "output_name": pja.output_name, + "action_arguments": pja.action_arguments, + } + for pja in step.post_job_actions + ] else: inputs = step.module.get_runtime_inputs(connections=step.output_connections) - step_model = { - 'inputs': [input.to_dict(trans) for input in inputs.values()] + step_model = {"inputs": [input.to_dict(trans) for input in inputs.values()]} + step_model["replacement_parameters"] = step.module.get_replacement_parameters(step) + step_model["step_type"] = step.type + step_model["step_label"] = step.label + step_model["step_name"] = step.module.get_name() + step_model["step_version"] = step.module.get_version() + step_model["step_index"] = step.order_index + step_model["output_connections"] = [ + { + "input_step_index": step_order_indices.get(oc.input_step_id), + "output_step_index": step_order_indices.get(oc.output_step_id), + "input_name": oc.input_name, + "output_name": oc.output_name, } - step_model['replacement_parameters'] = step.module.get_replacement_parameters(step) - step_model['step_type'] = step.type - step_model['step_label'] = step.label - step_model['step_name'] = step.module.get_name() - step_model['step_version'] = step.module.get_version() - step_model['step_index'] = step.order_index - step_model['output_connections'] = [{ - 'input_step_index': step_order_indices.get(oc.input_step_id), - 'output_step_index': step_order_indices.get(oc.output_step_id), - 'input_name': oc.input_name, - 'output_name': oc.output_name - } for oc in step.output_connections] + for oc in step.output_connections + ] if step.annotations: - step_model['annotation'] = step.annotations[0].annotation + step_model["annotation"] = step.annotations[0].annotation if step.upgrade_messages: - step_model['messages'] = step.upgrade_messages + step_model["messages"] = step.upgrade_messages step_models.append(step_model) return { - 'id': trans.app.security.encode_id(stored.id), - 'history_id': trans.app.security.encode_id(history.id) if history else None, - 'name': stored.name, - 'steps': step_models, - 'step_version_changes': step_version_changes, - 'has_upgrade_messages': has_upgrade_messages, - 'workflow_resource_parameters': self._workflow_resource_parameters(trans, stored, workflow), + "id": trans.app.security.encode_id(stored.id), + "history_id": trans.app.security.encode_id(history.id) if history else None, + "name": stored.name, + "steps": step_models, + "step_version_changes": step_version_changes, + "has_upgrade_messages": has_upgrade_messages, + "workflow_resource_parameters": self._workflow_resource_parameters(trans, stored, workflow), } def _workflow_to_dict_preview(self, trans, workflow): @@ -689,9 +735,9 @@ class WorkflowContentsManager(UsesAnnotations): Used to create embedded workflow previews. """ if len(workflow.steps) == 0: - raise exceptions.MessageException('Workflow cannot be run because it does not have any steps.') + raise exceptions.MessageException("Workflow cannot be run because it does not have any steps.") if attach_ordered_steps(workflow, workflow.steps): - raise exceptions.MessageException('Workflow cannot be run because it contains cycles.') + raise exceptions.MessageException("Workflow cannot be run because it contains cycles.") # Ensure that the user has a history trans.get_history(most_recent=True, create=True) @@ -704,14 +750,17 @@ class WorkflowContentsManager(UsesAnnotations): conns = step.input_connections_by_name[prefix + param.name] if not isinstance(conns, list): conns = [conns] - value_list = ["Output '%s' from Step %d." % (conn.output_name, int(conn.output_step.order_index) + 1) for conn in conns] + value_list = [ + "Output '%s' from Step %d." % (conn.output_name, int(conn.output_step.order_index) + 1) + for conn in conns + ] value = ",".join(value_list) else: value = "Select at Runtime." else: - value = param.value_to_display_text(raw_value) or 'Unavailable.' + value = param.value_to_display_text(raw_value) or "Unavailable." input_dict["value"] = value - if hasattr(step, 'upgrade_messages') and step.upgrade_messages and param.name in step.upgrade_messages: + if hasattr(step, "upgrade_messages") and step.upgrade_messages and param.name in step.upgrade_messages: input_dict["upgrade_messages"] = step.upgrade_messages[param.name] def do_inputs(inputs, values, prefix, step, other_values=None): @@ -726,17 +775,27 @@ class WorkflowContentsManager(UsesAnnotations): nested_input_dicts = [] for i in range(len(repeat_values)): nested_input_dict = {} - index = repeat_values[i]['__index__'] + index = repeat_values[i]["__index__"] nested_input_dict["title"] = "%i. %s" % (i + 1, input.title) - nested_input_dict["inputs"] = do_inputs(input.inputs, repeat_values[i], f"{prefix + input.name}_{str(index)}|", step, other_values) + nested_input_dict["inputs"] = do_inputs( + input.inputs, + repeat_values[i], + f"{prefix + input.name}_{str(index)}|", + step, + other_values, + ) nested_input_dicts.append(nested_input_dict) input_dict["inputs"] = nested_input_dicts elif input.type == "conditional": group_values = values[input.name] - current_case = group_values['__current_case__'] + current_case = group_values["__current_case__"] new_prefix = f"{prefix + input.name}|" - row_for_param(input_dict, input.test_param, group_values[input.test_param.name], other_values, prefix, step) - input_dict["inputs"] = do_inputs(input.cases[current_case].inputs, group_values, new_prefix, step, other_values) + row_for_param( + input_dict, input.test_param, group_values[input.test_param.name], other_values, prefix, step + ) + input_dict["inputs"] = do_inputs( + input.cases[current_case].inputs, group_values, new_prefix, step, other_values + ) elif input.type == "section": new_prefix = f"{prefix + input.name}|" group_values = values[input.name] @@ -760,14 +819,14 @@ class WorkflowContentsManager(UsesAnnotations): step_dict["label"] = f"Unknown Tool with id '{e.tool_id}'" step_dicts.append(step_dict) continue - if step.type == 'tool' or step.type is None: + if step.type == "tool" or step.type is None: tool = trans.app.toolbox.get_tool(step.tool_id) if tool: step_dict["label"] = step.label or tool.name else: step_dict["label"] = f"Unknown Tool with id '{step.tool_id}'" step_dict["inputs"] = do_inputs(tool.inputs, step.state.inputs, "", step) - elif step.type == 'subworkflow': + elif step.type == "subworkflow": step_dict["label"] = step.label or (step.subworkflow.name if step.subworkflow else "Missing workflow.") errors = step.module.get_errors() if errors: @@ -784,20 +843,19 @@ class WorkflowContentsManager(UsesAnnotations): } def _workflow_resource_parameters(self, trans, stored, workflow): - """Get workflow scheduling resource parameters for this user and workflow or None if not configured. - """ + """Get workflow scheduling resource parameters for this user and workflow or None if not configured.""" return self._resource_mapper_function(trans=trans, stored_workflow=stored, workflow=workflow) def _workflow_to_dict_editor(self, trans, stored, workflow, tooltip=True, is_subworkflow=False): # Pack workflow data into a dictionary and return data = {} - data['name'] = workflow.name - data['steps'] = {} - data['upgrade_messages'] = {} - data['report'] = workflow.reports_config or {} - data['license'] = workflow.license - data['creator'] = workflow.creator_metadata - data['annotation'] = self.get_item_annotation_str(trans.sa_session, trans.user, stored) or '' + data["name"] = workflow.name + data["steps"] = {} + data["upgrade_messages"] = {} + data["report"] = workflow.reports_config or {} + data["license"] = workflow.license + data["creator"] = workflow.creator_metadata + data["annotation"] = self.get_item_annotation_str(trans.sa_session, trans.user, stored) or "" output_label_index = set() input_step_types = set(workflow.input_step_types) @@ -806,7 +864,7 @@ class WorkflowContentsManager(UsesAnnotations): # Load from database representation module = module_factory.from_workflow_step(trans, step, exact_tools=False) if not module: - raise exceptions.MessageException(f'Unrecognized step type: {step.type}') + raise exceptions.MessageException(f"Unrecognized step type: {step.type}") # Load label from state of data input modules, necessary for backward compatibility self.__set_default_label(step, module, step.tool_inputs) # Fix any missing parameters @@ -815,31 +873,31 @@ class WorkflowContentsManager(UsesAnnotations): upgrade_message_dict[module.get_name()] = "\n".join(module.version_changes) # Get user annotation. config_form = module.get_config_form(step=step) - annotation_str = self.get_item_annotation_str(trans.sa_session, trans.user, step) or '' + annotation_str = self.get_item_annotation_str(trans.sa_session, trans.user, step) or "" # Pack attributes into plain dictionary step_dict = { - 'id': step.order_index, - 'type': module.type, - 'label': module.label, - 'content_id': module.get_content_id(), - 'name': module.get_name(), - 'tool_state': module.get_tool_state(), - 'errors': module.get_errors(), - 'inputs': module.get_all_inputs(connectable_only=True), - 'outputs': module.get_all_outputs(), - 'config_form': config_form, - 'annotation': annotation_str, - 'post_job_actions': {}, - 'uuid': str(step.uuid) if step.uuid else None, - 'workflow_outputs': [] + "id": step.order_index, + "type": module.type, + "label": module.label, + "content_id": module.get_content_id(), + "name": module.get_name(), + "tool_state": module.get_tool_state(), + "errors": module.get_errors(), + "inputs": module.get_all_inputs(connectable_only=True), + "outputs": module.get_all_outputs(), + "config_form": config_form, + "annotation": annotation_str, + "post_job_actions": {}, + "uuid": str(step.uuid) if step.uuid else None, + "workflow_outputs": [], } if tooltip: - step_dict['tooltip'] = module.get_tooltip(static_path=url_for('/static')) + step_dict["tooltip"] = module.get_tooltip(static_path=url_for("/static")) # Connections input_connections = step.input_connections input_connections_type = {} multiple_input = {} # Boolean value indicating if this can be multiple - if (step.type is None or step.type == 'tool') and module.tool: + if (step.type is None or step.type == "tool") and module.tool: # Determine full (prefixed) names of valid input datasets data_input_names = {} @@ -851,16 +909,15 @@ class WorkflowContentsManager(UsesAnnotations): input_connections_type[input.name] = "dataset" if isinstance(input, DataCollectionToolParameter): input_connections_type[input.name] = "dataset_collection" + visit_input_values(module.tool.inputs, module.state.inputs, callback) # post_job_actions pja_dict = {} for pja in step.post_job_actions: pja_dict[pja.action_type + pja.output_name] = dict( - action_type=pja.action_type, - output_name=pja.output_name, - action_arguments=pja.action_arguments + action_type=pja.action_type, output_name=pja.output_name, action_arguments=pja.action_arguments ) - step_dict['post_job_actions'] = pja_dict + step_dict["post_job_actions"] = pja_dict # workflow outputs outputs = [] @@ -870,21 +927,21 @@ class WorkflowContentsManager(UsesAnnotations): output_label = output.label output_name = output.output_name output_uuid = str(output.uuid) if output.uuid else None - outputs.append({"output_name": output_name, - "uuid": output_uuid, - "label": output_label}) + outputs.append({"output_name": output_name, "uuid": output_uuid, "label": output_label}) if output_label is not None: if output_label in output_label_index: if output_label not in output_label_duplicate: output_label_duplicate.add(output_label) else: output_label_index.add(output_label) - step_dict['workflow_outputs'] = outputs + step_dict["workflow_outputs"] = outputs if len(output_label_duplicate) > 0: output_label_duplicate_string = ", ".join(output_label_duplicate) - upgrade_message_dict['output_label_duplicate'] = f"Ignoring duplicate labels: {output_label_duplicate_string}." + upgrade_message_dict[ + "output_label_duplicate" + ] = f"Ignoring duplicate labels: {output_label_duplicate_string}." if upgrade_message_dict: - data['upgrade_messages'][step.order_index] = upgrade_message_dict + data["upgrade_messages"][step.order_index] = upgrade_message_dict # Encode input connections as dictionary input_conn_dict: model.InputConnDictType = {} @@ -900,14 +957,14 @@ class WorkflowContentsManager(UsesAnnotations): input_conn_dict[conn.input_name] = [conn_dict] else: input_conn_dict[conn.input_name] = conn_dict - step_dict['input_connections'] = input_conn_dict + step_dict["input_connections"] = input_conn_dict # Position - step_dict['position'] = step.position + step_dict["position"] = step.position # Add to return value - data['steps'][step.order_index] = step_dict + data["steps"][step.order_index] = step_dict if is_subworkflow: - data['steps'] = self._resolve_collection_type(data['steps']) + data["steps"] = self._resolve_collection_type(data["steps"]) return data @staticmethod @@ -916,40 +973,40 @@ class WorkflowContentsManager(UsesAnnotations): Given a tool step and its input steps guess that maximum level of mapping over. All data outputs of a step need to be mapped over to this level. """ - max_map_over = '' - for input_name, input_connections in current_step['input_connections'].items(): + max_map_over = "" + for input_name, input_connections in current_step["input_connections"].items(): if isinstance(input_connections, dict): # if input does not accept multiple inputs input_connections = [input_connections] for input_value in input_connections: current_data_input = None - for current_input in current_step['inputs']: - if current_input['name'] == input_name: + for current_input in current_step["inputs"]: + if current_input["name"] == input_name: current_data_input = current_input # we've got one of the tools' input data definitions break if current_data_input is None: log.info(f"failed to find input {input_name} for get_step_map_over") continue - input_step = steps[input_value['id']] - for input_step_data_output in input_step['outputs']: - if input_step_data_output['name'] == input_value['output_name']: - collection_type = input_step_data_output.get('collection_type') + input_step = steps[input_value["id"]] + for input_step_data_output in input_step["outputs"]: + if input_step_data_output["name"] == input_value["output_name"]: + collection_type = input_step_data_output.get("collection_type") # This is the defined incoming collection type, in reality there may be additional # mapping over of the workflows' data input, but this should be taken care of by the workflow editor / # outer workflow. if collection_type: - if current_data_input.get('input_type') == 'dataset' and current_data_input.get('multiple'): + if current_data_input.get("input_type") == "dataset" and current_data_input.get("multiple"): # We reduce the innermost collection - if ':' in collection_type: + if ":" in collection_type: # more than one layer of nesting and multiple="true" input, # we consume the innermost collection - collection_type = ":".join(collection_type.rsplit(':')[:-1]) + collection_type = ":".join(collection_type.rsplit(":")[:-1]) else: # We've reduced a list or a pair collection_type = None - elif current_data_input.get('input_type') == 'dataset_collection': - current_collection_types = current_data_input['collection_types'] + elif current_data_input.get("input_type") == "dataset_collection": + current_collection_types = current_data_input["collection_types"] if not current_collection_types: # Accepts any input dataset collection, no mapping collection_type = None @@ -958,19 +1015,19 @@ class WorkflowContentsManager(UsesAnnotations): collection_type = None else: outer_map_over = collection_type - for accepted_collection_type in current_data_input['collection_types']: + for accepted_collection_type in current_data_input["collection_types"]: # need to find the lowest level of mapping over, # for collection_type = 'list:list:list' and accepted_collection_type = ['list:list', 'list'] # it'd be outer_map_over == 'list' if collection_type.endswith(accepted_collection_type): - _outer_map_over = collection_type[:-(len(accepted_collection_type) + 1)] - if len(_outer_map_over.split(':')) < len(outer_map_over.split(':')): + _outer_map_over = collection_type[: -(len(accepted_collection_type) + 1)] + if len(_outer_map_over.split(":")) < len(outer_map_over.split(":")): outer_map_over = _outer_map_over collection_type = outer_map_over # If there is mapping over, we're going to assume it is linked, everything else is (probably) # too hard to display in the workflow editor. With this assumption we should be able to # set the maximum mapping over level to the most deeply nested map_over - if collection_type and len(collection_type.split(':')) >= len(max_map_over.split(':')): + if collection_type and len(collection_type.split(":")) >= len(max_map_over.split(":")): max_map_over = collection_type if max_map_over: return max_map_over @@ -985,26 +1042,26 @@ class WorkflowContentsManager(UsesAnnotations): """ for order_index in sorted(steps): step = steps[order_index] - if step['type'] == 'tool' and not step.get('errors'): + if step["type"] == "tool" and not step.get("errors"): map_over = self.get_step_map_over(step, steps) - for step_data_output in step['outputs']: - if step_data_output.get('collection_type_source') and step_data_output['collection_type'] is None: - collection_type_source = step_data_output['collection_type_source'] - for input_connection in step['input_connections'].get(collection_type_source, []): - input_step = steps[input_connection['id']] - for input_step_data_output in input_step['outputs']: - if input_step_data_output['name'] == input_connection['output_name']: - step_data_output['collection_type'] = input_step_data_output.get('collection_type') + for step_data_output in step["outputs"]: + if step_data_output.get("collection_type_source") and step_data_output["collection_type"] is None: + collection_type_source = step_data_output["collection_type_source"] + for input_connection in step["input_connections"].get(collection_type_source, []): + input_step = steps[input_connection["id"]] + for input_step_data_output in input_step["outputs"]: + if input_step_data_output["name"] == input_connection["output_name"]: + step_data_output["collection_type"] = input_step_data_output.get("collection_type") if map_over: collection_type = map_over - step_data_output['collection'] = True - if step_data_output.get('collection_type'): + step_data_output["collection"] = True + if step_data_output.get("collection_type"): collection_type = f"{map_over}:{step_data_output['collection_type']}" - step_data_output['collection_type'] = collection_type + step_data_output["collection_type"] = collection_type return steps def _workflow_to_dict_export(self, trans, stored=None, workflow=None, internal=False): - """ Export the workflow contents to a dictionary ready for JSON-ification and export. + """Export the workflow contents to a dictionary ready for JSON-ification and export. If internal, use content_ids instead subworkflow definitions. """ @@ -1012,7 +1069,7 @@ class WorkflowContentsManager(UsesAnnotations): tag_str = "" if stored is not None: if stored.id: - annotation_str = self.get_item_annotation_str(trans.sa_session, trans.user, stored) or '' + annotation_str = self.get_item_annotation_str(trans.sa_session, trans.user, stored) or "" tag_str = stored.make_tag_string_list() else: # dry run with flushed workflow objects, just use the annotation @@ -1022,91 +1079,86 @@ class WorkflowContentsManager(UsesAnnotations): # Pack workflow data into a dictionary and return data: Dict[str, Any] = {} - data['a_galaxy_workflow'] = 'true' # Placeholder for identifying galaxy workflow - data['format-version'] = "0.1" - data['name'] = workflow.name - data['annotation'] = annotation_str - data['tags'] = tag_str + data["a_galaxy_workflow"] = "true" # Placeholder for identifying galaxy workflow + data["format-version"] = "0.1" + data["name"] = workflow.name + data["annotation"] = annotation_str + data["tags"] = tag_str if workflow.uuid is not None: - data['uuid'] = str(workflow.uuid) + data["uuid"] = str(workflow.uuid) steps: Dict[int, Dict[str, Any]] = {} - data['steps'] = steps + data["steps"] = steps if workflow.reports_config: - data['report'] = workflow.reports_config + data["report"] = workflow.reports_config if workflow.creator_metadata: - data['creator'] = workflow.creator_metadata + data["creator"] = workflow.creator_metadata if workflow.license: - data['license'] = workflow.license + data["license"] = workflow.license # For each step, rebuild the form and encode the state for step in workflow.steps: # Load from database representation module = module_factory.from_workflow_step(trans, step) if not module: - raise exceptions.MessageException(f'Unrecognized step type: {step.type}') + raise exceptions.MessageException(f"Unrecognized step type: {step.type}") # Get user annotation. - annotation_str = self.get_item_annotation_str(trans.sa_session, trans.user, step) or '' + annotation_str = self.get_item_annotation_str(trans.sa_session, trans.user, step) or "" content_id = module.get_content_id() # Export differences for backward compatibility tool_state = module.get_export_state() # Step info step_dict = { - 'id': step.order_index, - 'type': module.type, - 'content_id': content_id, - 'tool_id': content_id, # For workflows exported to older Galaxies, - # eliminate after a few years... - 'tool_version': step.tool_version, - 'name': module.get_name(), - 'tool_state': json.dumps(tool_state), - 'errors': module.get_errors(), - 'uuid': str(step.uuid), - 'label': step.label or None, - 'annotation': annotation_str + "id": step.order_index, + "type": module.type, + "content_id": content_id, + "tool_id": content_id, # For workflows exported to older Galaxies, + # eliminate after a few years... + "tool_version": step.tool_version, + "name": module.get_name(), + "tool_state": json.dumps(tool_state), + "errors": module.get_errors(), + "uuid": str(step.uuid), + "label": step.label or None, + "annotation": annotation_str, } # Add tool shed repository information and post-job actions to step dict. - if module.type == 'tool': + if module.type == "tool": if module.tool and module.tool.tool_shed: step_dict["tool_shed_repository"] = { - 'name': module.tool.repository_name, - 'owner': module.tool.repository_owner, - 'changeset_revision': module.tool.changeset_revision, - 'tool_shed': module.tool.tool_shed + "name": module.tool.repository_name, + "owner": module.tool.repository_owner, + "changeset_revision": module.tool.changeset_revision, + "tool_shed": module.tool.tool_shed, } tool_representation = None dynamic_tool = step.dynamic_tool if dynamic_tool: tool_representation = dynamic_tool.value - step_dict['tool_representation'] = tool_representation - if util.is_uuid(step_dict['content_id']): - step_dict['content_id'] = None - step_dict['tool_id'] = None + step_dict["tool_representation"] = tool_representation + if util.is_uuid(step_dict["content_id"]): + step_dict["content_id"] = None + step_dict["tool_id"] = None pja_dict = {} for pja in step.post_job_actions: pja_dict[pja.action_type + pja.output_name] = dict( - action_type=pja.action_type, - output_name=pja.output_name, - action_arguments=pja.action_arguments) - step_dict['post_job_actions'] = pja_dict + action_type=pja.action_type, output_name=pja.output_name, action_arguments=pja.action_arguments + ) + step_dict["post_job_actions"] = pja_dict - if module.type == 'subworkflow' and not internal: - del step_dict['content_id'] - del step_dict['errors'] - del step_dict['tool_version'] - del step_dict['tool_state'] + if module.type == "subworkflow" and not internal: + del step_dict["content_id"] + del step_dict["errors"] + del step_dict["tool_version"] + del step_dict["tool_state"] subworkflow = step.subworkflow - subworkflow_as_dict = self._workflow_to_dict_export( - trans, - stored=None, - workflow=subworkflow - ) - step_dict['subworkflow'] = subworkflow_as_dict + subworkflow_as_dict = self._workflow_to_dict_export(trans, stored=None, workflow=subworkflow) + step_dict["subworkflow"] = subworkflow_as_dict # Data inputs, legacy section not used anywhere within core input_dicts = [] step_state = module.state.inputs or {} - if module.type != 'tool': + if module.type != "tool": name = step_state.get("name") or module.label if name: input_dicts.append({"name": name, "description": annotation_str}) @@ -1118,8 +1170,10 @@ class WorkflowContentsManager(UsesAnnotations): # Input type is described by a dict, e.g. indexed parameters. for partval in val.values(): if type(partval) == RuntimeValue: - input_dicts.append({"name": name, "description": f"runtime parameter for tool {module.get_name()}"}) - step_dict['inputs'] = input_dicts + input_dicts.append( + {"name": name, "description": f"runtime parameter for tool {module.get_name()}"} + ) + step_dict["inputs"] = input_dicts # User outputs workflow_outputs_dicts = [] @@ -1130,13 +1184,13 @@ class WorkflowContentsManager(UsesAnnotations): uuid=str(workflow_output.uuid) if workflow_output.uuid is not None else None, ) workflow_outputs_dicts.append(workflow_output_dict) - step_dict['workflow_outputs'] = workflow_outputs_dicts + step_dict["workflow_outputs"] = workflow_outputs_dicts # All step outputs - step_dict['outputs'] = [] + step_dict["outputs"] = [] if type(module) is ToolModule: for output in module.get_data_outputs(): - step_dict['outputs'].append({'name': output['name'], 'type': output['extensions'][0]}) + step_dict["outputs"].append({"name": output["name"], "type": output["extensions"][0]}) step_in = {} for step_input in step.inputs: @@ -1148,13 +1202,14 @@ class WorkflowContentsManager(UsesAnnotations): # Connections input_connections = step.input_connections - if step.type is None or step.type == 'tool': + if step.type is None or step.type == "tool": # Determine full (prefixed) names of valid input datasets data_input_names = {} def callback(input, prefixed_name, **kwargs): if isinstance(input, DataToolParameter) or isinstance(input, DataCollectionToolParameter): data_input_names[prefixed_name] = True + # FIXME: this updates modules silently right now; messages from updates should be provided. module.check_and_update_state() if module.tool: @@ -1170,10 +1225,7 @@ class WorkflowContentsManager(UsesAnnotations): for conn in input_connections: if conn.input_name != input_name: continue - input_conn = dict( - id=conn.output_step.order_index, - output_name=conn.output_name - ) + input_conn = dict(id=conn.output_step.order_index, output_name=conn.output_name) if conn.input_subworkflow_step is not None: subworkflow_step_id = conn.input_subworkflow_step.order_index input_conn["input_subworkflow_step_id"] = subworkflow_step_id @@ -1196,9 +1248,9 @@ class WorkflowContentsManager(UsesAnnotations): back_compat_input_conn_dict[input_name] = input_conn_list[0] else: back_compat_input_conn_dict[input_name] = input_conn_list - step_dict['input_connections'] = back_compat_input_conn_dict + step_dict["input_connections"] = back_compat_input_conn_dict # Position - step_dict['position'] = step.position + step_dict["position"] = step.position # Add to return value steps[step.order_index] = step_dict return data @@ -1206,21 +1258,21 @@ class WorkflowContentsManager(UsesAnnotations): def _workflow_to_dict_instance(self, stored, workflow, legacy=True): encode = self.app.security.encode_id sa_session = self.app.model.context - item = stored.to_dict(view='element', value_mapper={'id': encode}) - item['name'] = workflow.name - item['url'] = url_for('workflow', id=item['id']) - item['owner'] = stored.user.username + item = stored.to_dict(view="element", value_mapper={"id": encode}) + item["name"] = workflow.name + item["url"] = url_for("workflow", id=item["id"]) + item["owner"] = stored.user.username inputs = {} for step in workflow.input_steps: step_type = step.type - step_label = step.label or step.tool_inputs.get('name') + step_label = step.label or step.tool_inputs.get("name") if step_label: label = step_label elif step_type == "data_input": label = "Input Dataset" elif step_type == "data_collection_input": label = "Input Dataset Collection" - elif step_type == 'parameter_input': + elif step_type == "parameter_input": label = "Input Parameter" else: raise ValueError(f"Invalid step_type {step_type}") @@ -1229,11 +1281,11 @@ class WorkflowContentsManager(UsesAnnotations): else: index = step.order_index step_uuid = str(step.uuid) if step.uuid else None - inputs[index] = {'label': label, 'value': '', 'uuid': step_uuid} - item['inputs'] = inputs - item['annotation'] = self.get_item_annotation_str(sa_session, stored.user, stored) - item['license'] = workflow.license - item['creator'] = workflow.creator_metadata + inputs[index] = {"label": label, "value": "", "uuid": step_uuid} + item["inputs"] = inputs + item["annotation"] = self.get_item_annotation_str(sa_session, stored.user, stored) + item["license"] = workflow.license + item["creator"] = workflow.creator_metadata steps = {} steps_to_order_index = {} for step in workflow.steps: @@ -1241,37 +1293,41 @@ class WorkflowContentsManager(UsesAnnotations): for step in workflow.steps: step_id = step.id if legacy else step.order_index step_type = step.type - step_dict = {'id': step_id, - 'type': step_type, - 'tool_id': step.tool_id, - 'tool_version': step.tool_version, - 'annotation': self.get_item_annotation_str(sa_session, stored.user, step), - 'tool_inputs': step.tool_inputs, - 'input_steps': {}} + step_dict = { + "id": step_id, + "type": step_type, + "tool_id": step.tool_id, + "tool_version": step.tool_version, + "annotation": self.get_item_annotation_str(sa_session, stored.user, step), + "tool_inputs": step.tool_inputs, + "input_steps": {}, + } - if step_type == 'subworkflow': - del step_dict['tool_id'] - del step_dict['tool_version'] - del step_dict['tool_inputs'] - step_dict['workflow_id'] = encode(step.subworkflow.id) + if step_type == "subworkflow": + del step_dict["tool_id"] + del step_dict["tool_version"] + del step_dict["tool_inputs"] + step_dict["workflow_id"] = encode(step.subworkflow.id) for conn in step.input_connections: step_id = step.id if legacy else step.order_index source_id = conn.output_step_id source_step = source_id if legacy else steps_to_order_index[source_id] - step_dict['input_steps'][conn.input_name] = {'source_step': source_step, - 'step_output': conn.output_name} + step_dict["input_steps"][conn.input_name] = { + "source_step": source_step, + "step_output": conn.output_name, + } steps[step_id] = step_dict - item['steps'] = steps + item["steps"] = steps return item def __walk_step_dicts(self, data): - """ Walk over the supplied step dictionaries and return them in a way + """Walk over the supplied step dictionaries and return them in a way designed to preserve step order when possible. """ - supplied_steps = data['steps'] + supplied_steps = data["steps"] # Try to iterate through imported workflow in such a way as to # preserve step order. step_indices = list(supplied_steps.keys()) @@ -1301,8 +1357,8 @@ class WorkflowContentsManager(UsesAnnotations): raise exceptions.DuplicatedIdentifierException(f"Duplicated step label '{label}' in request.") discovered_labels.add(label) - if 'workflow_outputs' in step_dict: - outputs = step_dict['workflow_outputs'] + if "workflow_outputs" in step_dict: + outputs = step_dict["workflow_outputs"] # outputs may be list of name (deprecated legacy behavior) # or dictionary of names to {uuid: , label:
              - '''.format(head_html, exception, extra) + """.format( + head_html, exception, extra + ) def make_error_middleware(app, global_conf, **kw): @@ -492,7 +502,7 @@ def make_error_middleware(app, global_conf, **kw): doc_lines = cast(str, ErrorMiddleware.__doc__).splitlines(True) for i in range(len(doc_lines)): - if doc_lines[i].strip().startswith('Settings'): - make_error_middleware.__doc__ = ''.join(doc_lines[i:]) + if doc_lines[i].strip().startswith("Settings"): + make_error_middleware.__doc__ = "".join(doc_lines[i:]) break del i, doc_lines diff --git a/lib/galaxy/web/framework/middleware/profile.py b/lib/galaxy/web/framework/middleware/profile.py index f3cfd09bc47..6e9d3d597e7 100644 --- a/lib/galaxy/web/framework/middleware/profile.py +++ b/lib/galaxy/web/framework/middleware/profile.py @@ -9,7 +9,6 @@ import threading import markupsafe from paste import response - template = """ -""") +""" + ) assert "