diff --git a/.circleci/config.yml b/.circleci/config.yml index 5535e10190d..f779bac7fcf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -27,7 +27,7 @@ variables: jobs: get_code: docker: - - image: circleci/python:3.5 + - image: circleci/python:3.6 <<: *set_workdir steps: # Replace standard code checkout with shallow clone to speed things up. @@ -83,34 +83,34 @@ jobs: key: v1-repo-{{ .Environment.CIRCLE_SHA1 }} paths: - ~/repo - py35_docstring: + py36_docstring: docker: - - image: circleci/python:3.5 + - image: circleci/python:3.6 <<: *set_workdir steps: - *restore_repo_cache - *install_tox - - run: tox -e py35-lint_docstring_include_list - py35_lint: + - run: tox -e py36-lint_docstring_include_list + py36_lint: docker: - - image: circleci/python:3.5 + - image: circleci/python:3.6 <<: *set_workdir steps: - *restore_repo_cache - *install_tox - - run: tox -e py35-lint - py35_unit: + - run: tox -e py36-lint + py36_unit: docker: - - image: circleci/python:3.5 + - image: circleci/python:3.6 <<: *set_workdir steps: - *restore_repo_cache - *install_tox - *install_ffprobe - - run: tox -e py35-unit - py35_first_startup: + - run: tox -e py36-unit + py36_first_startup: docker: - - image: circleci/python:3.5 + - image: circleci/python:3.6 <<: *set_workdir steps: - *restore_repo_cache @@ -118,10 +118,10 @@ jobs: - run: wget -q https://github.com/jmchilton/galaxy-downloads/raw/master/db_gx_rev_0141.sqlite - run: mv db_gx_rev_0141.sqlite database/universe.sqlite - *install_tox - - run: tox -e py35-first_startup + - run: tox -e py36-first_startup validate_test_tools: docker: - - image: circleci/python:3.5 + - image: circleci/python:3.6 <<: *set_workdir steps: - *restore_repo_cache @@ -131,7 +131,7 @@ jobs: - run: tox -e validate_test_tools test_galaxy_packages: docker: - - image: circleci/python:3.5 + - image: circleci/python:3.6 <<: *set_workdir steps: - *restore_repo_cache @@ -167,13 +167,13 @@ workflows: get_code_and_test: jobs: - get_code - - py35_docstring: + - py36_docstring: <<: *requires_get_code - - py35_lint: + - py36_lint: <<: *requires_get_code - - py35_unit: + - py36_unit: <<: *requires_get_code - - py35_first_startup: + - py36_first_startup: <<: *requires_get_code - test_galaxy_packages: <<: *requires_get_code diff --git a/README.rst b/README.rst index cc66ea4bcdb..f260d040869 100644 --- a/README.rst +++ b/README.rst @@ -24,7 +24,7 @@ Community support is available at `Galaxy Help Galaxy Quickstart ================= -Galaxy requires Python 3.5 or 3.6 . To check your Python version, run: +Galaxy requires Python 3.6 . To check your Python version, run: .. code:: console diff --git a/client/src/assets/images/circle.py b/client/src/assets/images/circle.py index cbd5d06bade..b0d60f02adc 100755 --- a/client/src/assets/images/circle.py +++ b/client/src/assets/images/circle.py @@ -2,7 +2,6 @@ """ usage: %prog width height bg_color hatch_color [color alpha stop_pos] + """ -from __future__ import division import sys from math import pi diff --git a/config/plugins/webhooks/demo/tour_generator/__init__.py b/config/plugins/webhooks/demo/tour_generator/__init__.py index 080706fa881..f61ddaaf323 100644 --- a/config/plugins/webhooks/demo/tour_generator/__init__.py +++ b/config/plugins/webhooks/demo/tour_generator/__init__.py @@ -8,7 +8,7 @@ from galaxy.util import Params log = logging.getLogger(__name__) -class TourGenerator(object): +class TourGenerator: def __init__(self, trans, tool_id, tool_version): self._trans = trans self._tool = self._trans.app.toolbox.get_tool(tool_id, tool_version) @@ -177,14 +177,14 @@ class TourGenerator(object): if name in test_inputs: hid = self._hids[name] dataset = self._test.inputs[name][0] - step['content'] = 'Select dataset: %s: %s' % ( + step['content'] = 'Select dataset: {}: {}'.format( hid, dataset ) else: step['content'] = 'Select a dataset' elif input.type == 'conditional': - param_id = '%s|%s' % (input.name, input.test_param.name) + param_id = f'{input.name}|{input.test_param.name}' step['title'] = input.test_param.label step['element'] = '[tour_id="%s"]' % param_id params = [] @@ -203,7 +203,7 @@ class TourGenerator(object): cases[key] = value.label for case_id, case_title in cases.items(): - tour_id = '%s|%s' % (input.name, case_id) + tour_id = f'{input.name}|{case_id}' if tour_id in self._test.inputs.keys(): if case_id in self._data_inputs.keys(): hid = self._hids[case_id] diff --git a/contrib/galaxy_config_merger.py b/contrib/galaxy_config_merger.py index d813dbb7b80..b5a24c60d5c 100644 --- a/contrib/galaxy_config_merger.py +++ b/contrib/galaxy_config_merger.py @@ -23,14 +23,12 @@ THE ORIGINAL WORK IS WITH YOU. Script for merging specific local Galaxy config galaxy.ini.cri with default Galaxy galaxy.ini.sample ''' -from __future__ import print_function +import configparser import logging import optparse import sys -from six.moves import configparser - def main(): # logging configuration @@ -51,12 +49,12 @@ def main(): config_sample = configparser.RawConfigParser() config_sample.read(options.sample) - config_sample_content = open(options.sample, 'r').read() + config_sample_content = open(options.sample).read() config = configparser.RawConfigParser() config.read(options.config) - logging.info("Merging your own config file %s into the sample one %s." % (options.config, options.sample)) + logging.info(f"Merging your own config file {options.config} into the sample one {options.sample}.") logging.info("---------- DIFFERENCE ANALYSIS BEGIN ----------") for section in config.sections(): if not config_sample.has_section(section): @@ -65,13 +63,13 @@ def main(): for (name, value) in config.items(section): if not config_sample.has_option(section, name): if not "#%s" % name in config_sample_content: - logging.warning("-MISSING- section [%s] option '%s' not found in sample file. It will be ignored." % (section, name)) + logging.warning(f"-MISSING- section [{section}] option '{name}' not found in sample file. It will be ignored.") else: - logging.info("-notset- section [%s] option '%s' not set in sample file. It will be added." % (section, name)) + logging.info(f"-notset- section [{section}] option '{name}' not set in sample file. It will be added.") config_sample.set(section, name, value) else: if not config_sample.get(section, name) == value: - logging.info("- diff - section [%s] option '%s' has different value ('%s':'%s'). It will be modified." % (section, name, config_sample.get(section, name), value)) + logging.info("- diff - section [{}] option '{}' has different value ('{}':'{}'). It will be modified.".format(section, name, config_sample.get(section, name), value)) config_sample.set(section, name, value) logging.info("---------- DIFFERENCE ANALYSIS END ----------") diff --git a/contrib/nagios/check_galaxy.py b/contrib/nagios/check_galaxy.py index d61ff42d84f..6a5a9f6a138 100755 --- a/contrib/nagios/check_galaxy.py +++ b/contrib/nagios/check_galaxy.py @@ -3,7 +3,6 @@ check_galaxy can be run by hand, although it is meant to run from cron via the check_galaxy.sh script in Galaxy's cron/ directory. """ -from __future__ import print_function import formatter import getopt @@ -14,13 +13,12 @@ import socket import sys import time import warnings -from user import home - -from six.moves.urllib.request import ( +from urllib.request import ( build_opener, HTTPCookieProcessor, - Request + Request, ) +from user import home with warnings.catch_warnings(): warnings.simplefilter('ignore') @@ -52,7 +50,7 @@ handler = args[3] warntime = 240 new_history = False -for o, a in opts: +for o, _ in opts: if o == "-n": if debug: print("Specified -n, will create a new history") @@ -73,7 +71,7 @@ tc.agent("Mozilla/5.0 (compatible; check_galaxy/0.2)") tc.config('use_tidy', 0) -class Browser(object): +class Browser: def __init__(self): self.server = server self.handler = handler @@ -93,7 +91,7 @@ class Browser(object): self.opener = build_opener(HTTPCookieProcessor(tc.get_browser().cj)) def get(self, path): - tc.go("%s%s" % (self.server, path)) + tc.go(f"{self.server}{path}") tc.code(200) def req(self, path, data=None, method=None): @@ -105,7 +103,7 @@ class Browser(object): if method: req.get_method = lambda: method res = self.opener.open(req) - print('==> at %s (%s)' % (url, method or 'GET')) + print('==> at {} ({})'.format(url, method or 'GET')) assert res.getcode() == 200, url return res @@ -226,7 +224,7 @@ class Browser(object): if self.hda_state != "ok": self.get("/datasets/%s/stderr" % self.hda_id) print(tc.browser.get_html()) - raise Exception("HDA %s NOT OK: %s" % (self.hda_id, self.hda_state)) + raise Exception(f"HDA {self.hda_id} NOT OK: {self.hda_state}") def check_hda_content(self): self.get("/datasets/%s/display?to_ext=txt" % self.hda_id) @@ -240,7 +238,7 @@ class Browser(object): def delete_datasets(self): for hda in self.undeleted_hdas: - path = '/api/histories/%s/contents/%s' % (self.history_id, hda['id']) + path = '/api/histories/{}/contents/{}'.format(self.history_id, hda['id']) self.req(path, method='DELETE') hdas = [hda['id'] for hda in self.undeleted_hdas] if hdas: diff --git a/cron/add_manual_builds.py b/cron/add_manual_builds.py index 2d1bd58b2e0..8a4197eef5b 100644 --- a/cron/add_manual_builds.py +++ b/cron/add_manual_builds.py @@ -6,7 +6,6 @@ Adds Manually created builds and chrom info to Galaxy's info tables Usage: python add_manual_builds.py input_file builds.txt chrom_length_dir """ -from __future__ import print_function import os import sys diff --git a/cron/build_chrom_db.py b/cron/build_chrom_db.py index b3c7094238b..7289dc72202 100644 --- a/cron/build_chrom_db.py +++ b/cron/build_chrom_db.py @@ -11,14 +11,13 @@ All chromInfo is placed in a path with the convention Usage: python build_chrom_db.py dbpath/ [builds_file] """ -from __future__ import print_function import fileinput import os import sys +from urllib.parse import urlencode import requests -from six.moves.urllib.parse import urlencode import parse_builds # noqa: I100,I202 @@ -78,5 +77,5 @@ if __name__ == "__main__": for chrominfo in getchrominfo("http://genome-test.gi.ucsc.edu/cgi-bin/hgTables?", build): print("\t".join(chrominfo), file=outfile) except Exception as e: - print("Failed to retrieve %s: %s" % (build, e)) + print(f"Failed to retrieve {build}: {e}") os.remove(outfile_name) diff --git a/cron/parse_builds.py b/cron/parse_builds.py index 597c36f8066..45361c3aeeb 100644 --- a/cron/parse_builds.py +++ b/cron/parse_builds.py @@ -4,7 +4,6 @@ Connects to the URL specified and outputs builds available at that DSN in tabular format. UCSC Main gateway is used as default. build description """ -from __future__ import print_function import sys import xml.etree.ElementTree as ElementTree diff --git a/cron/parse_builds_3_sites.py b/cron/parse_builds_3_sites.py index b3cd3bd5d64..c1eb4d0a147 100644 --- a/cron/parse_builds_3_sites.py +++ b/cron/parse_builds_3_sites.py @@ -2,7 +2,6 @@ """ Connects to sites and determines which builds are available at each. """ -from __future__ import print_function import xml.etree.ElementTree as ElementTree diff --git a/doc/parse_gx_xsd.py b/doc/parse_gx_xsd.py index b033608cc5c..2b58b650e83 100644 --- a/doc/parse_gx_xsd.py +++ b/doc/parse_gx_xsd.py @@ -1,17 +1,15 @@ -# coding: utf-8 # TODO: Add examples, tables and best practice links to command # TODO: Examples of truevalue, falsevalue # TODO: Test param extra_file # Things dropped from schema_template.md (still documented inside schema). # - request_parameter_translation -from __future__ import print_function import sys +from io import StringIO from lxml import etree -from six import StringIO -with open(sys.argv[2], "r") as f: +with open(sys.argv[2]) as f: xmlschema_doc = etree.parse(f) markdown_buffer = StringIO() @@ -22,7 +20,7 @@ def main(): toc_list = [] content_list = [] found_tag = False - with open(sys.argv[1], "r") as markdown_template: + with open(sys.argv[1]) as markdown_template: for line in markdown_template: if line.startswith("$tag:"): found_tag = True @@ -41,7 +39,7 @@ def main(): print(el, end='') -class Tag(object): +class Tag: def __init__(self, line): assert line.startswith("$tag:") @@ -71,7 +69,7 @@ class Tag(object): return " > ".join("``%s``" % p for p in self.title.split("|")) def build_toc_entry(self): - return "* [%s](%s)" % (self._pretty_title, self._anchor) + return f"* [{self._pretty_title}]({self._anchor})" def build_help(self): tag = xmlschema_doc.find(self.xpath) @@ -113,7 +111,7 @@ def _build_tag(tag, hide_attributes): doc = _doc_or_none(_type_el(element)) assert doc is not None, "Documentation for %s is empty" % element.attrib["name"] doc = doc.strip() - assertions_buffer.write("``%s`` | %s\n" % (element.attrib["name"], doc)) + assertions_buffer.write("``{}`` | {}\n".format(element.attrib["name"], doc)) text = text.replace(line, assertions_buffer.getvalue()) tag_help.write(text) best_practices = _get_bp_link(annotation_el) @@ -165,7 +163,7 @@ def _build_attributes_table(tag, attributes, hide_attributes=False, attribute_na if best_practices: details += """ Find the Intergalactic Utilities Commision suggested best practices for this element [here](%s).""" % best_practices - attribute_table.write("``%s`` | %s | %s\n" % (name, details, use)) + attribute_table.write(f"``{name}`` | {details} | {use}\n") return attribute_table.getvalue() diff --git a/doc/source/conf.py b/doc/source/conf.py index 2c57af0c459..7f4c6a4baa3 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Galaxy documentation build configuration file, created by # sphinx-quickstart on Tue Mar 6 10:44:44 2012. @@ -80,8 +79,8 @@ source_suffix = ['.rst', '.md'] master_doc = 'index' # General information about the project. -project = u'Galaxy Project' -copyright = str(datetime.datetime.now().year) + u', Galaxy Committers' +project = 'Galaxy Project' +copyright = str(datetime.datetime.now().year) + ', Galaxy Committers' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -233,8 +232,8 @@ latex_elements = { # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'Galaxy.tex', u'Galaxy Code Documentation', - u'Galaxy Team', 'manual'), + ('index', 'Galaxy.tex', 'Galaxy Code Documentation', + 'Galaxy Team', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -263,8 +262,8 @@ latex_documents = [ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'galaxy', u'Galaxy Documentation', - [u'Galaxy Team'], 1) + ('index', 'galaxy', 'Galaxy Documentation', + ['Galaxy Team'], 1) ] # If true, show URL addresses after external links. @@ -277,8 +276,8 @@ man_pages = [ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'Galaxy', u'Galaxy Documentation', - u'Galaxy Team', 'Galaxy', 'Data intensive biology for everyone.', + ('index', 'Galaxy', 'Galaxy Documentation', + 'Galaxy Team', 'Galaxy', 'Data intensive biology for everyone.', 'Miscellaneous'), ] @@ -293,7 +292,7 @@ texinfo_documents = [ # -- ReadTheDocs.org Settings ------------------------------------------------ -class Mock(object): +class Mock: def __init__(self, *args, **kwargs): pass diff --git a/doc/source/conf.versioning.py b/doc/source/conf.versioning.py index 8b112f7afca..d7e108125de 100644 --- a/doc/source/conf.versioning.py +++ b/doc/source/conf.versioning.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Galaxy sphinxcontrib-simpleversioning documentation build configuration file # diff --git a/lib/galaxy/actions/admin.py b/lib/galaxy/actions/admin.py index 3afc8b569f2..6072d1aa886 100644 --- a/lib/galaxy/actions/admin.py +++ b/lib/galaxy/actions/admin.py @@ -76,7 +76,7 @@ class AdminActions: quota.description = params.description self.sa_session.add(quota) self.sa_session.flush() - message = "Quota '{}' has been renamed to '{}'.".format(old_name, params.name) + message = f"Quota '{old_name}' has been renamed to '{params.name}'." return message def _manage_users_and_groups_for_quota(self, quota, params, decode_id=None): @@ -122,7 +122,7 @@ class AdminActions: else: if params.default != 'no': self.app.quota_agent.set_default_quota(params.default, quota) - message = "Quota '{}' is now the default for {} users.".format(quota.name, params.default) + message = f"Quota '{quota.name}' is now the default for {params.default} users." else: if quota.default: message = "Quota '{}' is no longer the default for {} users.".format(quota.name, quota.default[0].type) diff --git a/lib/galaxy/auth/providers/pam_auth.py b/lib/galaxy/auth/providers/pam_auth.py index 334b7726be9..4037d6d91f0 100644 --- a/lib/galaxy/auth/providers/pam_auth.py +++ b/lib/galaxy/auth/providers/pam_auth.py @@ -109,23 +109,23 @@ class PAM(AuthProvider): pam_service = options.get('pam-service', 'galaxy') use_helper = string_as_bool(options.get('use-external-helper', False)) - log.debug("PAM auth: will use external helper: {}".format(use_helper)) + log.debug(f"PAM auth: will use external helper: {use_helper}") authenticated = False if use_helper: authentication_helper = options.get('authentication-helper-script', '/bin/false').strip() - log.debug("PAM auth: external helper script: {}".format(authentication_helper)) + log.debug(f"PAM auth: external helper script: {authentication_helper}") if not authentication_helper.startswith('/'): # don't accept relative path authenticated = False else: - auth_cmd = shlex.split('/usr/bin/sudo -n {}'.format(authentication_helper)) - log.debug("PAM auth: external helper cmd: {}".format(auth_cmd)) - message = '{}\n{}\n{}\n'.format(pam_service, pam_username, password) + auth_cmd = shlex.split(f'/usr/bin/sudo -n {authentication_helper}') + log.debug(f"PAM auth: external helper cmd: {auth_cmd}") + message = f'{pam_service}\n{pam_username}\n{password}\n' try: output = commands.execute(auth_cmd, input=message) except commands.CommandLineException as e: if e.stderr != '': - log.debug("PAM auth: external authentication script had errors: status {} error {}".format(e.returncode, e.stderr)) + log.debug(f"PAM auth: external authentication script had errors: status {e.returncode} error {e.stderr}") output = e.stdout if output.strip() == 'True': authenticated = True diff --git a/lib/galaxy/auth/util.py b/lib/galaxy/auth/util.py index b893d23d835..09ad3e8d53d 100644 --- a/lib/galaxy/auth/util.py +++ b/lib/galaxy/auth/util.py @@ -86,7 +86,7 @@ def parse_auth_results(trans, auth_results, options): i += 1 else: raise Conflict("Cannot make unique username") - log.debug("Email: {}, auto-register with username: {}".format(auto_email, auto_username)) + log.debug(f"Email: {auto_email}, auto-register with username: {auto_username}") auth_return["auto_reg"] = string_as_bool(options.get('auto-register', False)) auth_return["email"] = auto_email auth_return["username"] = auto_username diff --git a/lib/galaxy/authnz/custos_authnz.py b/lib/galaxy/authnz/custos_authnz.py index 3f9ed0bcc0f..f01d65eba14 100644 --- a/lib/galaxy/authnz/custos_authnz.py +++ b/lib/galaxy/authnz/custos_authnz.py @@ -3,12 +3,12 @@ import hashlib import logging import os from datetime import datetime, timedelta +from urllib.parse import quote import jwt import requests from oauthlib.common import generate_nonce from requests_oauthlib import OAuth2Session -from six.moves.urllib.parse import quote from galaxy import exceptions from galaxy import util @@ -248,16 +248,16 @@ class CustosAuthnz(IdentityProvider): base_url = self.config["url"] # Remove potential trailing slash to avoid "//realms" base_url = base_url if base_url[-1] != "/" else base_url[:-1] - return "{}/.well-known/openid-configuration".format(base_url) + return f"{base_url}/.well-known/openid-configuration" else: - raise Exception("Unknown Custos provider name: {}".format(provider)) + raise Exception(f"Unknown Custos provider name: {provider}") def _fetch_well_known_oidc_config(self, well_known_uri): try: return requests.get(well_known_uri, verify=self._get_verify_param()).json() except Exception: - log.error("Failed to load well-known OIDC config URI: {}".format(well_known_uri)) + log.error(f"Failed to load well-known OIDC config URI: {well_known_uri}") raise def _load_well_known_oidc_config(self, well_known_oidc_config): diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index 8cc8104eb7c..b9bdcf78ab8 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -1,3 +1,4 @@ +import builtins import copy import json import logging @@ -10,7 +11,6 @@ from cloudauthz import CloudAuthz from cloudauthz.exceptions import ( CloudAuthzBaseException ) -from six.moves import builtins from galaxy import exceptions from galaxy import model @@ -89,7 +89,7 @@ class AuthnzManager: except ImportError: raise except etree.ParseError as e: - raise etree.ParseError("Invalid configuration at `{}`: {} -- unable to continue.".format(config_file, e)) + raise etree.ParseError(f"Invalid configuration at `{config_file}`: {e} -- unable to continue.") def _get_idp_icon(self, idp): return self.oidc_backends_config[idp].get('icon') or DEFAULT_OIDC_IDP_ICONS.get(idp) @@ -109,7 +109,7 @@ class AuthnzManager: "skipping the node.".format(child.tag)) continue if 'name' not in child.attrib: - log.error("Could not find a node attribute 'name'; skipping the node '{}'.".format(child.tag)) + log.error(f"Could not find a node attribute 'name'; skipping the node '{child.tag}'.") continue idp = child.get('name').lower() if idp in BACKENDS_NAME: @@ -127,7 +127,7 @@ class AuthnzManager: except ImportError: raise except etree.ParseError as e: - raise etree.ParseError("Invalid configuration at `{}`: {} -- unable to continue.".format(config_file, e)) + raise etree.ParseError(f"Invalid configuration at `{config_file}`: {e} -- unable to continue.") def _parse_idp_config(self, config_xml): rtv = { @@ -188,10 +188,10 @@ class AuthnzManager: else: return True, "", identity_provider_class(unified_provider_name, self.oidc_config, self.oidc_backends_config[unified_provider_name]) except Exception as e: - log.exception('An error occurred when loading {}'.format(identity_provider_class.__name__)) + log.exception(f'An error occurred when loading {identity_provider_class.__name__}') return False, unicodify(e), None else: - msg = 'The requested identity provider, `{}`, is not a recognized/expected provider.'.format(provider) + msg = f'The requested identity provider, `{provider}`, is not a recognized/expected provider.' log.debug(msg) return False, msg, None @@ -281,13 +281,13 @@ class AuthnzManager: return False, message, None elif provider in KEYCLOAK_BACKENDS: if (self.allowed_idps and (idphint not in self.allowed_idps)): - msg = 'An error occurred when authenticating a user. Invalid EntityID: `{}`'.format(idphint) + msg = f'An error occurred when authenticating a user. Invalid EntityID: `{idphint}`' log.exception(msg) return False, msg, None - return True, "Redirecting to the `{}` identity provider for authentication".format(provider), backend.authenticate(trans, idphint) - return True, "Redirecting to the `{}` identity provider for authentication".format(provider), backend.authenticate(trans) + return True, f"Redirecting to the `{provider}` identity provider for authentication", backend.authenticate(trans, idphint) + return True, f"Redirecting to the `{provider}` identity provider for authentication", backend.authenticate(trans) except Exception: - msg = 'An error occurred when authenticating a user on `{}` identity provider'.format(provider) + msg = f'An error occurred when authenticating a user on `{provider}` identity provider' log.exception(msg) return False, msg, None @@ -322,7 +322,7 @@ class AuthnzManager: # check if logout is enabled for this idp and return false if not unified_provider_name = self._unify_provider_name(provider) if self.oidc_backends_config[unified_provider_name]['enable_idp_logout'] is False: - return False, "IDP logout is not enabled for {}".format(provider), None + return False, f"IDP logout is not enabled for {provider}", None success, message, backend = self._get_authnz_backend(provider) if success is False: diff --git a/lib/galaxy/authnz/psa_authnz.py b/lib/galaxy/authnz/psa_authnz.py index 1c6e60ff30b..c595b9e094a 100644 --- a/lib/galaxy/authnz/psa_authnz.py +++ b/lib/galaxy/authnz/psa_authnz.py @@ -316,11 +316,11 @@ def contains_required_data(response=None, is_new=False, **kwargs): # sent back from the identity provider. PSA internally handles such # scenarios; however, this case is implemented to prevent uncaught # server-side errors. - raise MalformedContents(err_msg="`response` not found. {}".format(hint_msg)) + raise MalformedContents(err_msg=f"`response` not found. {hint_msg}") if not response.get("id_token"): # This can happen if a non-OIDC compliant backend is used; # e.g., an OAuth2.0-based backend that only generates access token. - raise MalformedContents(err_msg="Missing identity token. {}".format(hint_msg)) + raise MalformedContents(err_msg=f"Missing identity token. {hint_msg}") if is_new and not response.get("refresh_token"): # An identity provider (e.g., Google) sends a refresh token the first # time user consents Galaxy's access (i.e., the first time user logs in @@ -331,7 +331,7 @@ def contains_required_data(response=None, is_new=False, **kwargs): # user has provided consent. This can also happen under dev efforts. # The solution is to revoke the consent by visiting the identity provider's # website, and then retry the login process. - raise MalformedContents(err_msg="Missing refresh token. {}".format(hint_msg)) + raise MalformedContents(err_msg=f"Missing refresh token. {hint_msg}") def verify(strategy=None, response=None, details=None, **kwargs): @@ -344,7 +344,7 @@ def verify(strategy=None, response=None, details=None, **kwargs): if provider.lower() == "gcp": result = requests.post( - "https://iam.googleapis.com/v1/projects/-/serviceAccounts/{}:getIamPolicy".format(endpoint), + f"https://iam.googleapis.com/v1/projects/-/serviceAccounts/{endpoint}:getIamPolicy", headers={ 'Authorization': 'Bearer {}'.format(response.get("access_token")), 'Accept': 'application/json'}) @@ -364,7 +364,7 @@ def verify(strategy=None, response=None, details=None, **kwargs): # sensitive information that should not be exposed to users. raise Exception(res["error"]["message"]) else: - raise Exception("`{}` is an unsupported secondary authorization provider, contact admin.".format(provider)) + raise Exception(f"`{provider}` is an unsupported secondary authorization provider, contact admin.") def allowed_to_disconnect(name=None, user=None, user_storage=None, strategy=None, diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index 61e9e1eb704..fc33292e699 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -4,6 +4,7 @@ Universe configuration builder. # absolute_import needed for tool_shed package. import collections +import configparser import errno import ipaddress import logging @@ -22,7 +23,6 @@ from datetime import timedelta import yaml from beaker.cache import CacheManager from beaker.util import parse_cache_config_options -from six.moves import configparser from galaxy.config.schema import AppSchema from galaxy.containers import parse_containers_config diff --git a/lib/galaxy/config/config_manage.py b/lib/galaxy/config/config_manage.py index b7fcf21fdc8..b8e6dfd87ba 100644 --- a/lib/galaxy/config/config_manage.py +++ b/lib/galaxy/config/config_manage.py @@ -8,12 +8,12 @@ from collections import ( namedtuple, OrderedDict ) +from io import StringIO from textwrap import TextWrapper import requests import yaml from boltons.iterutils import remap -from six import StringIO try: from pykwalify.core import Core @@ -405,7 +405,7 @@ def _to_rst(args, app_desc, heading_level="~"): def _write_option_rst(args, rst, key, heading_level, option_value): title = "``%s``" % key heading = heading_level * len(title) - rst.write("{}\n{}\n{}\n\n".format(heading, title, heading)) + rst.write(f"{heading}\n{title}\n{heading}\n\n") option, value = _parse_option_value(option_value) desc = option["desc"] rst.write(":Description:\n") @@ -649,7 +649,7 @@ def _is_ini(path): def _replace_file(args, f, app_desc, from_path, to_path): _write_to_file(args, f, to_path) backup_path = "%s.backup" % from_path - print("Moving [{}] to [{}]".format(from_path, backup_path)) + print(f"Moving [{from_path}] to [{backup_path}]") if args.dry_run: print("... skipping because --dry-run is enabled.") else: @@ -661,7 +661,7 @@ def _build_sample_yaml(args, app_desc): UWSGI_OPTIONS.update(SHED_ONLY_UWSGI_OPTIONS) schema = app_desc.schema f = StringIO() - for key, value in UWSGI_OPTIONS.items(): + for value in UWSGI_OPTIONS.values(): for field in ["desc", "default"]: if field not in value: continue @@ -693,7 +693,7 @@ def _write_to_file(args, f, path): contents = f if args.dry_run: contents_indented = "\n".join(" |%s" % l for l in contents.splitlines()) - print("Overwriting {} with the following contents:\n{}".format(path, contents_indented)) + print(f"Overwriting {path} with the following contents:\n{contents_indented}") print("... skipping because --dry-run is enabled.") else: print("Overwriting %s" % path) @@ -743,10 +743,10 @@ def _write_option(args, f, key, option_value, as_comment=False, uwsgi_hack=False if uwsgi_hack: if option.get("type", "str") == "bool": value = str(value).lower() - key_val_str = "{}: {}".format(key, value) + key_val_str = f"{key}: {value}" else: key_val_str = yaml.dump({key: value}, width=float("inf")).lstrip("{").rstrip("\n}") - lines = "{}{}{}".format(comment, as_comment_str, key_val_str) + lines = f"{comment}{as_comment_str}{key_val_str}" lines_idented = "\n".join(" %s" % l for l in lines.split("\n")) f.write("%s\n\n" % lines_idented) @@ -774,7 +774,7 @@ def _server_paste_to_uwsgi(app_desc, server_config, applied_filters): if server_config.get("use", "egg:Paste#http") != "egg:Paste#http": raise Exception("Unhandled paste server 'use' value [%s], file must be manually migrate.") - uwsgi_dict["http"] = "{}:{}".format(host, port) + uwsgi_dict["http"] = f"{host}:{port}" # default changing from 10 to 8 uwsgi_dict["threads"] = int(server_config.get("threadpool_workers", 8)) # required for static... @@ -791,7 +791,7 @@ def _server_paste_to_uwsgi(app_desc, server_config, applied_filters): uwsgi_dict["http-auto-gzip"] = True if prefix: - uwsgi_dict["mount"] = "{}={}".format(prefix, app_desc.uwsgi_module) + uwsgi_dict["mount"] = f"{prefix}={app_desc.uwsgi_module}" uwsgi_dict["manage-script-name"] = True else: uwsgi_dict["module"] = app_desc.uwsgi_module diff --git a/lib/galaxy/config/schema.py b/lib/galaxy/config/schema.py index b1d461900b1..c357f06a350 100644 --- a/lib/galaxy/config/schema.py +++ b/lib/galaxy/config/schema.py @@ -84,7 +84,7 @@ class AppSchema(Schema): def check_type_is_str_or_any(option, key): if option.get('type') not in ('str', 'any'): - message = "Invalid schema: property '{}' should have type 'str'".format(key) + message = f"Invalid schema: property '{key}' should have type 'str'" raise_error(message) def check_is_dag(): diff --git a/lib/galaxy/containers/__init__.py b/lib/galaxy/containers/__init__.py index 26e8100df5f..b6c775ab58d 100644 --- a/lib/galaxy/containers/__init__.py +++ b/lib/galaxy/containers/__init__.py @@ -17,7 +17,6 @@ from abc import ( from collections import namedtuple import yaml -from six.moves import shlex_quote from galaxy.exceptions import ContainerCLIError from galaxy.util.submodules import import_submodules @@ -234,14 +233,14 @@ class ContainerInterface(metaclass=ABCMeta): def _stringify_kwopt_string(self, flag, val): """ """ - return '{flag} {value}'.format(flag=flag, value=shlex_quote(str(val))) + return '{flag} {value}'.format(flag=flag, value=shlex.quote(str(val))) def _stringify_kwopt_list(self, flag, val): """ """ if isinstance(val, str): return self._stringify_kwopt_string(flag, val) - return ' '.join('{flag} {value}'.format(flag=flag, value=shlex_quote(str(v))) for v in val) + return ' '.join('{flag} {value}'.format(flag=flag, value=shlex.quote(str(v))) for v in val) def _stringify_kwopt_list_of_kvpairs(self, flag, val): """ @@ -253,7 +252,7 @@ class ContainerInterface(metaclass=ABCMeta): else: # {'foo': 'bar', 'baz': 'quux'} for k, v in dict(val).items(): - l.append('{k}={v}'.format(k=k, v=v)) + l.append(f'{k}={v}') return self._stringify_kwopt_list(flag, l) def _stringify_kwopt_list_of_kovtrips(self, flag, val): @@ -263,7 +262,7 @@ class ContainerInterface(metaclass=ABCMeta): return self._stringify_kwopt_string(flag, val) l = [] for k, o, v in val: - l.append('{k}{o}{v}'.format(k=k, o=o, v=v)) + l.append(f'{k}{o}{v}') return self._stringify_kwopt_list(flag, l) def _run_command(self, command, verbose=False): @@ -275,7 +274,7 @@ class ContainerInterface(metaclass=ABCMeta): if p.returncode == 0: return stdout.strip() else: - msg = "Command '{}' returned non-zero exit status {}".format(command, p.returncode) + msg = f"Command '{command}' returned non-zero exit status {p.returncode}" log.error(msg + ': ' + stderr.strip()) raise ContainerCLIError( msg, @@ -324,7 +323,7 @@ class ContainerInterfaceConfig(dict): try: return self[name] except KeyError: - raise AttributeError("'{}' object has no attribute '{}'".format(self.__class__.__name__, name)) + raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{name}'") def get(self, name, default=None): try: diff --git a/lib/galaxy/containers/docker.py b/lib/galaxy/containers/docker.py index 57d80cee289..93ef6ec1498 100644 --- a/lib/galaxy/containers/docker.py +++ b/lib/galaxy/containers/docker.py @@ -4,6 +4,7 @@ Interface to Docker import logging import os +import shlex from functools import partial from itertools import cycle, repeat from time import sleep @@ -18,7 +19,6 @@ try: except ImportError: ConnectionError = None ReadTimeout = None -from six.moves import shlex_quote from galaxy.containers import ContainerInterface from galaxy.containers.docker_decorators import ( @@ -130,7 +130,7 @@ class DockerCLIInterface(DockerInterface): global_kwopts = [] if self._conf.host: global_kwopts.append('--host') - global_kwopts.append(shlex_quote(self._conf.host)) + global_kwopts.append(shlex.quote(self._conf.host)) if self._conf.force_tlsverify: global_kwopts.append('--tlsverify') self._docker_command = self._conf['command_template'].format( @@ -142,9 +142,9 @@ class DockerCLIInterface(DockerInterface): def _filter_by_id_or_name(self, id, name): if id: - return '--filter id={}'.format(id) + return f'--filter id={id}' elif name: - return '--filter name={}'.format(name) + return f'--filter name={name}' return None def _stringify_kwopt_docker_volumes(self, flag, val): @@ -159,7 +159,7 @@ class DockerCLIInterface(DockerInterface): for hostvol, guestopts in val.items(): if isinstance(guestopts, str): # {'/host/vol': '/container/vol'} - l.append('{}:{}'.format(hostvol, guestopts)) + l.append(f'{hostvol}:{guestopts}') else: # {'/host/vol': {'bind': '/container/vol'}} # {'/host/vol': {'bind': '/container/vol', 'mode': 'rw'}} @@ -198,7 +198,7 @@ class DockerCLIInterface(DockerInterface): return self._run_docker(subcommand='inspect', args=container_id)[0] except (IndexError, ContainerCLIError) as exc: msg = "Invalid container id: %s" % container_id - if exc.stdout == '[]' and exc.stderr == 'Error: no such object: {container_id}'.format(container_id=container_id): + if exc.stdout == '[]' and exc.stderr == f'Error: no such object: {container_id}': log.warning(msg) return [] else: @@ -210,7 +210,7 @@ class DockerCLIInterface(DockerInterface): return self._run_docker(subcommand='image inspect', args=image)[0] except (IndexError, ContainerCLIError) as exc: msg = "%s not pulled, cannot get digest" % image - if exc.stdout == '[]' and exc.stderr == 'Error: no such image: {image}'.format(image=image): + if exc.stdout == '[]' and exc.stderr == f'Error: no such image: {image}': log.warning(msg, image) return [] else: diff --git a/lib/galaxy/containers/docker_model.py b/lib/galaxy/containers/docker_model.py index d3b2df7a751..65c9cabeca1 100644 --- a/lib/galaxy/containers/docker_model.py +++ b/lib/galaxy/containers/docker_model.py @@ -3,13 +3,13 @@ Model objects for docker objects """ import logging +import shlex try: import docker except ImportError: from galaxy.util.bunch import Bunch docker = Bunch(errors=Bunch(NotFound=None)) -from six.moves import shlex_quote from galaxy.containers import ( Container, @@ -106,7 +106,7 @@ class DockerVolume(ContainerVolume): def __str__(self): volume_str = ":".join(filter(lambda x: x is not None, (self.host_path, self.path, self.mode))) if "$" not in volume_str: - volume_for_cmd_line = shlex_quote(volume_str) + volume_for_cmd_line = shlex.quote(volume_str) else: # e.g. $_GALAXY_JOB_TMP_DIR:$_GALAXY_JOB_TMP_DIR:rw so don't single quote. volume_for_cmd_line = '"%s"' % volume_str @@ -393,10 +393,10 @@ class DockerServiceConstraint: return hash((self._name, self._op, self._value)) def __repr__(self): - return '{}({}{}{})'.format(self.__class__.__name__, self._name, self._op, self._value) + return f'{self.__class__.__name__}({self._name}{self._op}{self._value})' def __str__(self): - return '{}{}{}'.format(self._name, self._op, self._value) + return f'{self._name}{self._op}{self._value}' @staticmethod def split_constraint_string(constraint_str): @@ -508,7 +508,7 @@ class DockerNode: @property def state(self): - return ('{}-{}'.format(self._status, self._availability)).lower() + return (f'{self._status}-{self._availability}').lower() @property def cpus(self): @@ -597,10 +597,10 @@ class DockerNodeLabel: return hash((self._name, self._value)) def __repr__(self): - return '{}({}: {})'.format(self.__class__.__name__, self._name, self._value) + return f'{self.__class__.__name__}({self._name}: {self._value})' def __str__(self): - return '{}: {}'.format(self._name, self._value) + return f'{self._name}: {self._value}' @property def name(self): @@ -612,12 +612,12 @@ class DockerNodeLabel: @property def constraint_string(self): - return 'node.labels.{name}=={value}'.format(name=self.name, value=self.value) + return f'node.labels.{self.name}=={self.value}' @property def constraint(self): return DockerServiceConstraint( - name='node.labels.{name}'.format(name=self.name), + name=f'node.labels.{self.name}', op='==', value=self.value ) @@ -744,7 +744,7 @@ class DockerTask: @property def state(self): - return ('{}-{}'.format(self._desired_state, self._state)).lower() + return (f'{self._desired_state}-{self._state}').lower() @property def current_state(self): diff --git a/lib/galaxy/containers/docker_swarm.py b/lib/galaxy/containers/docker_swarm.py index 95b838a13ef..4edc04bc6f5 100644 --- a/lib/galaxy/containers/docker_swarm.py +++ b/lib/galaxy/containers/docker_swarm.py @@ -331,7 +331,7 @@ class DockerSwarmCLIInterface(DockerSwarmInterface, DockerCLIInterface): @docker_columns def service_ps(self, service_id): - return self._run_docker(subcommand='service ps', args='--no-trunc {}'.format(service_id)) + return self._run_docker(subcommand='service ps', args=f'--no-trunc {service_id}') def service_rm(self, service_ids): service_ids = ' '.join(service_ids) @@ -347,7 +347,7 @@ class DockerSwarmCLIInterface(DockerSwarmInterface, DockerCLIInterface): @docker_columns def node_ps(self, node_id): - return self._run_docker(subcommand='node ps', args='--no-trunc {}'.format(node_id)) + return self._run_docker(subcommand='node ps', args=f'--no-trunc {node_id}') def node_update(self, node_id, **kwopts): return self._run_docker(subcommand='node update', args='{kwopts} {node_id}'.format( diff --git a/lib/galaxy/datatypes/anvio.py b/lib/galaxy/datatypes/anvio.py index 6faf20cf4cc..bfce608b7d4 100644 --- a/lib/galaxy/datatypes/anvio.py +++ b/lib/galaxy/datatypes/anvio.py @@ -37,7 +37,7 @@ class AnvioComposite(Html): missing_text = '' if not os.path.exists(os.path.join(dataset.extra_files_path, composite_name)): missing_text = ' (missing)' - rval.append('
  • {}{}{}
    • '.format(composite_name, composite_name, opt_text, missing_text)) + rval.append(f'
  • {composite_name}{opt_text}{missing_text}
    • ') rval.append("") defined_files = map(lambda x: x[0], defined_files) extra_files = [] @@ -49,7 +49,7 @@ class AnvioComposite(Html): if extra_files: rval.append("

    This composite dataset contains these undefined files:

    ') if not (defined_files or extra_files): rval.append("

    This composite dataset does not contain any files!

    ') return "\n".join(rval) @@ -204,7 +204,7 @@ class Velvet(Html): if composite_file.get('description'): rval.append('
  • {} ({}){}
    • '.format(fn, fn, composite_file.get('description'), opt_text)) else: - rval.append('
  • {}{}
    • '.format(fn, fn, opt_text)) + rval.append(f'
  • {fn}{opt_text}
    • ') rval.append('') with open(dataset.file_name, 'w') as f: f.write("\n".join(rval)) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 1dc84499f82..c0912eade57 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -403,7 +403,7 @@ class BamNative(CompressedArchive): # Galaxy display each tag as separate column because 'tostring()' funcition put tabs in between each tag of tags column. # Below code will remove spaces between each tag. bamline_modified = ('\t').join(bamline.split()[:11] + [(' ').join(bamline.split()[11:])]) - ck_data = "{}\n{}".format(ck_data, bamline_modified) + ck_data = f"{ck_data}\n{bamline_modified}" else: # Nothing to enumerate; we've either offset to the end # of the bamfile, or there is no data. (possible with @@ -489,9 +489,9 @@ class Bam(BamNative): # we start another process and discard stderr. if index_flag == '-b': # IOError: No such file or directory: '-b' if index_flag is set to -b (pysam 0.15.4) - cmd = ['python', '-c', "import pysam; pysam.set_verbosity(0); pysam.index('{}', '{}')".format(file_name, index_name)] + cmd = ['python', '-c', f"import pysam; pysam.set_verbosity(0); pysam.index('{file_name}', '{index_name}')"] else: - cmd = ['python', '-c', "import pysam; pysam.set_verbosity(0); pysam.index('{}', '{}', '{}')".format(index_flag, file_name, index_name)] + cmd = ['python', '-c', f"import pysam; pysam.set_verbosity(0); pysam.index('{index_flag}', '{file_name}', '{index_name}')"] with open(os.devnull, 'w') as devnull: subprocess.check_call(cmd, stderr=devnull, shell=False) needs_sorting = False @@ -1711,7 +1711,7 @@ class CuffDiffSQlite(SQlite): for gene_id, gene_name in result: if gene_name is None: continue - gene = '{}: {}'.format(gene_id, gene_name) + gene = f'{gene_id}: {gene_name}' if gene not in genes: genes.append(gene) samples_query = 'SELECT DISTINCT(sample_name) as sample_name FROM samples ORDER BY sample_name' @@ -1859,7 +1859,7 @@ class BlibSQlite(SQlite): c = conn.cursor() tables_query = "SELECT majorVersion,minorVersion FROM LibInfo" (majorVersion, minorVersion) = c.execute(tables_query).fetchall()[0] - dataset.metadata.blib_version = '{}.{}'.format(majorVersion, minorVersion) + dataset.metadata.blib_version = f'{majorVersion}.{minorVersion}' except Exception as e: log.warning('%s, set_meta Exception: %s', self, e) diff --git a/lib/galaxy/datatypes/blast.py b/lib/galaxy/datatypes/blast.py index 64253673672..8037fe58362 100644 --- a/lib/galaxy/datatypes/blast.py +++ b/lib/galaxy/datatypes/blast.py @@ -147,7 +147,7 @@ class BlastXml(GenericXml): raise ValueError("The header in BLAST XML file %s is too long" % f) if "" not in header: h.close() - raise ValueError("{} is not a BLAST XML file:\n{}\n...".format(f, header)) + raise ValueError(f"{f} is not a BLAST XML file:\n{header}\n...") if f == split_files[0]: out.write(header) old_header = header @@ -223,7 +223,7 @@ class _BlastDb(Data): if not msg: msg = title # Galaxy assumes HTML for the display of composite datatypes, - return smart_str("{}
    {}
    ".format(title, msg)) + return smart_str(f"{title}
    {msg}
    ") def merge(split_files, output_file): """Merge BLAST databases (not implemented for now).""" diff --git a/lib/galaxy/datatypes/converters/fastq_to_fqtoc.py b/lib/galaxy/datatypes/converters/fastq_to_fqtoc.py index c7445131451..e2fb2a447f5 100644 --- a/lib/galaxy/datatypes/converters/fastq_to_fqtoc.py +++ b/lib/galaxy/datatypes/converters/fastq_to_fqtoc.py @@ -34,7 +34,7 @@ def main(): current_line += 1 if 0 == current_line % lines_per_chunk: chunk_end = in_file.tell() - out_file.write('{{"start":"{}","end":"{}","sequences":"{}"}},'.format(chunk_begin, chunk_end, sequences)) + out_file.write(f'{{"start":"{chunk_begin}","end":"{chunk_end}","sequences":"{sequences}"}},') chunk_begin = chunk_end line = in_file.readline() diff --git a/lib/galaxy/datatypes/converters/lped_to_fped_converter.py b/lib/galaxy/datatypes/converters/lped_to_fped_converter.py index 467e48a8892..e3a27b190a0 100644 --- a/lib/galaxy/datatypes/converters/lped_to_fped_converter.py +++ b/lib/galaxy/datatypes/converters/lped_to_fped_converter.py @@ -41,7 +41,7 @@ def rgConv(inpedfilepath, outhtmlname, outfilepath): try: mf = open(inmap) except Exception: - sys.exit('{} cannot open inmap file {} - do you have permission?\n'.format(prog, inmap)) + sys.exit(f'{prog} cannot open inmap file {inmap} - do you have permission?\n') try: rsl = [x.split()[1] for x in mf] except Exception: @@ -94,8 +94,8 @@ def main(): flist = os.listdir(outfilepath) with open(outhtmlname, 'w') as f: f.write(galhtmlprefix % prog) - print('## Rgenetics: http://rgenetics.org Galaxy Tools {} {}'.format(prog, timenow())) # becomes info - f.write('
    ## Rgenetics: http://rgenetics.org Galaxy Tools {} {}\n
      '.format(prog, timenow())) + print(f'## Rgenetics: http://rgenetics.org Galaxy Tools {prog} {timenow()}') # becomes info + f.write(f'
      ## Rgenetics: http://rgenetics.org Galaxy Tools {prog} {timenow()}\n
        ') for i, data in enumerate(flist): f.write('
      1. {}
        2. \n'.format(os.path.split(data)[-1], os.path.split(data)[-1])) f.write("
      ") diff --git a/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py b/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py index 9914d457cb5..e5a491184da 100644 --- a/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py +++ b/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py @@ -99,7 +99,7 @@ def main(): flist = os.listdir(outfilepath) with open(outhtmlname, 'w') as f: f.write(galhtmlprefix % prog) - s = '## Rgenetics: http://rgenetics.org Galaxy Tools {} {}'.format(prog, timenow()) # becomes info + s = f'## Rgenetics: http://rgenetics.org Galaxy Tools {prog} {timenow()}' # becomes info print(s) f.write('
      %s\n
        ' % (s)) for i, data in enumerate(flist): diff --git a/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py b/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py index 571ee3f4c9a..18c393244f3 100644 --- a/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py +++ b/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py @@ -60,8 +60,8 @@ def makeLDreduced(basename, infpath=None, outfpath=None, plinke='plink', forcere inbase = os.path.join(infpath) plinktasks = [] vclbase = [plinke, '--noweb'] - plinktasks += [['--bfile', inbase, '--indep-pairwise {} {} {}'.format(winsize, winmove, r2thresh), '--out %s' % outbase], - ['--bfile', inbase, '--extract {}.prune.in --make-bed --out {}'.format(outbase, outbase)]] + plinktasks += [['--bfile', inbase, f'--indep-pairwise {winsize} {winmove} {r2thresh}', '--out %s' % outbase], + ['--bfile', inbase, f'--extract {outbase}.prune.in --make-bed --out {outbase}']] vclbase = [plinke, '--noweb'] pruneLD(plinktasks=plinktasks, cd=outfpath, vclbase=vclbase) @@ -99,10 +99,10 @@ def main(): flist = os.listdir(outfilepath) with open(outhtmlname, 'w') as f: f.write(galhtmlprefix % prog) - s1 = '## Rgenetics: http://rgenetics.org Galaxy Tools {} {}'.format(prog, timenow()) # becomes info - s2 = 'Input {}, winsize={}, winmove={}, r2thresh={}'.format(base_name, winsize, winmove, r2thresh) - print('{} {}'.format(s1, s2)) - f.write('
        {}\n{}\n
          '.format(s1, s2)) + s1 = f'## Rgenetics: http://rgenetics.org Galaxy Tools {prog} {timenow()}' # becomes info + s2 = f'Input {base_name}, winsize={winsize}, winmove={winmove}, r2thresh={r2thresh}' + print(f'{s1} {s2}') + f.write(f'
          {s1}\n{s2}\n
            ') for i, data in enumerate(flist): f.write('
          1. {}
            2. \n'.format(os.path.split(data)[-1], os.path.split(data)[-1])) f.write("
          ") diff --git a/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py b/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py index 7da52062df7..c16685e3989 100644 --- a/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py +++ b/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py @@ -65,7 +65,7 @@ def main(): flist = os.listdir(outfilepath) with open(outhtmlname, 'w') as f: f.write(galhtmlprefix % prog) - s = '## Rgenetics: http://bitbucket.org/rgalaxy Galaxy Tools {} {}'.format(prog, timenow()) # becomes info + s = f'## Rgenetics: http://bitbucket.org/rgalaxy Galaxy Tools {prog} {timenow()}' # becomes info print(s) f.write('
          %s\n
            ' % (s)) for i, data in enumerate(flist): diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index 1df1b34f16b..8fc19432efe 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -65,7 +65,7 @@ class DatatypeValidation: return DatatypeValidation("unknown", "Dataset validation unimplemented for this datatype.") def __repr__(self): - return "DatatypeValidation[state={},message={}]".format(self.state, self.message) + return f"DatatypeValidation[state={self.state},message={self.message}]" def validate(dataset_instance): @@ -313,7 +313,7 @@ class Data(metaclass=DataMeta): display_name = os.path.splitext(outfname)[0] if not display_name.endswith(ext): - display_name = '{}_{}'.format(display_name, ext) + display_name = f'{display_name}_{ext}' error, msg = self._archive_main_file(archive, display_name, path)[:2] if not error: @@ -339,7 +339,7 @@ class Data(metaclass=DataMeta): outext = 'tgz' if do_action == 'tbz': outext = 'tbz' - trans.response.headers["Content-Disposition"] = 'attachment; filename="{}.{}"'.format(outfname, outext) + trans.response.headers["Content-Disposition"] = f'attachment; filename="{outfname}.{outext}"' archive.wsgi_status = trans.response.wsgi_status() archive.wsgi_headeritems = trans.response.wsgi_headeritems() return archive.stream @@ -438,7 +438,7 @@ class Data(metaclass=DataMeta): self._clean_and_set_mime_type(trans, mime) return self._yield_user_file_content(trans, data, file_path) else: - return webob.exc.HTTPNotFound("Could not find '{}' on the extra files path {}.".format(filename, file_path)) + return webob.exc.HTTPNotFound(f"Could not find '{filename}' on the extra files path {file_path}.") self._clean_and_set_mime_type(trans, data.get_mime()) trans.log_event("Display dataset id: %s" % str(data.id)) @@ -627,7 +627,7 @@ class Data(metaclass=DataMeta): return getattr(self, self.supported_display_apps[type]['file_function'])(dataset, **kwd) except Exception: log.exception('Function %s is referred to in datatype %s for displaying as type %s, but is not accessible', self.supported_display_apps[type]['file_function'], self.__class__.__name__, type) - return "This display type ({}) is not implemented for this datatype ({}).".format(type, dataset.ext) + return f"This display type ({type}) is not implemented for this datatype ({dataset.ext})." def get_display_links(self, dataset, type, app, base_url, target_frame='_blank', **kwd): """ @@ -657,7 +657,7 @@ class Data(metaclass=DataMeta): converter = trans.app.datatypes_registry.get_converter_by_target_type(original_dataset.ext, target_type) if converter is None: - raise Exception("A converter does not exist for {} to {}.".format(original_dataset.ext, target_type)) + raise Exception(f"A converter does not exist for {original_dataset.ext} to {target_type}.") # Generate parameter dictionary params = {} # determine input parameter name and add to params @@ -683,7 +683,7 @@ class Data(metaclass=DataMeta): value.visible = False if return_output: return converted_dataset - return "The file conversion of {} on data {} has been added to the Queue.".format(converter.name, original_dataset.hid) + return f"The file conversion of {converter.name} on data {original_dataset.hid} has been added to the Queue." # We need to clear associated files before we set metadata # so that as soon as metadata starts to be set, e.g. implicitly converted datasets are deleted and no longer available 'while' metadata is being set, not just after @@ -856,7 +856,7 @@ class Text(Data): sample_lines = dataset_read.count('\n') est_lines = int(sample_lines * (float(dataset.get_size()) / float(sample_size))) except UnicodeDecodeError: - log.error('Unable to estimate lines in file {}'.format(dataset.file_name)) + log.error(f'Unable to estimate lines in file {dataset.file_name}') est_lines = None return est_lines @@ -880,7 +880,7 @@ class Text(Data): if line and not line.startswith('#'): data_lines += 1 except UnicodeDecodeError: - log.error('Unable to count lines in file {}'.format(dataset.file_name)) + log.error(f'Unable to count lines in file {dataset.file_name}') data_lines = None return data_lines diff --git a/lib/galaxy/datatypes/dataproviders/column.py b/lib/galaxy/datatypes/dataproviders/column.py index bbd49ae5fff..1306338714d 100644 --- a/lib/galaxy/datatypes/dataproviders/column.py +++ b/lib/galaxy/datatypes/dataproviders/column.py @@ -4,8 +4,7 @@ is further subdivided into multiple data (e.g. columns from a line). """ import logging import re - -from six.moves.urllib.parse import unquote_plus +from urllib.parse import unquote_plus from . import line diff --git a/lib/galaxy/datatypes/dataproviders/decorators.py b/lib/galaxy/datatypes/dataproviders/decorators.py index 15f5a79f77e..76d05776f4f 100644 --- a/lib/galaxy/datatypes/dataproviders/decorators.py +++ b/lib/galaxy/datatypes/dataproviders/decorators.py @@ -17,8 +17,7 @@ DataProvider related decorators. import copy import logging from functools import wraps - -from six.moves.urllib.parse import unquote +from urllib.parse import unquote log = logging.getLogger(__name__) @@ -104,8 +103,8 @@ def dataprovider_factory(name, settings=None): def named_dataprovider_factory(func): setattr(func, _DATAPROVIDER_METHOD_NAME_KEY, name) - setattr(func, 'parse_query_string_settings', parse_query_string_settings) - setattr(func, 'settings', settings) + func.parse_query_string_settings = parse_query_string_settings + func.settings = settings # TODO: I want a way to inherit settings from the previous provider( this_name ) instead of defining over and over @wraps(func) diff --git a/lib/galaxy/datatypes/dataproviders/external.py b/lib/galaxy/datatypes/dataproviders/external.py index 936fc7cfc17..432627fad2a 100644 --- a/lib/galaxy/datatypes/dataproviders/external.py +++ b/lib/galaxy/datatypes/dataproviders/external.py @@ -6,9 +6,11 @@ import gzip import logging import subprocess import tempfile - -from six.moves.urllib.parse import urlencode, urlparse -from six.moves.urllib.request import urlopen +from urllib.parse import ( + urlencode, + urlparse, +) +from urllib.request import urlopen from . import ( base, diff --git a/lib/galaxy/datatypes/display_applications/application.py b/lib/galaxy/datatypes/display_applications/application.py index 0224dc7fa19..f0ee85e1362 100644 --- a/lib/galaxy/datatypes/display_applications/application.py +++ b/lib/galaxy/datatypes/display_applications/application.py @@ -2,8 +2,7 @@ import logging from collections import OrderedDict from copy import deepcopy - -from six.moves.urllib.parse import quote_plus +from urllib.parse import quote_plus from galaxy.util import ( parse_xml, @@ -192,7 +191,7 @@ class DynamicDisplayApplicationBuilder: # now populate links.append(DisplayApplicationLink.from_elem(new_elem, display_application, other_values=dynamic_values)) else: - log.warning('Invalid dynamic display application link specified in {}: "{}"'.format(filename, line)) + log.warning(f'Invalid dynamic display application link specified in {filename}: "{line}"') self.links = links def __iter__(self): diff --git a/lib/galaxy/datatypes/display_applications/parameters.py b/lib/galaxy/datatypes/display_applications/parameters.py index 374a6ca1500..2534a936689 100644 --- a/lib/galaxy/datatypes/display_applications/parameters.py +++ b/lib/galaxy/datatypes/display_applications/parameters.py @@ -1,7 +1,6 @@ # Contains parameters that are used in Display Applications import mimetypes - -from six.moves.urllib.parse import quote_plus +from urllib.parse import quote_plus from galaxy.util import string_as_bool from galaxy.util.bunch import Bunch @@ -83,7 +82,7 @@ class DisplayApplicationDataParameter(DisplayApplicationParameter): data = data.value if self.metadata: rval = getattr(data.metadata, self.metadata, None) - assert rval, 'Unknown metadata name ({}) provided for dataset type ({}).'.format(self.metadata, data.datatype.__class__.name) + assert rval, f'Unknown metadata name ({self.metadata}) provided for dataset type ({data.datatype.__class__.name}).' return Bunch(file_name=rval.file_name, state=data.state, states=data.states, extension='data') elif self.extensions and (self.force_conversion or not isinstance(data.datatype, self.formats)): for ext in self.extensions: diff --git a/lib/galaxy/datatypes/genetics.py b/lib/galaxy/datatypes/genetics.py index 7668217f56f..24a60abb68d 100644 --- a/lib/galaxy/datatypes/genetics.py +++ b/lib/galaxy/datatypes/genetics.py @@ -15,9 +15,9 @@ import logging import os import re import sys +from urllib.parse import quote_plus from markupsafe import escape -from six.moves.urllib.parse import quote_plus from galaxy.datatypes import metadata from galaxy.datatypes.data import ( @@ -107,7 +107,7 @@ class GenomeGraphs(Tabular): display_url = quote_plus(display_url) # was display_url = quote_plus( "%s/display_as?id=%i&display_app=%s" % (base_url, dataset.id, type) ) # redirect_url = quote_plus( "%sdb=%s&position=%s:%s-%s&hgt.customText=%%s" % (site_url, dataset.dbkey, chrom, start, stop) ) - sl = ["{}db={}".format(site_url, dataset.dbkey), ] + sl = [f"{site_url}db={dataset.dbkey}", ] # sl.append("&hgt.customText=%s") sl.append("&hgGenome_dataSetName={}&hgGenome_dataSetDescription={}".format(dataset.name, 'GalaxyGG_data')) sl.append("&hgGenome_formatType=best guess&hgGenome_markerType=best guess") @@ -117,7 +117,7 @@ class GenomeGraphs(Tabular): s = ''.join(sl) s = quote_plus(s) redirect_url = s - link = '{}?redirect_url={}&display_url={}'.format(internal_url, redirect_url, display_url) + link = f'{internal_url}?redirect_url={redirect_url}&display_url={display_url}' ret_val.append((site_name, link)) return ret_val @@ -286,7 +286,7 @@ class Rgenetics(Html): if composite_file.get('description'): rval.append('
          1. {} ({}){}
            2. '.format(fn, fn, composite_file.get('description'), opt_text)) else: - rval.append('
          2. {}{}
            3. '.format(fn, fn, opt_text)) + rval.append(f'
          3. {fn}{opt_text}
            4. ') rval.append('
          ') return "\n".join(rval) @@ -296,11 +296,11 @@ class Rgenetics(Html): """ efp = dataset.extra_files_path flist = os.listdir(efp) - rval = ['Files for Composite Dataset {}

          Composite {} contains:

            '.format(dataset.name, dataset.name)] + rval = [f'Files for Composite Dataset {dataset.name}

            Composite {dataset.name} contains:

              '] for fname in flist: sfname = os.path.split(fname)[-1] f, e = os.path.splitext(fname) - rval.append('
            • {}
              • '.format(sfname, sfname)) + rval.append(f'
            • {sfname}
              • ') rval.append('
            ') with open(dataset.file_name, 'w') as f: f.write("\n".join(rval)) @@ -334,7 +334,7 @@ class Rgenetics(Html): return False if len(flist) == 0: if verbose: - gal_Log.debug('@@@rgenetics set_meta failed - {} efp {} is empty?'.format(dataset.name, efp)) + gal_Log.debug(f'@@@rgenetics set_meta failed - {dataset.name} efp {efp} is empty?') return False self.regenerate_primary_file(dataset) if not dataset.info: @@ -555,7 +555,7 @@ class IdeasPre(Html): rval.append('
              ') for composite_name in self.get_composite_files(dataset=dataset).keys(): fn = composite_name - rval.append('
            • {fn}
              • ') rval.append('
            \n') return "\n".join(rval) @@ -566,7 +566,7 @@ class IdeasPre(Html): rval.append('
              ') for fname in os.listdir(dataset.extra_files_path): fn = os.path.split(fname)[-1] - rval.append('
            • {}
              • '.format(fn, fn)) + rval.append(f'
            • {fn}
              • ') rval.append('
            ') with open(dataset.file_name, 'w') as f: f.write("\n".join(rval)) @@ -752,7 +752,7 @@ class RexpBase(Html): rval = ['Files for Composite Dataset %s

            Comprises the following files:

              ' % (bn)] for fname in flist: sfname = os.path.split(fname)[-1] - rval.append('
            • {}'.format(sfname, sfname)) + rval.append(f'
            • {sfname}') rval.append('
            ') with open(dataset.file_name, 'w') as f: f.write("\n".join(rval)) diff --git a/lib/galaxy/datatypes/gis.py b/lib/galaxy/datatypes/gis.py index d77a308ed1c..af2af2062a3 100644 --- a/lib/galaxy/datatypes/gis.py +++ b/lib/galaxy/datatypes/gis.py @@ -42,7 +42,7 @@ class Shapefile(Binary): if composite_file.get('description'): rval.append('
          • {} ({}){}
            • '.format(fn, fn, composite_file.get('description'), opt_text)) else: - rval.append('
          • {}{}
            • '.format(fn, fn, opt_text)) + rval.append(f'
          • {fn}{opt_text}
            • ') rval.append('
        \n') return "\n".join(rval) diff --git a/lib/galaxy/datatypes/images.py b/lib/galaxy/datatypes/images.py index 8fd4eee22df..1a3ad984643 100644 --- a/lib/galaxy/datatypes/images.py +++ b/lib/galaxy/datatypes/images.py @@ -4,8 +4,7 @@ Image classes import base64 import logging import zipfile - -from six.moves.urllib.parse import quote_plus +from urllib.parse import quote_plus from galaxy.datatypes.text import Html as HtmlFromText from galaxy.util import nice_size @@ -52,7 +51,7 @@ class Image(data.Data): name = hda.name or '' with open(dataset.file_name, "rb") as f: base64_image_data = base64.b64encode(f.read()).decode("utf-8") - return "![{}](data:image/{};base64,{})".format(name, self.file_ext, base64_image_data) + return f"![{name}](data:image/{self.file_ext};base64,{base64_image_data})" class Jpg(Image): @@ -177,14 +176,14 @@ def create_applet_tag_peek(class_name, archive, params): height="30" width="200" align="center" > """.format(class_name, archive) for name, value in params.items(): - text += """""".format(name, value) + text += f"""""" text += """ """.format(class_name, archive) for name, value in params.items(): - text += """""".format(name, value) + text += f"""""" text += """
        You must install and enable Java in your browser in order to access this applet.
         """ @@ -265,7 +264,7 @@ class Laj(data.Text): "alignfile1": "display?id=%s" % dataset.id, "buttonlabel": "Launch LAJ", "title": "LAJ in Galaxy", - "posturl": quote_plus("history_add_to?%s" % "&".join("{}={}".format(key, value) for key, value in {'history_id': dataset.history_id, 'ext': 'lav', 'name': 'LAJ Output', 'info': 'Added by LAJ', 'dbkey': dataset.dbkey, 'copy_access_from': dataset.id}.items())), + "posturl": quote_plus("history_add_to?%s" % "&".join(f"{key}={value}" for key, value in {'history_id': dataset.history_id, 'ext': 'lav', 'name': 'LAJ Output', 'info': 'Added by LAJ', 'dbkey': dataset.dbkey, 'copy_access_from': dataset.id}.items())), "noseq": "true" } class_name = "edu.psu.cse.bio.laj.LajApplet.class" diff --git a/lib/galaxy/datatypes/interval.py b/lib/galaxy/datatypes/interval.py index 8aa4b3b08ee..ef6e3f9a455 100644 --- a/lib/galaxy/datatypes/interval.py +++ b/lib/galaxy/datatypes/interval.py @@ -5,9 +5,9 @@ import logging import math import sys import tempfile +from urllib.parse import quote_plus from bx.intervals.io import GenomicIntervalReader, ParseError -from six.moves.urllib.parse import quote_plus from galaxy import util from galaxy.datatypes import metadata @@ -264,7 +264,7 @@ class Interval(Tabular): (base_url, app.url_for(controller='root'), dataset.id, type)) redirect_url = quote_plus("%sdb=%s&position=%s:%s-%s&hgt.customText=%%s" % (site_url, dataset.dbkey, chrom, start, stop)) - link = '{}?redirect_url={}&display_url={}'.format(internal_url, redirect_url, display_url) + link = f'{internal_url}?redirect_url={redirect_url}&display_url={display_url}' ret_val.append((site_name, link)) return ret_val @@ -604,11 +604,11 @@ class _RemoteCallMixin: the data available, followed by redirecting to the remote site with a link back to the available information. """ - internal_url = "%s" % app.url_for(controller='dataset', dataset_id=dataset.id, action='display_at', filename='{}_{}'.format(type, site_name)) + internal_url = "%s" % app.url_for(controller='dataset', dataset_id=dataset.id, action='display_at', filename=f'{type}_{site_name}') base_url = app.config.get("display_at_callback", base_url) display_url = quote_plus("%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" % (base_url, app.url_for(controller='root'), dataset.id, type)) - link = '{}?redirect_url={}&display_url={}'.format(internal_url, redirect_url, display_url) + link = f'{internal_url}?redirect_url={redirect_url}&display_url={display_url}' return link @@ -794,7 +794,7 @@ class Gff(Tabular, _RemoteCallMixin): if site_name in app.datatypes_registry.get_display_sites('gbrowse'): if seqid.startswith('chr') and len(seqid) > 3: seqid = seqid[3:] - redirect_url = quote_plus("{}/?q={}:{}..{}&eurl=%s".format(site_url, seqid, start, stop)) + redirect_url = quote_plus(f"{site_url}/?q={seqid}:{start}..{stop}&eurl=%s") link = self._get_remote_call_url(redirect_url, site_name, dataset, type, app, base_url) ret_val.append((site_name, link)) return ret_val @@ -1146,7 +1146,7 @@ class Wiggle(Tabular, _RemoteCallMixin): if site_name in app.datatypes_registry.get_display_sites('gbrowse'): if chrom.startswith('chr') and len(chrom) > 3: chrom = chrom[3:] - redirect_url = quote_plus("{}/?q={}:{}..{}&eurl=%s".format(site_url, chrom, start, stop)) + redirect_url = quote_plus(f"{site_url}/?q={chrom}:{start}..{stop}&eurl=%s") link = self._get_remote_call_url(redirect_url, site_name, dataset, type, app, base_url) ret_val.append((site_name, link)) return ret_val @@ -1157,7 +1157,7 @@ class Wiggle(Tabular, _RemoteCallMixin): if chrom is not None: for site_name, site_url in app.datatypes_registry.get_legacy_sites_by_build('ucsc', dataset.dbkey): if site_name in app.datatypes_registry.get_display_sites('ucsc'): - redirect_url = quote_plus("{}db={}&position={}:{}-{}&hgt.customText=%s".format(site_url, dataset.dbkey, chrom, start, stop)) + redirect_url = quote_plus(f"{site_url}db={dataset.dbkey}&position={chrom}:{start}-{stop}&hgt.customText=%s") link = self._get_remote_call_url(redirect_url, site_name, dataset, type, app, base_url) ret_val.append((site_name, link)) return ret_val @@ -1324,8 +1324,8 @@ class CustomTrack(Tabular): if site_name in app.datatypes_registry.get_display_sites('ucsc'): internal_url = "%s" % app.url_for(controller='dataset', dataset_id=dataset.id, action='display_at', filename='ucsc_' + site_name) display_url = quote_plus("%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" % (base_url, app.url_for(controller='root'), dataset.id, type)) - redirect_url = quote_plus("{}db={}&position={}:{}-{}&hgt.customText=%s".format(site_url, dataset.dbkey, chrom, start, stop)) - link = '{}?redirect_url={}&display_url={}'.format(internal_url, redirect_url, display_url) + redirect_url = quote_plus(f"{site_url}db={dataset.dbkey}&position={chrom}:{start}-{stop}&hgt.customText=%s") + link = f'{internal_url}?redirect_url={redirect_url}&display_url={display_url}' ret_val.append((site_name, link)) return ret_val diff --git a/lib/galaxy/datatypes/isa.py b/lib/galaxy/datatypes/isa.py index c1a823baa67..b4dc7b3b337 100644 --- a/lib/galaxy/datatypes/isa.py +++ b/lib/galaxy/datatypes/isa.py @@ -274,7 +274,7 @@ class _Isa(data.Data): """ else: html = '' - html += '

        {} {}

        '.format(investigation.title, investigation.identifier) + html += f'

        {investigation.title} {investigation.identifier}

        ' # Loop on all studies for study in investigation.studies: diff --git a/lib/galaxy/datatypes/microarrays.py b/lib/galaxy/datatypes/microarrays.py index 89410770475..a0649d93ed5 100644 --- a/lib/galaxy/datatypes/microarrays.py +++ b/lib/galaxy/datatypes/microarrays.py @@ -38,9 +38,9 @@ class GenericMicroarrayFile(data.Text): def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: if dataset.metadata.block_count == 1: - dataset.blurb = "{} {}: Format {}, 1 block, {} headers and {} columns".format(dataset.metadata.file_type, dataset.metadata.version_number, dataset.metadata.file_format, dataset.metadata.number_of_optional_header_records, dataset.metadata.number_of_data_columns) + dataset.blurb = f"{dataset.metadata.file_type} {dataset.metadata.version_number}: Format {dataset.metadata.file_format}, 1 block, {dataset.metadata.number_of_optional_header_records} headers and {dataset.metadata.number_of_data_columns} columns" else: - dataset.blurb = "{} {}: Format {}, {} blocks, {} headers and {} columns".format(dataset.metadata.file_type, dataset.metadata.version_number, dataset.metadata.file_format, dataset.metadata.block_count, dataset.metadata.number_of_optional_header_records, dataset.metadata.number_of_data_columns) + dataset.blurb = f"{dataset.metadata.file_type} {dataset.metadata.version_number}: Format {dataset.metadata.file_format}, {dataset.metadata.block_count} blocks, {dataset.metadata.number_of_optional_header_records} headers and {dataset.metadata.number_of_data_columns} columns" dataset.peek = get_file_peek(dataset.file_name) else: dataset.peek = 'file does not exist' diff --git a/lib/galaxy/datatypes/molecules.py b/lib/galaxy/datatypes/molecules.py index 3176d2b9c54..c3823e9c7bf 100644 --- a/lib/galaxy/datatypes/molecules.py +++ b/lib/galaxy/datatypes/molecules.py @@ -514,7 +514,7 @@ class PDB(GenericMolFile): hetatm_numbers = count_special_lines("^HETATM", dataset.file_name) chain_ids = ','.join(dataset.metadata.chain_ids) if len(dataset.metadata.chain_ids) > 0 else 'None' dataset.peek = get_file_peek(dataset.file_name) - dataset.blurb = "{} atoms and {} HET-atoms\nchain_ids: {}".format(atom_numbers, hetatm_numbers, chain_ids) + dataset.blurb = f"{atom_numbers} atoms and {hetatm_numbers} HET-atoms\nchain_ids: {chain_ids}" else: dataset.peek = 'file does not exist' dataset.blurb = 'file purged from disk' @@ -565,7 +565,7 @@ class PDBQT(GenericMolFile): root_numbers = count_special_lines("^ROOT", dataset.file_name) branch_numbers = count_special_lines("^BRANCH", dataset.file_name) dataset.peek = get_file_peek(dataset.file_name) - dataset.blurb = "{} roots and {} branches".format(root_numbers, branch_numbers) + dataset.blurb = f"{root_numbers} roots and {branch_numbers} branches" else: dataset.peek = 'file does not exist' dataset.blurb = 'file purged from disk' diff --git a/lib/galaxy/datatypes/ngsindex.py b/lib/galaxy/datatypes/ngsindex.py index 341ca6ac60f..5c7960ec64b 100644 --- a/lib/galaxy/datatypes/ngsindex.py +++ b/lib/galaxy/datatypes/ngsindex.py @@ -36,7 +36,7 @@ class BowtieIndex(Html): rval = ['Files for Composite Dataset %s

        Comprises the following files:

          ' % (bn)] for fname in flist: sfname = os.path.split(fname)[-1] - rval.append('
        • {}'.format(sfname, sfname)) + rval.append(f'
        • {sfname}') rval.append('
        ') with open(dataset.file_name, 'w') as f: f.write("\n".join(rval)) diff --git a/lib/galaxy/datatypes/proteomics.py b/lib/galaxy/datatypes/proteomics.py index 90a53026fa1..9e920fbf0d6 100644 --- a/lib/galaxy/datatypes/proteomics.py +++ b/lib/galaxy/datatypes/proteomics.py @@ -48,7 +48,7 @@ class Wiff(Binary): if composite_file.get('description'): rval.append('
      1. {} ({}){}
        2. '.format(fn, fn, composite_file.get('description'), opt_text)) else: - rval.append('
      2. {}{}
        3. '.format(fn, fn, opt_text)) + rval.append(f'
      3. {fn}{opt_text}
        4. ') rval.append('
      ') return "\n".join(rval) @@ -900,7 +900,7 @@ class SPLib(Msp): if composite_file.get('description'): rval.append('
    1. {} ({}){}
      2. '.format(fn, fn, composite_file.get('description'), opt_text)) else: - rval.append('
    2. {}{}
      3. '.format(fn, fn, opt_text)) + rval.append(f'
    3. {fn}{opt_text}
      4. ') rval.append('
    ') return "\n".join(rval) @@ -995,7 +995,7 @@ class ImzML(Binary): if composite_file.get('description'): rval.append('
  • {} ({}){}
    • '.format(fn, fn, composite_file.get('description'), opt_text)) else: - rval.append('
  • {}{}
    • '.format(fn, fn, opt_text)) + rval.append(f'
  • {fn}{opt_text}
    • ') rval.append('') return "\n".join(rval) @@ -1041,6 +1041,6 @@ class Analyze75(Binary): if composite_file.get('description'): rval.append('
  • {} ({}){}
    • '.format(fn, fn, composite_file.get('description'), opt_text)) else: - rval.append('
  • {}{}
    • '.format(fn, fn, opt_text)) + rval.append(f'
  • {fn}{opt_text}
    • ') rval.append('') return "\n".join(rval) diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 69e2d88de0f..5c489e9d61a 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -330,7 +330,7 @@ class Registry: self.datatype_info_dicts.append(datatype_info_dict) for auto_compressed_type in auto_compressed_types: - compressed_extension = "{}.{}".format(extension, auto_compressed_type) + compressed_extension = f"{extension}.{auto_compressed_type}" upper_compressed_type = auto_compressed_type[0].upper() + auto_compressed_type[1:] auto_compressed_type_name = datatype_class_name + upper_compressed_type attributes = {} @@ -372,7 +372,7 @@ class Registry: if not override: # Do not load the datatype since it conflicts with an existing datatype which we are not supposed # to override. - self.log.debug("Ignoring conflicting datatype with extension '{}' from {}.".format(extension, config)) + self.log.debug(f"Ignoring conflicting datatype with extension '{extension}' from {config}.") # Load datatype sniffers from the config - we'll do this even if one or more datatypes were not properly processed in the config # since sniffers are not tightly coupled with datatypes. self.load_datatype_sniffers(root, @@ -413,7 +413,7 @@ class Registry: if not os.path.exists(path): sample_path = "%s.sample" % path if os.path.exists(sample_path): - self.log.debug("Build site file [{}] not found using sample [{}].".format(path, sample_path)) + self.log.debug(f"Build site file [{path}] not found using sample [{sample_path}].") path = sample_path self.build_sites[site_type] = path @@ -703,13 +703,13 @@ class Registry: del self.datatypes_by_extension[extension].display_applications[display_app.id] if inherit and (self.datatypes_by_extension[extension], display_app) in self.inherit_display_application_by_class: self.inherit_display_application_by_class.remove((self.datatypes_by_extension[extension], display_app)) - self.log.debug("Deactivated display application '{}' for datatype '{}'.".format(display_app.id, extension)) + self.log.debug(f"Deactivated display application '{display_app.id}' for datatype '{extension}'.") else: self.display_applications[display_app.id] = display_app self.datatypes_by_extension[extension].add_display_application(display_app) if inherit and (self.datatypes_by_extension[extension], display_app) not in self.inherit_display_application_by_class: self.inherit_display_application_by_class.append((self.datatypes_by_extension[extension], display_app)) - self.log.debug("Loaded display application '{}' for datatype '{}', inherit={}.".format(display_app.id, extension, inherit)) + self.log.debug(f"Loaded display application '{display_app.id}' for datatype '{extension}', inherit={inherit}.") except Exception: if deactivate: self.log.exception("Error deactivating display application (%s)" % config_path) @@ -720,7 +720,7 @@ class Registry: for d_type2, display_app in self.inherit_display_application_by_class: current_app = d_type1.get_display_application(display_app.id, None) if current_app is None and isinstance(d_type1, type(d_type2)): - self.log.debug("Adding inherited display application '{}' to datatype '{}'".format(display_app.id, extension)) + self.log.debug(f"Adding inherited display application '{display_app.id}' to datatype '{extension}'") d_type1.add_display_application(display_app) def reload_display_applications(self, display_application_ids=None): @@ -894,7 +894,7 @@ class Registry: for convert_ext in self.get_converters_by_datatype(ext): convert_ext_datatype = self.get_datatype_by_extension(convert_ext) if convert_ext_datatype is None: - self.log.warning("Datatype class not found for extension '{}', which is used as target for conversion from datatype '{}'".format(convert_ext, dataset.ext)) + self.log.warning(f"Datatype class not found for extension '{convert_ext}', which is used as target for conversion from datatype '{dataset.ext}'") elif convert_ext_datatype.matches_any(accepted_formats): converted_dataset = dataset and dataset.get_converted_files_by_type(convert_ext) if converted_dataset: @@ -919,7 +919,7 @@ class Registry: help_txt = meta_spec.desc if not help_txt or help_txt == meta_name: help_txt = "" - inputs.append(''.format(meta_name, meta_name, meta_spec.default, help_txt)) + inputs.append(f'') rval[ext] = "\n".join(inputs) if 'auto' not in rval and 'txt' in rval: # need to manually add 'auto' datatype rval['auto'] = rval['txt'] diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index 374bd591e74..a7a744e1d6d 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -186,7 +186,7 @@ class Sequence(data.Text): ds = input_datasets[ds_no] base_name = os.path.basename(ds.file_name) part_path = os.path.join(dir, base_name) - split_data = dict(class_name='{}.{}'.format(cls.__module__, cls.__name__), + split_data = dict(class_name=f'{cls.__module__}.{cls.__name__}', output_name=part_path, input_name=ds.file_name, args=dict(start_sequence=start_sequence, num_sequences=sequences_per_file[part_no])) @@ -431,7 +431,7 @@ class Fasta(Sequence): part_dir = subdir_generator_function() part_path = os.path.join(part_dir, os.path.basename(input_file)) part_file = open(part_path, 'w') - log.debug("Writing {} part to {}".format(input_file, part_path)) + log.debug(f"Writing {input_file} part to {part_path}") start_offset = 0 while True: offset = f.tell() @@ -444,7 +444,7 @@ class Fasta(Sequence): part_dir = subdir_generator_function() part_path = os.path.join(part_dir, os.path.basename(input_file)) part_file = open(part_path, 'w') - log.debug("Writing {} part to {}".format(input_file, part_path)) + log.debug(f"Writing {input_file} part to {part_path}") start_offset = f.tell() part_file.write(line) except Exception as e: @@ -467,7 +467,7 @@ class Fasta(Sequence): part_dir = subdir_generator_function() part_path = os.path.join(part_dir, os.path.basename(input_file)) part_file = open(part_path, 'w') - log.debug("Writing {} part to {}".format(input_file, part_path)) + log.debug(f"Writing {input_file} part to {part_path}") rec_count = 0 while True: line = f.readline() @@ -481,7 +481,7 @@ class Fasta(Sequence): part_dir = subdir_generator_function() part_path = os.path.join(part_dir, os.path.basename(input_file)) part_file = open(part_path, 'w') - log.debug("Writing {} part to {}".format(input_file, part_path)) + log.debug(f"Writing {input_file} part to {part_path}") rec_count = 1 part_file.write(line) except Exception as e: @@ -633,7 +633,7 @@ class Fastg(Sequence): dataset.blurb += '\nversion=%s' % dataset.metadata.version for k, v in dataset.metadata.properties.items(): if k != 'version': - dataset.blurb += '\n{}={}'.format(k, v) + dataset.blurb += f'\n{k}={v}' else: dataset.peek = 'file does not exist' dataset.blurb = 'file purged from disk' diff --git a/lib/galaxy/datatypes/spaln.py b/lib/galaxy/datatypes/spaln.py index 15f6a7e2cff..f0a4d87ba8c 100644 --- a/lib/galaxy/datatypes/spaln.py +++ b/lib/galaxy/datatypes/spaln.py @@ -88,7 +88,7 @@ class _SpalnDb(Data): for i, fname in enumerate(flist): sfname = os.path.split(fname)[-1] f, e = os.path.splitext(fname) - rval.append('
  • {}
    • '.format(sfname, sfname)) + rval.append(f'
  • {sfname}
    • ') rval.append("") with open(dataset.file_name, "w") as f: f.write("\n".join(rval)) diff --git a/lib/galaxy/datatypes/speech.py b/lib/galaxy/datatypes/speech.py index 2e6d7da8ab4..b45eee0336f 100644 --- a/lib/galaxy/datatypes/speech.py +++ b/lib/galaxy/datatypes/speech.py @@ -25,7 +25,7 @@ class TextGrid(Text): def sniff(self, filename): - with open(filename, 'r') as fd: + with open(filename) as fd: text = fd.read(len(self.header)) return text == self.header @@ -56,7 +56,7 @@ class BPF(Text): def set_meta(self, dataset, overwrite=True, **kwd): """Set the metadata for this dataset from the file contents""" types = set() - with open(dataset.dataset.file_name, 'r') as fd: + with open(dataset.dataset.file_name) as fd: for line in fd: # Split the line on a colon rather than regexing it parts = line.split(':') diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index e41df4e1043..fde556b8bd6 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -305,7 +305,7 @@ class Tabular(TabularData): if column_type2 == column_type: return False # neither column type was found in our ordered list, this cannot happen - raise ValueError("Tried to compare unknown column types: {} and {}".format(column_type1, column_type2)) + raise ValueError(f"Tried to compare unknown column types: {column_type1} and {column_type2}") def is_int(column_text): # Don't allow underscores in numeric literals (PEP 515) diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index b1884da52be..9aef9d3193a 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -6,11 +6,10 @@ import json import logging import os import re +import shlex import subprocess import tempfile -from six.moves import shlex_quote - from galaxy.datatypes.data import get_file_peek, Text from galaxy.datatypes.metadata import MetadataElement, MetadataParameter from galaxy.datatypes.sniff import build_sniff_from_prefix, iter_headers @@ -181,7 +180,7 @@ class Ipynb(Json): ofilename = '%s.html' % ofilename except subprocess.CalledProcessError: ofilename = dataset.file_name - log.exception('Command "%s" failed. Could not convert the Jupyter Notebook to HTML, defaulting to plain text.', ' '.join(map(shlex_quote, cmd))) + log.exception('Command "%s" failed. Could not convert the Jupyter Notebook to HTML, defaulting to plain text.', ' '.join(map(shlex.quote, cmd))) return open(ofilename, mode='rb') def set_meta(self, dataset, **kwd): @@ -465,7 +464,7 @@ class Arff(Text): if not dataset.dataset.purged: dataset.peek = get_file_peek(dataset.file_name) dataset.blurb = "Attribute-Relation File Format (ARFF)" - dataset.blurb += ", {} comments, {} attributes".format(dataset.metadata.comment_lines, dataset.metadata.columns) + dataset.blurb += f", {dataset.metadata.comment_lines} comments, {dataset.metadata.columns} attributes" else: dataset.peek = 'file does not exist' dataset.blurb = 'file purged from disc' diff --git a/lib/galaxy/datatypes/tracks.py b/lib/galaxy/datatypes/tracks.py index c82332bc768..0ee0cfa5835 100644 --- a/lib/galaxy/datatypes/tracks.py +++ b/lib/galaxy/datatypes/tracks.py @@ -38,7 +38,7 @@ class UCSCTrackHub(Html): opt_text = '' if composite_file.optional: opt_text = ' (optional)' - rval.append('
  • {}{}'.format(composite_name, composite_name, opt_text)) + rval.append(f'
  • {composite_name}{opt_text}') rval.append('') return "\n".join(rval) diff --git a/lib/galaxy/datatypes/util/maf_utilities.py b/lib/galaxy/datatypes/util/maf_utilities.py index 0f800631482..6eba81a2da8 100644 --- a/lib/galaxy/datatypes/util/maf_utilities.py +++ b/lib/galaxy/datatypes/util/maf_utilities.py @@ -16,7 +16,6 @@ from errno import EMFILE import bx.align.maf import bx.interval_index_file import bx.intervals -from six.moves import xrange try: maketrans = str.maketrans @@ -132,7 +131,7 @@ class TempFileHandler: self.files[index].flush() def __del__(self): - for i in xrange(len(self.files)): + for i in range(len(self.files)): self.close(i, delete=True) @@ -295,7 +294,7 @@ def maf_index_by_uid(maf_uid, index_location_file): maf_files = fields[4].replace("\n", "").replace("\r", "").split(",") return bx.align.maf.MultiIndexed(maf_files, keep_open=True, parse_e_rows=False) except Exception as e: - raise Exception('MAF UID ({}) found, but configuration appears to be malformed: {}'.format(maf_uid, e)) + raise Exception(f'MAF UID ({maf_uid}) found, but configuration appears to be malformed: {e}') except Exception: pass return None @@ -347,7 +346,7 @@ def build_maf_index_species_chromosomes(filename, index_species=None): indexes.add(c.src, forward_strand_start, forward_strand_end, pos, max=c.src_size) except Exception as e: # most likely a bad MAF - log.debug('Building MAF index on {} failed: {}'.format(filename, e)) + log.debug(f'Building MAF index on {filename} failed: {e}') return (None, [], {}, 0) return (indexes, species, species_chromosomes, blocks) @@ -427,7 +426,7 @@ def orient_block_by_region(block, src, region, force_strand=None): def get_oriented_chopped_blocks_for_region(index, src, region, species=None, mincols=0, force_strand=None): - for block, idx, offset in get_oriented_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols, force_strand): + for block, _, _ in get_oriented_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols, force_strand): yield block @@ -488,7 +487,7 @@ def iter_blocks_split_by_species(block, species=None): # generator yielding only chopped and valid blocks for a specified region def get_chopped_blocks_for_region(index, src, region, species=None, mincols=0): - for block, idx, offset in get_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols): + for block, _, _ in get_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols): yield block @@ -512,7 +511,7 @@ def get_region_alignment(index, primary_species, chrom, start, end, strand='+', def reduce_block_by_primary_genome(block, species, chromosome, region_start): # returns ( startIndex, {species:texts} # where texts' contents are reduced to only positions existing in the primary genome - src = "{}.{}".format(species, chromosome) + src = f"{species}.{chromosome}" ref = block.get_component_by_src(src) start_offset = ref.start - region_start species_texts = {} @@ -533,7 +532,7 @@ def fill_region_alignment(alignment, index, primary_species, chrom, start, end, region = bx.intervals.Interval(start, end) region.chrom = chrom region.strand = strand - primary_src = "{}.{}".format(primary_species, chrom) + primary_src = f"{primary_species}.{chrom}" # Order blocks overlaping this position by score, lowest first blocks = [] @@ -684,9 +683,9 @@ def remove_temp_index_file(index_filename): def get_fasta_header(component, attributes={}, suffix=None): header = ">%s(%s):%i-%i|" % (component.src, component.strand, component.get_forward_strand_start(), component.get_forward_strand_end()) for key, value in attributes.items(): - header = "{}{}={}|".format(header, key, value) + header = f"{header}{key}={value}|" if suffix: - header = "{}{}".format(header, suffix) + header = f"{header}{suffix}" else: header = "{}{}".format(header, src_split(component.src)[0]) return header diff --git a/lib/galaxy/dependencies/pipfiles/default/Pipfile b/lib/galaxy/dependencies/pipfiles/default/Pipfile index 9857671ed42..d1012b7b110 100644 --- a/lib/galaxy/dependencies/pipfiles/default/Pipfile +++ b/lib/galaxy/dependencies/pipfiles/default/Pipfile @@ -11,7 +11,6 @@ name = "pypi" [dev-packages] gunicorn = "*" lxml = "!=4.2.2" -mock = "*" NoseHTML = "*" pygithub = "*" pytest = "*" @@ -61,7 +60,6 @@ Beaker = "*" dictobj = "*" nose = "*" Parsley = "*" -six = "*" sortedcontainers = "*" Whoosh = "*" galaxy_sequence_utils = "*" @@ -88,4 +86,4 @@ gxformat2 = "*" refgenconf = ">=0.7.0" [requires] -python_version = "3.5" +python_version = "3.6" diff --git a/lib/galaxy/files/__init__.py b/lib/galaxy/files/__init__.py index 89bfc92fefa..8f44b1c67a9 100644 --- a/lib/galaxy/files/__init__.py +++ b/lib/galaxy/files/__init__.py @@ -12,7 +12,7 @@ log = logging.getLogger(__name__) FileSourcePath = namedtuple('FileSourcePath', ['file_source', 'path']) -class ConfiguredFileSources(object): +class ConfiguredFileSources: """Load plugins and resolve Galaxy URIs to FileSource objects.""" def __init__(self, file_sources_config, conf_file=None, conf_dict=None, load_stock_plugins=False): @@ -161,7 +161,7 @@ class ConfiguredFileSources(object): return ConfiguredFileSources(file_sources_config, conf_dict=sources_as_dict) -class ConfiguredFileSourcesConfig(object): +class ConfiguredFileSourcesConfig: def __init__(self, symlink_allowlist=[], library_import_dir=None, user_library_import_dir=None, ftp_upload_dir=None, ftp_upload_purge=True): self.symlink_allowlist = symlink_allowlist @@ -202,7 +202,7 @@ class ConfiguredFileSourcesConfig(object): ) -class ProvidesUserFileSourcesUserContext(object): +class ProvidesUserFileSourcesUserContext: """Implement a FileSourcesUserContext from a Galaxy ProvidesUserContext (e.g. trans).""" def __init__(self, trans): @@ -228,7 +228,7 @@ class ProvidesUserFileSourcesUserContext(object): return user and user.extra_preferences -class DictFileSourcesUserContext(object): +class DictFileSourcesUserContext: def __init__(self, **kwd): self._kwd = kwd diff --git a/lib/galaxy/files/sources/__init__.py b/lib/galaxy/files/sources/__init__.py index f92bda85e72..83181a3c2b8 100644 --- a/lib/galaxy/files/sources/__init__.py +++ b/lib/galaxy/files/sources/__init__.py @@ -2,16 +2,13 @@ import abc import os import time -import six - from galaxy.util.template import fill_template DEFAULT_SCHEME = "gxfiles" DEFAULT_WRITABLE = False -@six.add_metaclass(abc.ABCMeta) -class FilesSource(object): +class FilesSource(metaclass=abc.ABCMeta): """ """ diff --git a/lib/galaxy/files/sources/galaxy.py b/lib/galaxy/files/sources/galaxy.py index 7cc0ae02f63..739bd2cc3c8 100644 --- a/lib/galaxy/files/sources/galaxy.py +++ b/lib/galaxy/files/sources/galaxy.py @@ -18,7 +18,7 @@ class UserFtpFilesSource(PosixFilesSource): if "delete_on_realize" not in posix_kwds: file_sources_config = kwd.get("file_sources_config") posix_kwds["delete_on_realize"] = file_sources_config.ftp_upload_purge - super(UserFtpFilesSource, self).__init__(**posix_kwds) + super().__init__(**posix_kwds) def get_prefix(self): return None @@ -38,7 +38,7 @@ class LibraryImportFilesSource(PosixFilesSource): doc=doc, ) posix_kwds.update(kwd) - super(LibraryImportFilesSource, self).__init__(**posix_kwds) + super().__init__(**posix_kwds) def get_prefix(self): return None @@ -58,7 +58,7 @@ class UserLibraryImportFilesSource(PosixFilesSource): doc=doc, ) posix_kwds.update(kwd) - super(UserLibraryImportFilesSource, self).__init__(**posix_kwds) + super().__init__(**posix_kwds) def get_prefix(self): return None diff --git a/lib/galaxy/forms/forms.py b/lib/galaxy/forms/forms.py index 9dbf12b4c0a..1b01ed38021 100644 --- a/lib/galaxy/forms/forms.py +++ b/lib/galaxy/forms/forms.py @@ -21,7 +21,7 @@ class FormDefinitionFactory: """ Return new FormDefinition. """ - assert form_type in self.form_types, 'Invalid FormDefinition type ( {} not in {} )'.format(form_type, self.form_types.keys()) + assert form_type in self.form_types, f'Invalid FormDefinition type ( {form_type} not in {self.form_types.keys()} )' assert name, 'FormDefinition requires a name' if description is None: description = '' @@ -113,7 +113,7 @@ class FormDefinitionFieldFactory: visible = string_as_bool(elem.get('visible', 'true')) field_layout = elem.get('layout', None) if field_layout: - assert layout and field_layout in layout, 'Invalid layout specified: {} not in {}'.format(field_layout, layout) + assert layout and field_layout in layout, f'Invalid layout specified: {field_layout} not in {layout}' field_layout = str(layout.index(field_layout)) # existing behavior: integer indexes are stored as strings. why? return self.new(name=name, label=label, required=required, helptext=helptext, default=default, visible=visible, layout=field_layout) diff --git a/lib/galaxy/job_execution/output_collect.py b/lib/galaxy/job_execution/output_collect.py index dfc78ce03a2..8ae9d4feb44 100644 --- a/lib/galaxy/job_execution/output_collect.py +++ b/lib/galaxy/job_execution/output_collect.py @@ -386,7 +386,7 @@ def collect_primary_datasets(job_context, output, input_ext): if dbkey == INPUT_DBKEY_TOKEN: dbkey = job_context.input_dbkey if filename_index == 0 and extra_file_collector.assign_primary_output and output_index == 0: - new_outdata_name = fields_match.name or "{} ({})".format(outdata.name, designation) + new_outdata_name = fields_match.name or f"{outdata.name} ({designation})" outdata.change_datatype(ext) outdata.dbkey = dbkey outdata.designation = designation @@ -399,7 +399,7 @@ def collect_primary_datasets(job_context, output, input_ext): primary_datasets[name] = OrderedDict() visible = fields_match.visible # Create new primary dataset - new_primary_name = fields_match.name or "{} ({})".format(outdata.name, designation) + new_primary_name = fields_match.name or f"{outdata.name} ({designation})" info = outdata.info # TODO: should be able to disambiguate files in different directories... @@ -418,7 +418,7 @@ def collect_primary_datasets(job_context, output, input_ext): dataset_attributes=new_primary_datasets_attributes, ) # Associate new dataset with job - job_context.add_output_dataset_association('__new_primary_file_{}|{}__'.format(name, designation), primary_data) + job_context.add_output_dataset_association(f'__new_primary_file_{name}|{designation}__', primary_data) if new_primary_datasets_attributes: extra_files_path = new_primary_datasets_attributes.get('extra_files', None) @@ -577,7 +577,7 @@ def read_exit_code_from(exit_code_file, id_tag): exit_code = int(exit_code_str) except ValueError: galaxy_id_tag = id_tag - log.warning("({}) Exit code '{}' invalid. Using 0.".format(galaxy_id_tag, exit_code_str)) + log.warning(f"({galaxy_id_tag}) Exit code '{exit_code_str}' invalid. Using 0.") exit_code = 0 return exit_code diff --git a/lib/galaxy/job_metrics/collectl/cli.py b/lib/galaxy/job_metrics/collectl/cli.py index 78eebbbad44..44d4845f2c2 100644 --- a/lib/galaxy/job_metrics/collectl/cli.py +++ b/lib/galaxy/job_metrics/collectl/cli.py @@ -110,13 +110,13 @@ class CollectlCli: if not interval2: return interval_arg self.__validate_interval_arg(interval2, multiple_of=int(interval)) - interval_arg = "{}:{}".format(interval_arg, interval2) + interval_arg = f"{interval_arg}:{interval2}" interval3 = kwargs.get("interval3", None) if not interval3: return interval_arg self.__validate_interval_arg(interval3, multiple_of=int(interval)) - interval_arg = "{}:{}".format(interval_arg, interval3) + interval_arg = f"{interval_arg}:{interval3}" return interval_arg def __validate_interval_arg(self, value, multiple_of=None): diff --git a/lib/galaxy/job_metrics/instrumenters/__init__.py b/lib/galaxy/job_metrics/instrumenters/__init__.py index b7ec78c9f9e..27a21a7114c 100644 --- a/lib/galaxy/job_metrics/instrumenters/__init__.py +++ b/lib/galaxy/job_metrics/instrumenters/__init__.py @@ -49,7 +49,7 @@ class InstrumentPlugin(metaclass=ABCMeta): """ Provide a common pattern for naming files used by instrumentation plugins - to ease their staging out of remote job directories. """ - return "{}_{}_{}".format(INSTRUMENT_FILE_PREFIX, self.plugin_type, name) + return f"{INSTRUMENT_FILE_PREFIX}_{self.plugin_type}_{name}" def _instrument_file_path(self, job_directory, name): return os.path.join(job_directory, self._instrument_file_name(name)) diff --git a/lib/galaxy/job_metrics/instrumenters/collectl.py b/lib/galaxy/job_metrics/instrumenters/collectl.py index 439031b266b..8d0a92e7a8b 100644 --- a/lib/galaxy/job_metrics/instrumenters/collectl.py +++ b/lib/galaxy/job_metrics/instrumenters/collectl.py @@ -48,7 +48,7 @@ class CollectlFormatter(formatting.JobMetricFormatter): else: value_str = str(value) resource_title = FORMATTED_RESOURCE_TITLES.get(resource_type, resource_type) - return ("{} ({})".format(resource_title, stat_type), value_str) + return (f"{resource_title} ({stat_type})", value_str) class CollectlPlugin(InstrumentPlugin): @@ -102,7 +102,7 @@ class CollectlPlugin(InstrumentPlugin): rel_path = filter(self._is_instrumented_collectl_log, contents)[0] path = os.path.join(job_directory, rel_path) except IndexError: - message = "Failed to find collectl log in directory {}, files were {}".format(job_directory, contents) + message = f"Failed to find collectl log in directory {job_directory}, files were {contents}" raise Exception(message) properties = dict( diff --git a/lib/galaxy/job_metrics/instrumenters/cpuinfo.py b/lib/galaxy/job_metrics/instrumenters/cpuinfo.py index 8917fad3393..f6e877a95a9 100644 --- a/lib/galaxy/job_metrics/instrumenters/cpuinfo.py +++ b/lib/galaxy/job_metrics/instrumenters/cpuinfo.py @@ -51,7 +51,7 @@ class CpuInfoPlugin(InstrumentPlugin): # If verbose, dump information about each processor # into database... key, value = line.split(":", 1) - key = "processor_{}_{}".format(current_processor, key.strip()) + key = f"processor_{current_processor}_{key.strip()}" value = value properties["processor_count"] = processor_count return properties diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index ee695304f41..89308fe235c 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -803,7 +803,7 @@ class JobConfiguration(ConfiguresHandlers): log.warning("A non-class name was found in __all__, ignoring: %s" % id) continue except AssertionError: - log.warning("Job runner classes must be subclassed from BaseJobRunner, {} has bases: {}".format(id, runner_class.__bases__)) + log.warning(f"Job runner classes must be subclassed from BaseJobRunner, {id} has bases: {runner_class.__bases__}") continue try: rval[id] = runner_class(self.app, runner.get('workers', JobConfiguration.DEFAULT_NWORKERS), **runner.get('kwds', {})) @@ -811,7 +811,7 @@ class JobConfiguration(ConfiguresHandlers): log.exception("Job runner '%s:%s' has not been converted to a new-style runner or encountered TypeError on load", module_name, class_name) rval[id] = runner_class(self.app) - log.debug("Loaded job runner '{}:{}' as '{}'".format(module_name, class_name, id)) + log.debug(f"Loaded job runner '{module_name}:{class_name}' as '{id}'") return rval def is_id(self, collection): @@ -847,13 +847,13 @@ class JobConfiguration(ConfiguresHandlers): destination.params = job_runners[destination.runner].url_to_destination(destination.url).params destination.converted = True if destination.params: - log.debug("Legacy destination with id '{}', url '{}' converted, got params:".format(id, destination.url)) + log.debug(f"Legacy destination with id '{id}', url '{destination.url}' converted, got params:") for k, v in destination.params.items(): - log.debug(" {}: {}".format(k, v)) + log.debug(f" {k}: {v}") else: - log.debug("Legacy destination with id '{}', url '{}' converted, got params:".format(id, destination.url)) + log.debug(f"Legacy destination with id '{id}', url '{destination.url}' converted, got params:") else: - log.warning("Legacy destination with id '{}' could not be converted: Unknown runner plugin: {}".format(id, destination.runner)) + log.warning(f"Legacy destination with id '{id}' could not be converted: Unknown runner plugin: {destination.runner}") class HasResourceParameters: @@ -1129,7 +1129,7 @@ class JobWrapper(HasResourceParameters): if version_string_cmd_raw: version_command_template = string.Template(version_string_cmd_raw) version_string_cmd = version_command_template.safe_substitute({"__tool_directory__": compute_environment.tool_directory()}) - self.write_version_cmd = "{} > {} 2>&1".format(version_string_cmd, compute_environment.version_path()) + self.write_version_cmd = f"{version_string_cmd} > {compute_environment.version_path()} 2>&1" else: self.write_version_cmd = None return self.extra_filenames @@ -1412,7 +1412,7 @@ class JobWrapper(HasResourceParameters): """ if job is None: job = self.get_job() - log.debug('({}) Persisting job destination (destination id: {})'.format(job.id, job_destination.id)) + log.debug(f'({job.id}) Persisting job destination (destination id: {job_destination.id})') job.destination_id = job_destination.id job.destination_params = job_destination.params job.job_runner_name = job_destination.runner @@ -1594,7 +1594,7 @@ class JobWrapper(HasResourceParameters): try: self.reclaim_ownership() except Exception: - log.exception('({}) Failed to change ownership of {}, failing'.format(job.id, self.working_directory)) + log.exception(f'({job.id}) Failed to change ownership of {self.working_directory}, failing') return fail() # if the job was deleted, don't finish it @@ -1641,7 +1641,7 @@ class JobWrapper(HasResourceParameters): for dataset_path in self.get_output_fnames(): try: shutil.move(dataset_path.false_path, dataset_path.real_path) - log.debug("finish(): Moved {} to {}".format(dataset_path.false_path, dataset_path.real_path)) + log.debug(f"finish(): Moved {dataset_path.false_path} to {dataset_path.real_path}") except OSError: # this can happen if Galaxy is restarted during the job's # finish method - the false_path file has already moved, @@ -1661,7 +1661,7 @@ class JobWrapper(HasResourceParameters): import_model_store = store.get_import_model_store_for_directory(os.path.join(self.working_directory, 'metadata', 'outputs_populated'), app=self.app, import_options=import_options) import_model_store.perform_import(history=job.history) except Exception: - log.exception("problem importing job outputs. stdout [{}] stderr [{}]".format(job.stdout, job.stderr)) + log.exception(f"problem importing job outputs. stdout [{job.stdout}] stderr [{job.stderr}]") raise output_dataset_associations = job.output_datasets + job.output_library_datasets for dataset_assoc in output_dataset_associations: @@ -1883,14 +1883,14 @@ class JobWrapper(HasResourceParameters): def get_env_setup_clause(self): if self.app.config.environment_setup_file is None: return '' - return '[ -f "{}" ] && . {}'.format(self.app.config.environment_setup_file, self.app.config.environment_setup_file) + return f'[ -f "{self.app.config.environment_setup_file}" ] && . {self.app.config.environment_setup_file}' def get_input_dataset_fnames(self, ds): filenames = [] filenames = [ds.file_name] # we will need to stage in metadata file names also # TODO: would be better to only stage in metadata files that are actually needed (found in command line, referenced in config files, etc.) - for key, value in ds.metadata.items(): + for _, value in ds.metadata.items(): if isinstance(value, model.MetadataFile): filenames.append(value.file_name) return filenames @@ -1938,7 +1938,7 @@ class JobWrapper(HasResourceParameters): for (hda, dataset_path) in self.output_hdas_and_paths.values(): if hda == dataset: return dataset_path - raise KeyError("Couldn't find job output for [{}] in [{}]".format(dataset, self.output_hdas_and_paths.values())) + raise KeyError(f"Couldn't find job output for [{dataset}] in [{self.output_hdas_and_paths.values()}]") def get_mutable_output_fnames(self): if self.output_paths is None: @@ -2100,7 +2100,7 @@ class JobWrapper(HasResourceParameters): dependency_shell_commands = metadata_tool.build_dependency_shell_commands(job_directory=self.working_directory, metadata=True) if dependency_shell_commands: dependency_shell_commands = "; ".join(dependency_shell_commands) - command = "{}; {}".format(dependency_shell_commands, command) + command = f"{dependency_shell_commands}; {command}" return command def check_for_entry_points(self, check_already_configured=True): diff --git a/lib/galaxy/jobs/actions/post.py b/lib/galaxy/jobs/actions/post.py index ffc9abd33ec..dc94ba99aef 100644 --- a/lib/galaxy/jobs/actions/post.py +++ b/lib/galaxy/jobs/actions/post.py @@ -364,14 +364,14 @@ class DeleteIntermediatesAction(DefaultJobAction): if wfi_step_job: jobs_to_check.append(wfi_step_job) else: - log.debug("No job found yet for wfi_step {}, (step {})".format(wfi_step, wfi_step.workflow_step)) + log.debug(f"No job found yet for wfi_step {wfi_step}, (step {wfi_step.workflow_step})") for j2c in jobs_to_check: creating_jobs = [] for input_dataset in j2c.input_datasets: if not input_dataset.dataset: - log.debug("PJA Async Issue: No dataset attached to input_dataset {} during handling of workflow invocation {}".format(input_dataset.id, wfi)) + log.debug(f"PJA Async Issue: No dataset attached to input_dataset {input_dataset.id} during handling of workflow invocation {wfi}") elif not input_dataset.dataset.creating_job: - log.debug("PJA Async Issue: No creating job attached to dataset {} during handling of workflow invocation {}".format(input_dataset.dataset.id, wfi)) + log.debug(f"PJA Async Issue: No creating job attached to dataset {input_dataset.dataset.id} during handling of workflow invocation {wfi}") else: creating_jobs.append((input_dataset, input_dataset.dataset.creating_job)) for (input_dataset, creating_job) in creating_jobs: @@ -383,7 +383,7 @@ class DeleteIntermediatesAction(DefaultJobAction): safe_to_delete = True for job_to_check in [d_j.job for d_j in input_dataset.dependent_jobs]: if job_to_check != job and job_to_check.state not in [job.states.OK, job.states.DELETED]: - log.trace("Workflow Intermediates cleanup attempted, but non-terminal state '{}' detected for job {}".format(job_to_check.state, job_to_check.id)) + log.trace(f"Workflow Intermediates cleanup attempted, but non-terminal state '{job_to_check.state}' detected for job {job_to_check.id}") safe_to_delete = False if safe_to_delete: # Support purging here too. diff --git a/lib/galaxy/jobs/command_factory.py b/lib/galaxy/jobs/command_factory.py index 706262de17e..db8ddbccd35 100644 --- a/lib/galaxy/jobs/command_factory.py +++ b/lib/galaxy/jobs/command_factory.py @@ -155,7 +155,7 @@ def __externalize_commands(job_wrapper, shell, commands_builder, remote_command_ tool_commands, ) write_script(local_container_script, script_contents, config) - commands = "{} {}".format(shell, local_container_script) + commands = f"{shell} {local_container_script}" # TODO: Cleanup for_pulsar hack. # - Integrate Pulsar sending tool_stdout/tool_stderr back # https://github.com/galaxyproject/pulsar/pull/202 @@ -170,7 +170,7 @@ def __externalize_commands(job_wrapper, shell, commands_builder, remote_command_ for_pulsar = True if not for_pulsar: commands += " > ../outputs/tool_stdout 2> ../outputs/tool_stderr" - log.info("Built script [{}] for tool command [{}]".format(local_container_script, tool_commands)) + log.info(f"Built script [{local_container_script}] for tool command [{tool_commands}]") return commands @@ -236,14 +236,14 @@ def __handle_metadata(commands_builder, job_wrapper, runner, remote_command_para metadata_command = metadata_command.strip() if metadata_command: # Place Galaxy and its dependencies in environment for metadata regardless of tool. - metadata_command = "{}{}".format(SETUP_GALAXY_FOR_METADATA, metadata_command) + metadata_command = f"{SETUP_GALAXY_FOR_METADATA}{metadata_command}" commands_builder.capture_return_code() commands_builder.append_command(metadata_command) def __copy_if_exists_command(work_dir_output): source_file, destination = work_dir_output - return "if [ -f {} ] ; then cp {} {} ; fi".format(source_file, source_file, destination) + return f"if [ -f {source_file} ] ; then cp {source_file} {destination} ; fi" class CommandsBuilder: diff --git a/lib/galaxy/jobs/handler.py b/lib/galaxy/jobs/handler.py index 047727e993d..eaf93a8a5c7 100644 --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -5,11 +5,11 @@ import datetime import os import time from collections import defaultdict - -from six.moves.queue import ( +from queue import ( Empty, - Queue + Queue, ) + from sqlalchemy.exc import OperationalError from sqlalchemy.sql.expression import ( and_, @@ -217,7 +217,7 @@ class JobHandlerQueue(Monitors): for job in jobs_at_startup: if not self.app.toolbox.has_tool(job.tool_id, job.tool_version, exact=True): - log.warning("({}) Tool '{}' removed from tool config, unable to recover job".format(job.id, job.tool_id)) + log.warning(f"({job.id}) Tool '{job.tool_id}' removed from tool config, unable to recover job") self.job_wrapper(job).fail('This tool was disabled before the job completed. Please contact your Galaxy administrator.') elif job.job_runner_name is not None and job.job_runner_external_id is None: # This could happen during certain revisions of Galaxy where a runner URL was persisted before the job was dispatched to a runner. @@ -238,7 +238,7 @@ class JobHandlerQueue(Monitors): log.info('(%s) Converted job from a URL to a destination and recovered' % (job.id)) elif job.job_runner_name is None: # Never (fully) dispatched - log.debug("({}) No job runner assigned and job still in '{}' state, adding to the job handler queue".format(job.id, job.state)) + log.debug(f"({job.id}) No job runner assigned and job still in '{job.state}' state, adding to the job handler queue") if self.track_jobs_in_database: job.set_state(model.Job.states.NEW) else: @@ -486,7 +486,7 @@ class JobHandlerQueue(Monitors): elif dataset_state == model.Dataset.states.ERROR: jobs_to_pause[job_id].append("Input dataset '%s' is in error state" % hda_name) elif dataset_state != model.Dataset.states.OK: - jobs_to_ignore[job_id].append("Input dataset '{}' is in {} state".format(hda_name, dataset_state)) + jobs_to_ignore[job_id].append(f"Input dataset '{hda_name}' is in {dataset_state} state") for job_id in sorted(jobs_to_pause): pause_message = ", ".join(jobs_to_pause[job_id]) pause_message = "%s. To resume this job fix the input dataset(s)." % pause_message @@ -557,7 +557,7 @@ class JobHandlerQueue(Monitors): # Cause the job_destination to be set and cached by the mapper job_destination = job_wrapper.job_destination except AssertionError as e: - log.warning("({}) Tool '{}' removed from tool config, unable to run job".format(job.id, job.tool_id)) + log.warning(f"({job.id}) Tool '{job.tool_id}' removed from tool config, unable to run job") job_wrapper.fail(e) return JOB_ERROR, job_destination except JobNotReadyException as e: @@ -631,7 +631,7 @@ class JobHandlerQueue(Monitors): continue # don't run jobs for which the input dataset was deleted if idata.deleted: - self.job_wrappers.pop(job.id, self.job_wrapper(job)).fail("input data {} (file: {}) was deleted before the job started".format(idata.hid, idata.file_name)) + self.job_wrappers.pop(job.id, self.job_wrapper(job)).fail(f"input data {idata.hid} (file: {idata.file_name}) was deleted before the job started") return JOB_INPUT_DELETED # an error in the input data causes us to bail immediately elif idata.state == idata.states.ERROR: @@ -1000,12 +1000,12 @@ class DefaultJobDispatcher: try: if isinstance(job_wrapper, TaskWrapper): # DBTODO Refactor - log.debug("({}) Dispatching task {} to {} runner".format(job_wrapper.job_id, job_wrapper.task_id, runner_name)) + log.debug(f"({job_wrapper.job_id}) Dispatching task {job_wrapper.task_id} to {runner_name} runner") else: - log.debug("({}) Dispatching to {} runner".format(job_wrapper.job_id, runner_name)) + log.debug(f"({job_wrapper.job_id}) Dispatching to {runner_name} runner") self.job_runners[runner_name].put(job_wrapper) except KeyError: - log.error('put(): ({}) Invalid job runner: {}'.format(job_wrapper.job_id, runner_name)) + log.error(f'put(): ({job_wrapper.job_id}) Invalid job runner: {runner_name}') job_wrapper.fail(DEFAULT_JOB_PUT_FAILURE_MESSAGE) def stop(self, job, job_wrapper): @@ -1023,11 +1023,11 @@ class DefaultJobDispatcher: job_runner_name = job.get_job_runner_name() if job_runner_name is not None: runner_name = job_runner_name.split(":", 1)[0] - log.debug("Stopping job {} in {} runner".format(job_wrapper.get_id_tag(), runner_name)) + log.debug(f"Stopping job {job_wrapper.get_id_tag()} in {runner_name} runner") try: self.job_runners[runner_name].stop_job(job_wrapper) except KeyError: - log.error('stop(): ({}) Invalid job runner: {}'.format(job_wrapper.get_id_tag(), runner_name)) + log.error(f'stop(): ({job_wrapper.get_id_tag()}) Invalid job runner: {runner_name}') # Job and output dataset states have already been updated, so nothing is done here. def recover(self, job, job_wrapper): @@ -1036,7 +1036,7 @@ class DefaultJobDispatcher: try: self.job_runners[runner_name].recover(job, job_wrapper) except KeyError: - log.error('recover(): ({}) Invalid job runner: {}'.format(job_wrapper.job_id, runner_name)) + log.error(f'recover(): ({job_wrapper.job_id}) Invalid job runner: {runner_name}') job_wrapper.fail(DEFAULT_JOB_PUT_FAILURE_MESSAGE) def shutdown(self): diff --git a/lib/galaxy/jobs/rule_helper.py b/lib/galaxy/jobs/rule_helper.py index 8f9a50b4062..36a05b773d3 100644 --- a/lib/galaxy/jobs/rule_helper.py +++ b/lib/galaxy/jobs/rule_helper.py @@ -187,7 +187,7 @@ class RuleHelper: invocation for jobs outside of workflows. """ if hash_by not in VALID_JOB_HASH_STRATEGIES: - message = "Do not know how to hash jobs by {}, must be one of {}".format(hash_by, VALID_JOB_HASH_STRATEGIES) + message = f"Do not know how to hash jobs by {hash_by}, must be one of {VALID_JOB_HASH_STRATEGIES}" raise Exception(message) if hash_by == "workflow_invocation": diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py index 434896cbbf0..ee7b9758974 100644 --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -9,10 +9,9 @@ import sys import threading import time import traceback - -from six.moves.queue import ( +from queue import ( Empty, - Queue + Queue, ) import galaxy.jobs @@ -90,7 +89,7 @@ class BaseJobRunner: """ self.work_queue = Queue() self.work_threads = [] - log.debug('Starting {} {} workers'.format(self.nworkers, self.runner_name)) + log.debug(f'Starting {self.nworkers} {self.runner_name} workers') for i in range(self.nworkers): worker = threading.Thread(name="%s.work_thread-%d" % (self.runner_name, i), target=self.run_next) worker.daemon = True @@ -129,7 +128,7 @@ class BaseJobRunner: except Exception: name = 'unknown' try: - action_str = 'galaxy.jobs.runners.{}.{}'.format(self.__class__.__name__.lower(), name) + action_str = f'galaxy.jobs.runners.{self.__class__.__name__.lower()}.{name}' action_timer = self.app.execution_timer_factory.get_timer( 'internals.%s' % action_str, 'job runner action %s for job ${job_id} executed' % (action_str) @@ -137,7 +136,7 @@ class BaseJobRunner: method(arg) log.trace(action_timer.to_str(job_id=job_id)) except Exception: - log.exception("({}) Unhandled exception calling {}".format(job_id, name)) + log.exception(f"({job_id}) Unhandled exception calling {name}") if not isinstance(arg, JobState): job_state = JobState(job_wrapper=arg, job_destination={}) else: @@ -151,7 +150,7 @@ class BaseJobRunner: put_timer = ExecutionTimer() job_wrapper.enqueue() self.mark_as_queued(job_wrapper) - log.debug("Job [{}] queued {}".format(job_wrapper.job_id, put_timer)) + log.debug(f"Job [{job_wrapper.job_id}] queued {put_timer}") def mark_as_queued(self, job_wrapper): self.work_queue.put((self.queue_job, job_wrapper)) @@ -222,12 +221,12 @@ class BaseJobRunner: # Make sure the job hasn't been deleted if job_state == model.Job.states.DELETED: - log.debug("({}) Job deleted by user before it entered the {} queue".format(job_id, self.runner_name)) + log.debug(f"({job_id}) Job deleted by user before it entered the {self.runner_name} queue") if self.app.config.cleanup_job in ("always", "onsuccess"): job_wrapper.cleanup() return False elif job_state != model.Job.states.QUEUED: - log.info("({}) Job is in state {}, skipping execution".format(job_id, job_state)) + log.info(f"({job_id}) Job is in state {job_state}, skipping execution") # cleanup may not be safe in all states return False @@ -352,13 +351,13 @@ class BaseJobRunner: tmp_dir=job_wrapper.working_directory, # We don't want to overwrite metadata that was copied over in init_meta(), as per established behavior kwds={'overwrite' : False}) - external_metadata_script = "{} {} {}".format(lib_adjust, venv, external_metadata_script) + external_metadata_script = f"{lib_adjust} {venv} {external_metadata_script}" if resolve_requirements: dependency_shell_commands = self.app.datatypes_registry.set_external_metadata_tool.build_dependency_shell_commands(job_directory=job_wrapper.working_directory) if dependency_shell_commands: if isinstance(dependency_shell_commands, list): dependency_shell_commands = "&&".join(dependency_shell_commands) - external_metadata_script = "{}&&{}".format(dependency_shell_commands, external_metadata_script) + external_metadata_script = f"{dependency_shell_commands}&&{external_metadata_script}" log.debug('executing external set_meta script for job %d: %s' % (job_wrapper.job_id, external_metadata_script)) external_metadata_proc = subprocess.Popen(args=external_metadata_script, shell=True, @@ -396,7 +395,7 @@ class BaseJobRunner: # Additional logging to enable if debugging from_work_dir handling, metadata # commands, etc... (or just peak in the job script.) job_id = job_wrapper.job_id - log.debug('({}) command is: {}'.format(job_id, command_line)) + log.debug(f'({job_id}) command is: {command_line}') options.update(**kwds) return job_script(**options) @@ -600,7 +599,7 @@ class JobState: if not hasattr(self, "job_id"): prefix = "(%s)" % self.job_wrapper.get_id_tag() else: - prefix = "({}/{})".format(self.job_wrapper.get_id_tag(), self.job_id) + prefix = f"({self.job_wrapper.get_id_tag()}/{self.job_id})" log.debug("{} Unable to cleanup {}: {}".format(prefix, file, unicodify(e))) diff --git a/lib/galaxy/jobs/runners/chronos.py b/lib/galaxy/jobs/runners/chronos.py index eb4e68a8afa..5f7aa3dfee5 100644 --- a/lib/galaxy/jobs/runners/chronos.py +++ b/lib/galaxy/jobs/runners/chronos.py @@ -196,7 +196,7 @@ class ChronosJobRunner(AsynchronousJobRunner): msg = 'Job {name!r} failed more than {retries!s} times' reason = msg.format(name=job_name, retries=str(max_retries)) return self._mark_as_failed(job_state, reason) - reason = 'Job {name!r} not found'.format(name=job_name) + reason = f'Job {job_name!r} not found' return self._mark_as_failed(job_state, reason) def _mark_as_successful(self, job_state): @@ -264,7 +264,7 @@ class ChronosJobRunner(AsynchronousJobRunner): jobs = self._chronos_client.list() job = [x for x in jobs if x['name'] == job_id] if len(job) > 1: - msg = 'Multiple jobs found with name {name!r}'.format(name=job_id) + msg = f'Multiple jobs found with name {job_id!r}' LOGGER.error(msg) raise ChronosRunnerException(msg) return job[0] if job else None diff --git a/lib/galaxy/jobs/runners/cli.py b/lib/galaxy/jobs/runners/cli.py index e770c34a90d..9345953a3f4 100644 --- a/lib/galaxy/jobs/runners/cli.py +++ b/lib/galaxy/jobs/runners/cli.py @@ -47,7 +47,7 @@ class ShellJobRunner(AsynchronousJobRunner): job_params = {'job_' + k: v for k, v in [kv.split('=', 1) for kv in job_params.split('&')]} params.update(shell_params) params.update(job_params) - log.debug("Converted URL '{}' to destination runner=cli, params={}".format(url, params)) + log.debug(f"Converted URL '{url}' to destination runner=cli, params={params}") # Create a dynamic JobDestination return JobDestination(runner='cli', params=params) @@ -94,7 +94,7 @@ class ShellJobRunner(AsynchronousJobRunner): job_wrapper.cleanup() return - log.debug("({}) submitting file: {}".format(galaxy_id_tag, ajs.job_file)) + log.debug(f"({galaxy_id_tag}) submitting file: {ajs.job_file}") returncode, stdout = self.submit(shell, job_interface, ajs.job_file, galaxy_id_tag, retry=MAX_SUBMIT_RETRY) if returncode != 0: @@ -108,7 +108,7 @@ class ShellJobRunner(AsynchronousJobRunner): job_wrapper.fail("failure submitting job") return - log.info("({}) queued with identifier: {}".format(galaxy_id_tag, external_job_id)) + log.info(f"({galaxy_id_tag}) queued with identifier: {external_job_id}") # store runner information for tracking if Galaxy restarts job_wrapper.set_external_id(external_job_id) @@ -131,8 +131,8 @@ class ShellJobRunner(AsynchronousJobRunner): cmd_out = shell.execute(job_interface.submit(job_file)) if cmd_out.returncode == 0: return cmd_out.returncode, cmd_out.stdout - stdout = '({}) submission failed (stdout): {}'.format(galaxy_id_tag, cmd_out.stdout) - stderr = '({}) submission failed (stderr): {}'.format(galaxy_id_tag, cmd_out.stderr) + stdout = f'({galaxy_id_tag}) submission failed (stdout): {cmd_out.stdout}' + stderr = f'({galaxy_id_tag}) submission failed (stderr): {cmd_out.stderr}' if retry > 0: log.debug("%s, retrying in %s seconds", stdout, timeout) log.debug("%s, retrying in %s seconds", stderr, timeout) @@ -161,15 +161,15 @@ class ShellJobRunner(AsynchronousJobRunner): if ajs.job_wrapper.get_state() == model.Job.states.DELETED: continue - log.debug("({}/{}) job not found in batch state check".format(id_tag, external_job_id)) + log.debug(f"({id_tag}/{external_job_id}) job not found in batch state check") shell_params, job_params = self.parse_destination_params(ajs.job_destination.params) shell, job_interface = self.get_cli_plugins(shell_params, job_params) cmd_out = shell.execute(job_interface.get_single_status(external_job_id)) state = job_interface.parse_single_status(cmd_out.stdout, external_job_id) if not state == model.Job.states.OK: - log.warning('({}/{}) job not found in batch state check, but found in individual state check'.format(id_tag, external_job_id)) + log.warning(f'({id_tag}/{external_job_id}) job not found in batch state check, but found in individual state check') if state != old_state: - log.debug("({}/{}) state change: from {} to {}".format(id_tag, external_job_id, old_state, state)) + log.debug(f"({id_tag}/{external_job_id}) state change: from {old_state} to {state}") if not state == model.Job.states.OK: # No need to change_state when the state is OK, this will be handled by `self.finish_job` ajs.job_wrapper.change_state(state) @@ -186,7 +186,7 @@ class ShellJobRunner(AsynchronousJobRunner): external_metadata = not asbool(ajs.job_wrapper.job_destination.params.get("embed_metadata_in_job", DEFAULT_EMBED_METADATA_IN_JOB)) if external_metadata: self.work_queue.put((self.handle_metadata_externally, ajs)) - log.debug('({}/{}) job execution finished, running job wrapper finish method'.format(id_tag, external_job_id)) + log.debug(f'({id_tag}/{external_job_id}) job execution finished, running job wrapper finish method') self.work_queue.put((self.finish_job, ajs)) else: new_watched.append(ajs) @@ -234,9 +234,9 @@ class ShellJobRunner(AsynchronousJobRunner): shell, job_interface = self.get_cli_plugins(shell_params, job_params) cmd_out = shell.execute(job_interface.delete(job.job_runner_external_id)) assert cmd_out.returncode == 0, cmd_out.stderr - log.debug("({}/{}) Terminated at user's request".format(job.id, job.job_runner_external_id)) + log.debug(f"({job.id}/{job.job_runner_external_id}) Terminated at user's request") except Exception as e: - log.debug("({}/{}) User killed running job, but error encountered during termination: {}".format(job.id, job.job_runner_external_id, e)) + log.debug(f"({job.id}/{job.job_runner_external_id}) User killed running job, but error encountered during termination: {e}") def recover(self, job, job_wrapper): """Recovers jobs stuck in the queued/running state when Galaxy started""" @@ -250,12 +250,12 @@ class ShellJobRunner(AsynchronousJobRunner): ajs.job_wrapper = job_wrapper ajs.job_destination = job_wrapper.job_destination if job.state == model.Job.states.RUNNING: - log.debug("({}/{}) is still in running state, adding to the runner monitor queue".format(job.id, job.job_runner_external_id)) + log.debug(f"({job.id}/{job.job_runner_external_id}) is still in running state, adding to the runner monitor queue") ajs.old_state = model.Job.states.RUNNING ajs.running = True self.monitor_queue.put(ajs) elif job.state == model.Job.states.QUEUED: - log.debug("({}/{}) is still in queued state, adding to the runner monitor queue".format(job.id, job.job_runner_external_id)) + log.debug(f"({job.id}/{job.job_runner_external_id}) is still in queued state, adding to the runner monitor queue") ajs.old_state = model.Job.states.QUEUED ajs.running = False self.monitor_queue.put(ajs) diff --git a/lib/galaxy/jobs/runners/condor.py b/lib/galaxy/jobs/runners/condor.py index e59e6952ded..41146d6fb64 100644 --- a/lib/galaxy/jobs/runners/condor.py +++ b/lib/galaxy/jobs/runners/condor.py @@ -63,7 +63,7 @@ class CondorJobRunner(AsynchronousJobRunner): Remove this function in 21.01 """ if cjs.job_wrapper is not None: - job_file = "{}/galaxy_{}.sh".format(self.app.config.cluster_files_directory, cjs.job_wrapper.job_id) + job_file = f"{self.app.config.cluster_files_directory}/galaxy_{cjs.job_wrapper.job_id}.sh" if not os.path.exists(cjs.job_file) and os.path.exists(job_file): cluster_files_dir_and_id = (self.app.config.cluster_files_directory, cjs.job_wrapper.get_id_tag()) cjs.output_file = "%s/galaxy_%s.o" % cluster_files_dir_and_id @@ -155,11 +155,11 @@ class CondorJobRunner(AsynchronousJobRunner): job_wrapper.cleanup() return - log.debug("({}) submitting file {}".format(galaxy_id_tag, executable)) + log.debug(f"({galaxy_id_tag}) submitting file {executable}") external_job_id, message = condor_submit(submit_file) if external_job_id is None: - log.debug("condor_submit failed for job {}: {}".format(job_wrapper.get_id_tag(), message)) + log.debug(f"condor_submit failed for job {job_wrapper.get_id_tag()}: {message}") if self.app.config.cleanup_job == "always": os.unlink(submit_file) cjs.cleanup() @@ -168,7 +168,7 @@ class CondorJobRunner(AsynchronousJobRunner): os.unlink(submit_file) - log.info("({}) queued as {}".format(galaxy_id_tag, external_job_id)) + log.info(f"({galaxy_id_tag}) queued as {external_job_id}") # store runner information for tracking if Galaxy restarts job_wrapper.set_external_id(external_job_id) @@ -201,8 +201,8 @@ class CondorJobRunner(AsynchronousJobRunner): cjs.user_log_size = log_size except Exception: # so we don't kill the monitor thread - log.exception("({}/{}) Unable to check job status".format(galaxy_id_tag, job_id)) - log.warning("({}/{}) job will now be errored".format(galaxy_id_tag, job_id)) + log.exception(f"({galaxy_id_tag}/{job_id}) Unable to check job status") + log.warning(f"({galaxy_id_tag}/{job_id}) job will now be errored") cjs.fail_message = "Cluster could not complete job" self.work_queue.put((self.fail_job, cjs)) continue @@ -212,10 +212,10 @@ class CondorJobRunner(AsynchronousJobRunner): cjs.job_wrapper.check_for_entry_points() if job_running and not cjs.running: - log.debug("({}/{}) job is now running".format(galaxy_id_tag, job_id)) + log.debug(f"({galaxy_id_tag}/{job_id}) job is now running") cjs.job_wrapper.change_state(model.Job.states.RUNNING) if not job_running and cjs.running: - log.debug("({}/{}) job has stopped running".format(galaxy_id_tag, job_id)) + log.debug(f"({galaxy_id_tag}/{job_id}) job has stopped running") # Will switching from RUNNING to QUEUED confuse Galaxy? # cjs.job_wrapper.change_state( model.Job.states.QUEUED ) if job_complete: @@ -223,11 +223,11 @@ class CondorJobRunner(AsynchronousJobRunner): external_metadata = not asbool(cjs.job_wrapper.job_destination.params.get("embed_metadata_in_job", True)) if external_metadata: self._handle_metadata_externally(cjs.job_wrapper, resolve_requirements=True) - log.debug("({}/{}) job has completed".format(galaxy_id_tag, job_id)) + log.debug(f"({galaxy_id_tag}/{job_id}) job has completed") self.work_queue.put((self.finish_job, cjs)) continue if job_failed: - log.debug("({}/{}) job failed".format(galaxy_id_tag, job_id)) + log.debug(f"({galaxy_id_tag}/{job_id}) job failed") cjs.failed = True self.work_queue.put((self.finish_job, cjs)) continue @@ -243,7 +243,7 @@ class CondorJobRunner(AsynchronousJobRunner): galaxy_id_tag = job_wrapper.get_id_tag() if job.container: try: - log.info("stop_job(): {}: trying to stop container .... ({})".format(job.id, external_id)) + log.info(f"stop_job(): {job.id}: trying to stop container .... ({external_id})") # self.watched = [cjs for cjs in self.watched if cjs.job_id != external_id] new_watch_list = list() cjs = None @@ -260,21 +260,21 @@ class CondorJobRunner(AsynchronousJobRunner): external_metadata = not asbool(cjs.job_wrapper.job_destination.params.get("embed_metadata_in_job", True)) if external_metadata: self._handle_metadata_externally(cjs.job_wrapper, resolve_requirements=True) - log.debug("({}/{}) job has completed".format(galaxy_id_tag, external_id)) + log.debug(f"({galaxy_id_tag}/{external_id}) job has completed") self.work_queue.put((self.finish_job, cjs)) except Exception as e: - log.warning("stop_job(): {}: trying to stop container failed. ({})".format(job.id, e)) + log.warning(f"stop_job(): {job.id}: trying to stop container failed. ({e})") try: self._kill_container(job_wrapper) except Exception as e: - log.warning("stop_job(): {}: trying to kill container failed. ({})".format(job.id, e)) + log.warning(f"stop_job(): {job.id}: trying to kill container failed. ({e})") failure_message = condor_stop(external_id) if failure_message: - log.debug("({}). Failed to stop condor {}".format(external_id, failure_message)) + log.debug(f"({external_id}). Failed to stop condor {failure_message}") else: failure_message = condor_stop(external_id) if failure_message: - log.debug("({}). Failed to stop condor {}".format(external_id, failure_message)) + log.debug(f"({external_id}). Failed to stop condor {failure_message}") def recover(self, job, job_wrapper): """Recovers jobs stuck in the queued/running state when Galaxy started""" @@ -293,11 +293,11 @@ class CondorJobRunner(AsynchronousJobRunner): cjs.register_cleanup_file_attribute('user_log') self.__old_state_paths(cjs) # remove in 21.01 if job.state == model.Job.states.RUNNING: - log.debug("({}/{}) is still in running state, adding to the DRM queue".format(job.id, job.job_runner_external_id)) + log.debug(f"({job.id}/{job.job_runner_external_id}) is still in running state, adding to the DRM queue") cjs.running = True self.monitor_queue.put(cjs) elif job.state == model.Job.states.QUEUED: - log.debug("({}/{}) is still in DRM queued state, adding to the DRM queue".format(job.id, job.job_runner_external_id)) + log.debug(f"({job.id}/{job.job_runner_external_id}) is still in DRM queued state, adding to the DRM queue") cjs.running = False self.monitor_queue.put(cjs) @@ -317,7 +317,7 @@ class CondorJobRunner(AsynchronousJobRunner): return self._run_command(cont.container_info['commands'][command], external_id)[0] def _run_command(self, command, external_job_id): - command = 'condor_ssh_to_job {} {}'.format(external_job_id, command) + command = f'condor_ssh_to_job {external_job_id} {command}' p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, close_fds=True, preexec_fn=os.setpgrp) stdout, stderr = p.communicate() diff --git a/lib/galaxy/jobs/runners/drmaa.py b/lib/galaxy/jobs/runners/drmaa.py index 47fdc21deb1..167f72c820c 100644 --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -106,7 +106,7 @@ class DRMAAJobRunner(AsynchronousJobRunner): native_spec = url.split('/')[2] if native_spec: params = dict(nativeSpecification=native_spec) - log.debug("Converted URL '{}' to destination runner=drmaa, params={}".format(url, params)) + log.debug(f"Converted URL '{url}' to destination runner=drmaa, params={params}") return JobDestination(runner='drmaa', params=params) else: log.debug("Converted URL '%s' to destination runner=drmaa" % url) @@ -218,7 +218,7 @@ class DRMAAJobRunner(AsynchronousJobRunner): if external_job_id is None: job_wrapper.fail("(%s) could not queue job" % galaxy_id_tag) return - log.info("({}) queued as {}".format(galaxy_id_tag, external_job_id)) + log.info(f"({galaxy_id_tag}) queued as {external_job_id}") # store runner information for tracking if Galaxy restarts job_wrapper.set_external_id(external_job_id) @@ -274,7 +274,7 @@ class DRMAAJobRunner(AsynchronousJobRunner): galaxy_id_tag = ajs.job_wrapper.get_id_tag() state = None try: - assert external_job_id not in (None, 'None'), '({}/{}) Invalid job id'.format(galaxy_id_tag, external_job_id) + assert external_job_id not in (None, 'None'), f'({galaxy_id_tag}/{external_job_id}) Invalid job id' state = self.ds.job_status(external_job_id) # Reset exception retries for retry_exception in RETRY_EXCEPTIONS_LOWER: @@ -306,8 +306,8 @@ class DRMAAJobRunner(AsynchronousJobRunner): return None except Exception: # so we don't kill the monitor thread - log.exception("({}/{}) unable to check job status".format(galaxy_id_tag, external_job_id)) - log.warning("({}/{}) job will now be errored".format(galaxy_id_tag, external_job_id)) + log.exception(f"({galaxy_id_tag}/{external_job_id}) unable to check job status") + log.warning(f"({galaxy_id_tag}/{external_job_id}) job will now be errored") ajs.fail_message = "Cluster could not complete job" self.work_queue.put((self.fail_job, ajs)) return None @@ -361,13 +361,13 @@ class DRMAAJobRunner(AsynchronousJobRunner): cmd = shlex.split(kill_script) cmd.extend([str(ext_id), str(self.userid)]) commands.execute(cmd) - log.info("({}/{}) Removed from DRM queue at user's request".format(job.id, ext_id)) + log.info(f"({job.id}/{ext_id}) Removed from DRM queue at user's request") except drmaa.InvalidJobException: - log.exception("({}/{}) User killed running job, but it was already dead".format(job.id, ext_id)) + log.exception(f"({job.id}/{ext_id}) User killed running job, but it was already dead") except commands.CommandLineException as e: log.error("({}/{}) User killed running job, but command execution failed: {}".format(job.id, ext_id, unicodify(e))) except Exception: - log.exception("({}/{}) User killed running job, but error encountered removing from DRM queue".format(job.id, ext_id)) + log.exception(f"({job.id}/{ext_id}) User killed running job, but error encountered removing from DRM queue") def recover(self, job, job_wrapper): """Recovers jobs stuck in the queued/running state when Galaxy started""" @@ -381,12 +381,12 @@ class DRMAAJobRunner(AsynchronousJobRunner): ajs.job_wrapper = job_wrapper ajs.job_destination = job_wrapper.job_destination if job.state == model.Job.states.RUNNING: - log.debug("({}/{}) is still in running state, adding to the DRM queue".format(job.id, job.get_job_runner_external_id())) + log.debug(f"({job.id}/{job.get_job_runner_external_id()}) is still in running state, adding to the DRM queue") ajs.old_state = drmaa.JobState.RUNNING ajs.running = True self.monitor_queue.put(ajs) elif job.get_state() == model.Job.states.QUEUED: - log.debug("({}/{}) is still in DRM queued state, adding to the DRM queue".format(job.id, job.get_job_runner_external_id())) + log.debug(f"({job.id}/{job.get_job_runner_external_id()}) is still in DRM queued state, adding to the DRM queue") ajs.old_state = drmaa.JobState.QUEUED_ACTIVE ajs.running = False self.monitor_queue.put(ajs) @@ -395,10 +395,10 @@ class DRMAAJobRunner(AsynchronousJobRunner): """ Stores the content of a DRMAA JobTemplate object in a file as a JSON string. Path is hard-coded, but it's no worse than other path in this module. Uses Galaxy's JobID, so file is expected to be unique.""" - filename = "{}/{}.jt_json".format(self.app.config.cluster_files_directory, job_wrapper.get_id_tag()) + filename = f"{self.app.config.cluster_files_directory}/{job_wrapper.get_id_tag()}.jt_json" with open(filename, 'w+') as fp: json.dump(jt, fp) - log.debug('({}) Job script for external submission is: {}'.format(job_wrapper.job_id, filename)) + log.debug(f'({job_wrapper.job_id}) Job script for external submission is: {filename}') return filename def external_runjob(self, external_runjob_script, jobtemplate_filename, username): diff --git a/lib/galaxy/jobs/runners/godocker.py b/lib/galaxy/jobs/runners/godocker.py index 90a53533197..bfe2d4b52d7 100644 --- a/lib/galaxy/jobs/runners/godocker.py +++ b/lib/galaxy/jobs/runners/godocker.py @@ -253,13 +253,13 @@ class GodockerJobRunner(AsynchronousJobRunner): job_wrapper.command_line = job.command_line ajs.job_wrapper = job_wrapper if job.state == model.Job.states.RUNNING: - log.debug("({}/{}) is still in running state, adding to the god queue".format(job.id, job.get_job_runner_external_id())) + log.debug(f"({job.id}/{job.get_job_runner_external_id()}) is still in running state, adding to the god queue") ajs.old_state = 'R' ajs.running = True self.monitor_queue.put(ajs) elif job.state == model.Job.states.QUEUED: - log.debug("({}/{}) is still in god queued state, adding to the god queue".format(job.id, job.get_job_runner_external_id())) + log.debug(f"({job.id}/{job.get_job_runner_external_id()}) is still in god queued state, adding to the god queue") ajs.old_state = 'Q' ajs.running = False self.monitor_queue.put(ajs) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index 5e03f9d26a4..c2574fb303b 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -527,7 +527,7 @@ class KubernetesJobRunner(AsynchronousJobRunner): self.__cleanup_k8s_job(job_to_delete) # TODO assert whether job parallelism == 0 # assert not job_to_delete.exists(), "Could not delete job,"+job.job_runner_external_id+" it still exists" - log.debug("({}/{}) Terminated at user's request".format(job.id, job.job_runner_external_id)) + log.debug(f"({job.id}/{job.job_runner_external_id}) Terminated at user's request") except Exception as e: log.exception("({}/{}) User killed running job, but error encountered during termination: {}".format( job.id, job.job_runner_external_id, e)) diff --git a/lib/galaxy/jobs/runners/local.py b/lib/galaxy/jobs/runners/local.py index dbc192ab047..c4f334cdb0b 100644 --- a/lib/galaxy/jobs/runners/local.py +++ b/lib/galaxy/jobs/runners/local.py @@ -94,7 +94,7 @@ class LocalJobRunner(BaseJobRunner): try: stdout_file = tempfile.NamedTemporaryFile(mode='wb+', suffix='_stdout', dir=job_wrapper.working_directory) stderr_file = tempfile.NamedTemporaryFile(mode='wb+', suffix='_stderr', dir=job_wrapper.working_directory) - log.debug('({}) executing job script: {}'.format(job_id, job_file)) + log.debug(f'({job_id}) executing job script: {job_file}') # The preexec_fn argument of Popen() is used to call os.setpgrp() in # the child process just before the child is executed. This will set # the PGID of the child process to its PID (i.e. ensures that it is diff --git a/lib/galaxy/jobs/runners/pbs.py b/lib/galaxy/jobs/runners/pbs.py index a0f6e43cfc9..c6aa74595a9 100644 --- a/lib/galaxy/jobs/runners/pbs.py +++ b/lib/galaxy/jobs/runners/pbs.py @@ -130,7 +130,7 @@ class PBSJobRunner(AsynchronousJobRunner): pbs_destination = '@%s' % server pbs_queue = url_split[3] or None if pbs_queue is not None: - pbs_destination = '{}{}'.format(pbs_queue, pbs_destination) + pbs_destination = f'{pbs_queue}{pbs_destination}' params = dict(destination=pbs_destination) @@ -147,7 +147,7 @@ class PBSJobRunner(AsynchronousJobRunner): param, value = opt.split(None, 1) params[param] = value - log.debug("Converted URL '{}' to destination runner=pbs, params={}".format(url, params)) + log.debug(f"Converted URL '{url}' to destination runner=pbs, params={params}") # Create a dynamic JobDestination return JobDestination(runner='pbs', params=params) @@ -186,7 +186,7 @@ class PBSJobRunner(AsynchronousJobRunner): try: rval.append(dict(name=getattr(pbs, 'ATTR_' + arg), value=value)) except AttributeError as e: - raise Exception("Invalid parameter '{}': {}".format(arg, e)) + raise Exception(f"Invalid parameter '{arg}': {e}") return rval def __get_pbs_server(self, job_destination_params): @@ -230,13 +230,13 @@ class PBSJobRunner(AsynchronousJobRunner): if c <= 0: errno, text = pbs.error() job_wrapper.fail("Unable to queue job for execution. Resubmitting the job may succeed.") - log.error("Connection to PBS server for submit failed: {}: {}".format(errno, text)) + log.error(f"Connection to PBS server for submit failed: {errno}: {text}") return # define job attributes - ofile = "{}/{}.o".format(self.app.config.cluster_files_directory, job_wrapper.job_id) - efile = "{}/{}.e".format(self.app.config.cluster_files_directory, job_wrapper.job_id) - ecfile = "{}/{}.ec".format(self.app.config.cluster_files_directory, job_wrapper.job_id) + ofile = f"{self.app.config.cluster_files_directory}/{job_wrapper.job_id}.o" + efile = f"{self.app.config.cluster_files_directory}/{job_wrapper.job_id}.e" + ecfile = f"{self.app.config.cluster_files_directory}/{job_wrapper.job_id}.ec" output_fnames = job_wrapper.get_output_fnames() @@ -262,7 +262,7 @@ class PBSJobRunner(AsynchronousJobRunner): ] # define PBS job options - attrs.append(dict(name=pbs.ATTR_N, value=str("{}_{}_{}".format(job_wrapper.job_id, job_wrapper.tool.id, job_wrapper.user)))) + attrs.append(dict(name=pbs.ATTR_N, value=str(f"{job_wrapper.job_id}_{job_wrapper.tool.id}_{job_wrapper.user}"))) job_attrs = pbs.new_attropl(len(attrs) + len(pbs_options)) for i, attr in enumerate(attrs + pbs_options): job_attrs[i].name = attr['name'] @@ -287,7 +287,7 @@ class PBSJobRunner(AsynchronousJobRunner): env_setup_commands = [stage_commands] script = self.get_job_file(job_wrapper, exit_code_path=ecfile, env_setup_commands=env_setup_commands, shell=job_wrapper.shell) - job_file = "{}/{}.sh".format(self.app.config.cluster_files_directory, job_wrapper.job_id) + job_file = f"{self.app.config.cluster_files_directory}/{job_wrapper.job_id}.sh" self.write_executable_script(job_file, script) # job was deleted while we were preparing it if job_wrapper.get_state() == model.Job.states.DELETED: @@ -302,7 +302,7 @@ class PBSJobRunner(AsynchronousJobRunner): # The job tag includes the job and the task identifier # (if a TaskWrapper was passed in): galaxy_job_id = job_wrapper.get_id_tag() - log.debug("({}) submitting file {}".format(galaxy_job_id, job_file)) + log.debug(f"({galaxy_job_id}) submitting file {job_file}") tries = 0 while tries < 5: @@ -320,9 +320,9 @@ class PBSJobRunner(AsynchronousJobRunner): return if pbs_queue_name is None: - log.debug("({}) queued in default queue as {}".format(galaxy_job_id, job_id)) + log.debug(f"({galaxy_job_id}) queued in default queue as {job_id}") else: - log.debug("({}) queued in {} queue as {}".format(galaxy_job_id, pbs_queue_name, job_id)) + log.debug(f"({galaxy_job_id}) queued in {pbs_queue_name} queue as {job_id}") # persist destination job_wrapper.set_job_destination(job_destination, job_id) @@ -356,7 +356,7 @@ class PBSJobRunner(AsynchronousJobRunner): old_state = pbs_job_state.old_state pbs_server_name = self.__get_pbs_server(pbs_job_state.job_destination.params) if pbs_server_name in failures: - log.debug("({}/{}) Skipping state check because PBS server connection failed".format(galaxy_job_id, job_id)) + log.debug(f"({galaxy_job_id}/{job_id}) Skipping state check because PBS server connection failed") new_watched.append(pbs_job_state) continue try: @@ -367,13 +367,13 @@ class PBSJobRunner(AsynchronousJobRunner): try: # Recheck to make sure it wasn't a communication problem self.check_single_job(pbs_server_name, job_id) - log.warning("({}/{}) PBS job was not in state check list, but was found with individual state check".format(galaxy_job_id, job_id)) + log.warning(f"({galaxy_job_id}/{job_id}) PBS job was not in state check list, but was found with individual state check") new_watched.append(pbs_job_state) except Exception: errno, text = pbs.error() if errno == 15001: # 15001 == job not in queue - log.debug("({}/{}) PBS job has left queue".format(galaxy_job_id, job_id)) + log.debug(f"({galaxy_job_id}/{job_id}) PBS job has left queue") self.work_queue.put((self.finish_job, pbs_job_state)) else: # Unhandled error, continue to monitor @@ -381,7 +381,7 @@ class PBSJobRunner(AsynchronousJobRunner): new_watched.append(pbs_job_state) continue if status.job_state != old_state: - log.debug("({}/{}) PBS job state changed from {} to {}".format(galaxy_job_id, job_id, old_state, status.job_state)) + log.debug(f"({galaxy_job_id}/{job_id}) PBS job state changed from {old_state} to {status.job_state}") if status.job_state == "R" and not pbs_job_state.running: pbs_job_state.running = True pbs_job_state.job_wrapper.change_state(model.Job.states.RUNNING) @@ -396,18 +396,18 @@ class PBSJobRunner(AsynchronousJobRunner): # "keep_completed" is enabled in PBS, so try to check exit status try: assert int(status.exit_status) == 0 - log.debug("({}/{}) PBS job has completed successfully".format(galaxy_job_id, job_id)) + log.debug(f"({galaxy_job_id}/{job_id}) PBS job has completed successfully") except AssertionError: exit_status = int(status.exit_status) error_message = JOB_EXIT_STATUS.get(exit_status, 'Unknown error: %s' % status.exit_status) pbs_job_state.fail_message = CLUSTER_ERROR_MESSAGE % error_message - log.error('({}/{}) PBS job failed: {}'.format(galaxy_job_id, job_id, error_message)) + log.error(f'({galaxy_job_id}/{job_id}) PBS job failed: {error_message}') pbs_job_state.stop_job = False self.work_queue.put((self.fail_job, pbs_job_state)) continue except AttributeError: # No exit_status, can't verify proper completion so we just have to assume success. - log.debug("({}/{}) PBS job has completed".format(galaxy_job_id, job_id)) + log.debug(f"({galaxy_job_id}/{job_id}) PBS job has completed") self.work_queue.put((self.finish_job, pbs_job_state)) continue pbs_job_state.old_state = status.job_state @@ -497,14 +497,14 @@ class PBSJobRunner(AsynchronousJobRunner): stage_name = os.path.join(self.app.config.pbs_stage_path, os.path.split(fname)[1]) else: stage_name = fname - stage += "{}@{}:{}".format(stage_name, self.app.config.pbs_dataset_server, fname) + stage += f"{stage_name}@{self.app.config.pbs_dataset_server}:{fname}" return stage def stop_job(self, job_wrapper): """Attempts to delete a job from the PBS queue""" job = job_wrapper.get_job() job_id = job.get_job_runner_external_id().encode('utf-8') - job_tag = "({}/{})".format(job.get_id_tag(), job_id) + job_tag = f"({job.get_id_tag()}/{job_id})" log.debug("%s Stopping PBS job" % job_tag) # Declare the connection handle c so that it can be cleaned up: @@ -526,7 +526,7 @@ class PBSJobRunner(AsynchronousJobRunner): % job_tag) except Exception: e = traceback.format_exc() - log.debug("{} Unable to stop job: {}".format(job_tag, e)) + log.debug(f"{job_tag} Unable to stop job: {e}") finally: # Cleanup: disconnect from the server. if (None is not c): @@ -536,22 +536,22 @@ class PBSJobRunner(AsynchronousJobRunner): """Recovers jobs stuck in the queued/running state when Galaxy started""" job_id = job.get_job_runner_external_id() pbs_job_state = AsynchronousJobState() - pbs_job_state.output_file = "{}/{}.o".format(self.app.config.cluster_files_directory, job.id) - pbs_job_state.error_file = "{}/{}.e".format(self.app.config.cluster_files_directory, job.id) - pbs_job_state.exit_code_file = "{}/{}.ec".format(self.app.config.cluster_files_directory, job.id) - pbs_job_state.job_file = "{}/{}.sh".format(self.app.config.cluster_files_directory, job.id) + pbs_job_state.output_file = f"{self.app.config.cluster_files_directory}/{job.id}.o" + pbs_job_state.error_file = f"{self.app.config.cluster_files_directory}/{job.id}.e" + pbs_job_state.exit_code_file = f"{self.app.config.cluster_files_directory}/{job.id}.ec" + pbs_job_state.job_file = f"{self.app.config.cluster_files_directory}/{job.id}.sh" pbs_job_state.job_id = str(job_id) pbs_job_state.runner_url = job_wrapper.get_job_runner_url() pbs_job_state.job_destination = job_wrapper.job_destination job_wrapper.command_line = job.command_line pbs_job_state.job_wrapper = job_wrapper if job.state == model.Job.states.RUNNING: - log.debug("({}/{}) is still in running state, adding to the PBS queue".format(job.id, job.get_job_runner_external_id())) + log.debug(f"({job.id}/{job.get_job_runner_external_id()}) is still in running state, adding to the PBS queue") pbs_job_state.old_state = 'R' pbs_job_state.running = True self.monitor_queue.put(pbs_job_state) elif job.state == model.Job.states.QUEUED: - log.debug("({}/{}) is still in PBS queued state, adding to the PBS queue".format(job.id, job.get_job_runner_external_id())) + log.debug(f"({job.id}/{job.get_job_runner_external_id()}) is still in PBS queued state, adding to the PBS queue") pbs_job_state.old_state = 'Q' pbs_job_state.running = False self.monitor_queue.put(pbs_job_state) diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index 242bef5d6c0..a7c9694552c 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -539,7 +539,7 @@ class PulsarJobRunner(AsynchronousJobRunner): def get_client_from_wrapper(self, job_wrapper): job_id = job_wrapper.job_id if hasattr(job_wrapper, 'task_id'): - job_id = "{}_{}".format(job_id, job_wrapper.task_id) + job_id = f"{job_id}_{job_wrapper.task_id}" params = job_wrapper.job_destination.params.copy() user = job_wrapper.get_job().user if user: @@ -685,7 +685,7 @@ class PulsarJobRunner(AsynchronousJobRunner): # Remote kill pulsar_url = job.job_runner_name job_id = job.job_runner_external_id - log.debug("Attempt remote Pulsar kill of job with url {} and id {}".format(pulsar_url, job_id)) + log.debug(f"Attempt remote Pulsar kill of job with url {pulsar_url} and id {job_id}") client = self.get_client(job.destination_params, job_id) client.kill() @@ -1010,7 +1010,7 @@ class PulsarComputeEnvironment(ComputeEnvironment): # first but that adds untested logic that wouln't ever be used. remote_input_path = self.path_mapper.remote_input_path_rewrite(metadata_val, client_input_path_type=CLIENT_INPUT_PATH_TYPES.INPUT_METADATA_PATH) if remote_input_path: - log.info("input_metadata_rewrite is {} from {}".format(remote_input_path, metadata_val)) + log.info(f"input_metadata_rewrite is {remote_input_path} from {metadata_val}") self.path_rewrites_input_metadata[metadata_val] = remote_input_path return remote_input_path diff --git a/lib/galaxy/jobs/runners/state_handlers/resubmit.py b/lib/galaxy/jobs/runners/state_handlers/resubmit.py index 2051b360755..4899466ce6e 100644 --- a/lib/galaxy/jobs/runners/state_handlers/resubmit.py +++ b/lib/galaxy/jobs/runners/state_handlers/resubmit.py @@ -93,7 +93,7 @@ def _handle_resubmit_definitions(resubmit_definitions, app, job_runner, job_stat external_id = getattr(job_state, "job_id", None) if external_id: - job_log_prefix = "({}/{})".format(job_state.job_wrapper.job_id, job_state.job_id) + job_log_prefix = f"({job_state.job_wrapper.job_id}/{job_state.job_id})" else: job_log_prefix = "(%s)" % (job_state.job_wrapper.job_id) diff --git a/lib/galaxy/jobs/runners/univa.py b/lib/galaxy/jobs/runners/univa.py index 2a89cdc1252..0db05fd3de9 100644 --- a/lib/galaxy/jobs/runners/univa.py +++ b/lib/galaxy/jobs/runners/univa.py @@ -91,13 +91,13 @@ class UnivaJobRunner(DRMAAJobRunner): ajs.fail_message = "This job failed because it was cancelled." drmaa_state = self.drmaa.JobState.FAILED elif ("signal" in extinfo and extinfo["signal"] == "SIGKILL") and time_wasted > time_granted: - log.error('({tag}/{jobid}) Job hit walltime'.format(tag=ajs.job_wrapper.get_id_tag(), jobid=ajs.job_id)) + log.error(f'({ajs.job_wrapper.get_id_tag()}/{ajs.job_id}) Job hit walltime') ajs.fail_message = "This job was terminated because it ran longer than the maximum allowed job run time." ajs.runner_state = ajs.runner_states.WALLTIME_REACHED drmaa_state = self.drmaa.JobState.FAILED # test wasted>granted memory only if failed != 0 and exit_status != 0, ie if marked as failed elif state == self.drmaa.JobState.FAILED and mem_wasted > mem_granted * slots: - log.error('({idtag}/{jobid}) Job hit memory limit ({used}>{limit})'.format(idtag=ajs.job_wrapper.get_id_tag(), jobid=ajs.job_id, used=mem_wasted, limit=mem_granted)) + log.error(f'({ajs.job_wrapper.get_id_tag()}/{ajs.job_id}) Job hit memory limit ({mem_wasted}>{mem_granted})') ajs.fail_message = "This job was terminated because it used more than the maximum allowed memory." ajs.runner_state = ajs.runner_states.MEMORY_LIMIT_REACHED drmaa_state = self.drmaa_job_states.FAILED @@ -106,10 +106,10 @@ class UnivaJobRunner(DRMAAJobRunner): # TODO return True? return True # job was not actually terminal elif state == self.drmaa.JobState.UNDETERMINED: - log.warning('({tag}/{jobid}) Job state could not be determined'.format(tag=ajs.job_wrapper.get_id_tag(), jobid=ajs.job_id)) + log.warning(f'({ajs.job_wrapper.get_id_tag()}/{ajs.job_id}) Job state could not be determined') drmaa_state = self.drmaa_job_states.FAILED else: - log.error("DRMAAUniva: job {job_id} determined unknown state {state}".format(job_id=ajs.job_id, state=state)) + log.error(f"DRMAAUniva: job {ajs.job_id} determined unknown state {state}") drmaa_state = self.drmaa_job_states.FAILED # by default, finish the job with the state from drmaa return super()._complete_terminal_job(ajs, drmaa_state=drmaa_state) @@ -203,7 +203,7 @@ class UnivaJobRunner(DRMAAJobRunner): try: stdout = commands.execute(cmd).strip() except commands.CommandLineException as e: - if slp <= 32 and "job id {jobid} not found".format(jobid=job_id) in e.stderr: + if slp <= 32 and f"job id {job_id} not found" in e.stderr: time.sleep(slp) slp *= 2 continue @@ -397,7 +397,7 @@ class UnivaJobRunner(DRMAAJobRunner): extinfo["slots"] = float(rv.resourceUsage['slots']) # log.debug("wait -> \texitStatus {0}\thasCoreDump {1}\thasExited {2}\thasSignal {3}\tjobId {4}\t\tterminatedSignal {5}\twasAborted {6}\tresourceUsage {7}".format(rv.exitStatus, rv.hasCoreDump, rv.hasExited, rv.hasSignal, rv.jobId, rv.terminatedSignal, rv.wasAborted, rv.resourceUsage)) if rv.wasAborted: - log.error("DRMAAUniva: job {job_id} was aborted according to wait()".format(job_id=job_id)) + log.error(f"DRMAAUniva: job {job_id} was aborted according to wait()") extinfo["deleted"] = True return self.drmaa.JobState.FAILED @@ -405,19 +405,19 @@ class UnivaJobRunner(DRMAAJobRunner): # but also violation of scheduler constraints state = self.drmaa.JobState.DONE if rv.exitStatus != 0: - log.error("DRMAAUniva: job {job_id} has exit status {status}".format(job_id=job_id, status=rv.exitStatus)) + log.error(f"DRMAAUniva: job {job_id} has exit status {rv.exitStatus}") extinfo["state"] = self.drmaa.JobState.FAILED if not rv.hasExited or rv.hasSignal: if rv.hasCoreDump != 0: - log.error("DRMAAUniva: job {job_id} has core dump".format(job_id=job_id)) + log.error(f"DRMAAUniva: job {job_id} has core dump") extinfo["state"] = self.drmaa.JobState.FAILED elif len(rv.terminatedSignal) > 0: - log.error("DRMAAUniva: job {job_id} was kill by signal {signal}".format(job_id=job_id, signal=rv.terminatedSignal)) + log.error(f"DRMAAUniva: job {job_id} was kill by signal {rv.terminatedSignal}") state = self.drmaa.JobState.FAILED extinfo["signal"] = rv.terminatedSignal elif rv.wasAborted == 0: - log.error("DRMAAUniva: job {job_id} has finished in unclear condition".format(job_id=job_id)) + log.error(f"DRMAAUniva: job {job_id} has finished in unclear condition") state = self.drmaa.JobState.FAILED # log.debug("UnivaJobRunner._get_drmaa_state_wait ({jobid}) -> {state}".format(jobid=job_id, state=self.drmaa_job_state_strings[state])) return state @@ -535,7 +535,7 @@ class UnivaJobRunner(DRMAAJobRunner): elif "w" in state: return self.drmaa.JobState.QUEUED_ACTIVE else: - log.error("DRMAAUniva: job {job_id} unknown state from qstat: {state}".format(job_id=job_id, state=state)) + log.error(f"DRMAAUniva: job {job_id} unknown state from qstat: {state}") return self.drmaa.JobState.UNDETERMINED @@ -569,12 +569,12 @@ def _parse_native_specs(job_id, native_spec): if m is not None: tme = _parse_time(m.group(1)) if tme is None: - log.error("DRMAAUniva: job {job_id} has unparsable time native spec {spec}".format(job_id=job_id, spec=native_spec)) + log.error(f"DRMAAUniva: job {job_id} has unparsable time native spec {native_spec}") # parse memory m = re.search(r"mem=([\d.]+[KGMT]?)[\s,]*", native_spec) if m is not None: mem = size_to_bytes(m.group(1)) # mem = _parse_mem(m.group(1)) if mem is None: - log.error("DRMAAUniva: job {job_id} has unparsable memory native spec {spec}".format(job_id=job_id, spec=native_spec)) + log.error(f"DRMAAUniva: job {job_id} has unparsable memory native spec {native_spec}") return tme, mem diff --git a/lib/galaxy/jobs/runners/util/cli/__init__.py b/lib/galaxy/jobs/runners/util/cli/__init__.py index 30788b9d3c8..bfd359a6edb 100644 --- a/lib/galaxy/jobs/runners/util/cli/__init__.py +++ b/lib/galaxy/jobs/runners/util/cli/__init__.py @@ -39,7 +39,7 @@ class CliInterface: def __load_from_path(module_path): base_module = importlib.import_module(module_path) for module_info in pkgutil.iter_modules(base_module.__path__): - module = importlib.import_module('{}.{}'.format(module_path, module_info.name)) + module = importlib.import_module(f'{module_path}.{module_info.name}') yield module def __load(module_path, d): diff --git a/lib/galaxy/jobs/runners/util/cli/job/lsf.py b/lib/galaxy/jobs/runners/util/cli/job/lsf.py index 0c4a805ac24..14b43946201 100644 --- a/lib/galaxy/jobs/runners/util/cli/job/lsf.py +++ b/lib/galaxy/jobs/runners/util/cli/job/lsf.py @@ -52,7 +52,7 @@ class LSF(BaseJobExec): # Generated template. template_scriptargs = '' for k, v in scriptargs.items(): - template_scriptargs += '#BSUB {} {}\n'.format(k, v) + template_scriptargs += f'#BSUB {k} {v}\n' return dict(headers=template_scriptargs) def submit(self, script_file): diff --git a/lib/galaxy/jobs/runners/util/cli/job/slurm.py b/lib/galaxy/jobs/runners/util/cli/job/slurm.py index 618ebc3b2a4..c13cc69e071 100644 --- a/lib/galaxy/jobs/runners/util/cli/job/slurm.py +++ b/lib/galaxy/jobs/runners/util/cli/job/slurm.py @@ -47,7 +47,7 @@ class Slurm(BaseJobExec): # Generated template. template_scriptargs = '' for k, v in scriptargs.items(): - template_scriptargs += '#SBATCH {} {}\n'.format(k, v) + template_scriptargs += f'#SBATCH {k} {v}\n' return dict(headers=template_scriptargs) def submit(self, script_file): diff --git a/lib/galaxy/jobs/runners/util/cli/job/torque.py b/lib/galaxy/jobs/runners/util/cli/job/torque.py index b67a985d7ca..ee86ca35017 100644 --- a/lib/galaxy/jobs/runners/util/cli/job/torque.py +++ b/lib/galaxy/jobs/runners/util/cli/job/torque.py @@ -59,7 +59,7 @@ class Torque(BaseJobExec): log.warning(ERROR_MESSAGE_UNRECOGNIZED_ARG % k) template_pbsargs = '' for k, v in pbsargs.items(): - template_pbsargs += '#PBS {} {}\n'.format(k, v) + template_pbsargs += f'#PBS {k} {v}\n' return dict(headers=template_pbsargs) def submit(self, script_file): diff --git a/lib/galaxy/jobs/runners/util/condor/__init__.py b/lib/galaxy/jobs/runners/util/condor/__init__.py index 54a6a062d3f..c23bc8f1375 100644 --- a/lib/galaxy/jobs/runners/util/condor/__init__.py +++ b/lib/galaxy/jobs/runners/util/condor/__init__.py @@ -58,7 +58,7 @@ def build_submit_description(executable, output, error, user_log, query_params): submit_description = [] for key, value in all_query_params.items(): - submit_description.append('{} = {}'.format(key, value)) + submit_description.append(f'{key} = {value}') submit_description.append('executable = ' + executable) submit_description.append('output = ' + output) submit_description.append('error = ' + error) diff --git a/lib/galaxy/jobs/runners/util/env.py b/lib/galaxy/jobs/runners/util/env.py index f3ef1c19781..63e79e41513 100644 --- a/lib/galaxy/jobs/runners/util/env.py +++ b/lib/galaxy/jobs/runners/util/env.py @@ -29,7 +29,7 @@ def env_to_statement(env): return execute name = env['name'] value = __escape(env['value'], env) - return '{}={}; export {}'.format(name, value, name) + return f'{name}={value}; export {name}' def __escape(value, env): diff --git a/lib/galaxy/jobs/transfer_manager.py b/lib/galaxy/jobs/transfer_manager.py index 2c00c19844a..47aa43fded3 100644 --- a/lib/galaxy/jobs/transfer_manager.py +++ b/lib/galaxy/jobs/transfer_manager.py @@ -5,12 +5,11 @@ IPC with multiple process configurations. import json import logging import os +import shlex import socket import subprocess import threading -from six.moves import shlex_quote - from galaxy.util import ( listify, sleeper, @@ -75,12 +74,12 @@ class TransferManager: # not the case, this process will need to be moved to a # non-blocking method. cmd = self.command + [tj.id] - log.debug('Transfer command is: %s', ' '.join(map(shlex_quote, cmd))) + log.debug('Transfer command is: %s', ' '.join(map(shlex.quote, cmd))) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) p.wait() output = p.stdout.read(32768) if p.returncode != 0: - log.error('Spawning transfer job failed: {}: {}'.format(tj.id, output)) + log.error(f'Spawning transfer job failed: {tj.id}: {output}') tj.state = tj.states.ERROR tj.info = 'Spawning transfer job failed: %s' % output.splitlines()[-1] self.sa_session.add(tj) @@ -154,7 +153,7 @@ class TransferManager: except Exception: self.sa_session.refresh(tj) if tj.state == tj.states.RUNNING: - log.error('Transfer job {} is marked as running but pid {} appears to be dead.'.format(tj.id, tj.pid)) + log.error(f'Transfer job {tj.id} is marked as running but pid {tj.pid} appears to be dead.') dead.append(tj) if dead: self.run(dead) diff --git a/lib/galaxy/managers/base.py b/lib/galaxy/managers/base.py index 62f9637a3a1..de978773283 100644 --- a/lib/galaxy/managers/base.py +++ b/lib/galaxy/managers/base.py @@ -114,8 +114,8 @@ def get_object(trans, id, class_name, check_ownership=False, check_accessible=Fa item = trans.sa_session.query(item_class).get(decoded_id) assert item is not None except Exception: - log.exception("Invalid {} id ( {} ) specified.".format(class_name, id)) - raise exceptions.MessageException("Invalid {} id ( {} ) specified".format(class_name, id), type="error") + log.exception(f"Invalid {class_name} id ( {id} ) specified.") + raise exceptions.MessageException(f"Invalid {class_name} id ( {id} ) specified", type="error") if check_ownership or check_accessible: security_check(trans, item, check_ownership, check_accessible) diff --git a/lib/galaxy/managers/citations.py b/lib/galaxy/managers/citations.py index f66c0c6131b..c8ac75f8aab 100644 --- a/lib/galaxy/managers/citations.py +++ b/lib/galaxy/managers/citations.py @@ -67,7 +67,7 @@ def parse_citation(elem, citation_manager): try: citation = citation_class(elem, citation_manager) except Exception as e: - raise Exception("Invalid citation of type '{}' with content '{}': {}".format(citation_type, elem.text, e)) + raise Exception(f"Invalid citation of type '{citation_type}' with content '{elem.text}': {e}") return citation diff --git a/lib/galaxy/managers/cloud.py b/lib/galaxy/managers/cloud.py index bdc2218065f..cc0733940a9 100644 --- a/lib/galaxy/managers/cloud.py +++ b/lib/galaxy/managers/cloud.py @@ -248,7 +248,7 @@ class CloudManager(sharable.SharableModelManager): try: bucket = connection.storage.buckets.get(bucket_name) if bucket is None: - raise RequestParameterInvalidException("The bucket `{}` not found.".format(bucket_name)) + raise RequestParameterInvalidException(f"The bucket `{bucket_name}` not found.") except Exception as e: raise ItemAccessibilityException("Could not get the bucket `{}`: {}".format(bucket_name, util.unicodify(e))) @@ -354,7 +354,7 @@ class CloudManager(sharable.SharableModelManager): incoming = (util.Params(args, sanitize=False)).__dict__ d2c = trans.app.toolbox.get_tool(SEND_TOOL, SEND_TOOL_VERSION) if not d2c: - log.debug("Failed to get the `send` tool per user `{}` request.".format(trans.user.id)) + log.debug(f"Failed to get the `send` tool per user `{trans.user.id}` request.") failed.append(json.dumps( { "object": object_label, diff --git a/lib/galaxy/managers/cloudauthzs.py b/lib/galaxy/managers/cloudauthzs.py index 12113f1871f..4f42d29698f 100644 --- a/lib/galaxy/managers/cloudauthzs.py +++ b/lib/galaxy/managers/cloudauthzs.py @@ -110,7 +110,7 @@ class CloudAuthzsDeserializer(base.ModelDeserializer): decoded_authn_id = self.app.security.decode_id(val) except Exception: log.debug("cannot decode authz_id `" + str(val) + "`") - raise MalformedId("Invalid `authz_id` {}!".format(val)) + raise MalformedId(f"Invalid `authz_id` {val}!") trans = context.get("trans") if trans is None: diff --git a/lib/galaxy/managers/collections_util.py b/lib/galaxy/managers/collections_util.py index af4a7315395..ae9da131f48 100644 --- a/lib/galaxy/managers/collections_util.py +++ b/lib/galaxy/managers/collections_util.py @@ -78,12 +78,12 @@ def get_collection(collection, name=""): hdas = [] if collection.has_subcollections: for element in collection.elements: - subnames, subhdas = get_collection_elements(element.child_collection, name="{}/{}".format(name, element.element_identifier)) + subnames, subhdas = get_collection_elements(element.child_collection, name=f"{name}/{element.element_identifier}") names.extend(subnames) hdas.extend(subhdas) else: for element in collection.elements: - names.append("{}/{}".format(name, element.element_identifier)) + names.append(f"{name}/{element.element_identifier}") hdas.append(element.dataset_instance) return names, hdas @@ -92,7 +92,7 @@ def get_collection_elements(collection, name=""): names = [] hdas = [] for element in collection.elements: - full_element_name = "{}/{}".format(name, element.element_identifier) + full_element_name = f"{name}/{element.element_identifier}" if element.is_collection: subnames, subhdas = get_collection(element.child_collection, name=full_element_name) names.extend(subnames) diff --git a/lib/galaxy/managers/jobs.py b/lib/galaxy/managers/jobs.py index 6efbf9f58d4..560be6ddcb4 100644 --- a/lib/galaxy/managers/jobs.py +++ b/lib/galaxy/managers/jobs.py @@ -38,7 +38,7 @@ def get_path_key(path_tuple): # we remove the last 2 items of the path tuple (values and list index) return path_key if path_key: - path_key = "{}{}{}".format(path_key, sep, p) + path_key = f"{path_key}{sep}{p}" else: path_key = p return path_key diff --git a/lib/galaxy/managers/library_datasets.py b/lib/galaxy/managers/library_datasets.py index 58e6b47f45a..44f45d9a791 100644 --- a/lib/galaxy/managers/library_datasets.py +++ b/lib/galaxy/managers/library_datasets.py @@ -123,7 +123,7 @@ class LibraryDatasetsManager(datasets.DatasetAssociationManager): continue if key in ('name'): if len(val) < MINIMUM_STRING_LENGTH: - raise RequestParameterInvalidException('{} must have at least length of {}'.format(key, MINIMUM_STRING_LENGTH)) + raise RequestParameterInvalidException(f'{key} must have at least length of {MINIMUM_STRING_LENGTH}') val = validation.validate_and_sanitize_basestring(key, val) validated_payload[key] = val if key in ('misc_info', 'message'): @@ -136,7 +136,7 @@ class LibraryDatasetsManager(datasets.DatasetAssociationManager): validated_payload[key] = val if key in ('genome_build'): if len(val) < MINIMUM_STRING_LENGTH: - raise RequestParameterInvalidException('{} must have at least length of {}'.format(key, MINIMUM_STRING_LENGTH)) + raise RequestParameterInvalidException(f'{key} must have at least length of {MINIMUM_STRING_LENGTH}') val = validation.validate_and_sanitize_basestring(key, val) validated_payload[key] = val if key in ('tags'): diff --git a/lib/galaxy/managers/markdown_parse.py b/lib/galaxy/managers/markdown_parse.py index 51243101851..299e25862f7 100644 --- a/lib/galaxy/managers/markdown_parse.py +++ b/lib/galaxy/managers/markdown_parse.py @@ -42,7 +42,7 @@ GALAXY_FLAVORED_MARKDOWN_CONTAINER_REGEX = r'(?P%s)' % "|".join(GALAX ARG_VAL_REGEX = r'''[\w_\-]+|\"[^\"]+\"|\'[^\']+\'''' FUNCTION_ARG = r'\s*\w+\s*=\s*(?:%s)\s*' % ARG_VAL_REGEX # embed commas between arguments -FUNCTION_MULTIPLE_ARGS = r'(?P{})(?P(?:,{})*)'.format(FUNCTION_ARG, FUNCTION_ARG) +FUNCTION_MULTIPLE_ARGS = fr'(?P{FUNCTION_ARG})(?P(?:,{FUNCTION_ARG})*)' FUNCTION_MULTIPLE_ARGS_PATTERN = re.compile(FUNCTION_MULTIPLE_ARGS) FUNCTION_CALL_LINE_TEMPLATE = r'\s*%s\s*\((?:' + FUNCTION_MULTIPLE_ARGS + r')?\)\s*' GALAXY_MARKDOWN_FUNCTION_CALL_LINE = re.compile(FUNCTION_CALL_LINE_TEMPLATE % GALAXY_FLAVORED_MARKDOWN_CONTAINER_REGEX) @@ -119,7 +119,7 @@ def validate_galaxy_markdown(galaxy_markdown, internal=True): if expecting_container_close_for: template = "Invalid line %d: %s" - msg = template % (last_line_no, "close of block for [{expected_for}] expected".format(expected_for=expecting_container_close_for)) + msg = template % (last_line_no, f"close of block for [{expecting_container_close_for}] expected") raise ValueError(msg) diff --git a/lib/galaxy/managers/markdown_util.py b/lib/galaxy/managers/markdown_util.py index 70b3e23608d..d493598c093 100644 --- a/lib/galaxy/managers/markdown_util.py +++ b/lib/galaxy/managers/markdown_util.py @@ -380,7 +380,7 @@ class ToBasicMarkdownDirectiveHandler(GalaxyInternalMarkdownDirectiveHandler): name = hda.name or '' with open(dataset.file_name, "rb") as f: base64_image_data = base64.b64encode(f.read()).decode("utf-8") - rval = ("![{}](data:image/png;base64,{})".format(name, base64_image_data), True) + rval = (f"![{name}](data:image/png;base64,{base64_image_data})", True) return rval def handle_dataset_peek(self, line, hda): @@ -423,7 +423,7 @@ class ToBasicMarkdownDirectiveHandler(GalaxyInternalMarkdownDirectiveHandler): walk_elements(element.child_collection, element_prefix + element.element_identifier + ":") else: for element in collection.elements: - markdown_wrapper[0] += "**Element:** {}{}\n\n".format(element_prefix, element.element_identifier) + markdown_wrapper[0] += f"**Element:** {element_prefix}{element.element_identifier}\n\n" markdown_wrapper[0] += self._display_dataset_content(element.hda, header="Element Contents") walk_elements(hdca.collection) markdown = '---\n%s\n---\n' % markdown_wrapper[0] @@ -450,7 +450,7 @@ class ToBasicMarkdownDirectiveHandler(GalaxyInternalMarkdownDirectiveHandler): markdown += "**%s**\n\n" % metric_plugin markdown += "| | |\n|---|--|\n" for title, value in metrics_for_plugin.items(): - markdown += "| {} | {} |\n".format(title, value) + markdown += f"| {title} | {value} |\n" return (markdown, True) def handle_job_parameters(self, line, job): @@ -677,7 +677,7 @@ history_dataset_collection_display(input={}) ref_object_type = "history_dataset" else: ref_object_type = "history_dataset_collection" - line = line.replace(target_match.group(), "{}_id={}".format(ref_object_type, ref_object.id)) + line = line.replace(target_match.group(), f"{ref_object_type}_id={ref_object.id}") return (line, False) workflow_markdown = _remap_galaxy_markdown_calls( diff --git a/lib/galaxy/managers/pages.py b/lib/galaxy/managers/pages.py index dd8fb30aba0..c7077005d95 100644 --- a/lib/galaxy/managers/pages.py +++ b/lib/galaxy/managers/pages.py @@ -7,9 +7,8 @@ from within Galaxy. """ import logging import re - -from six.moves.html_entities import name2codepoint -from six.moves.html_parser import HTMLParser +from html.entities import name2codepoint +from html.parser import HTMLParser from galaxy import exceptions, model from galaxy.managers import base, sharable @@ -310,11 +309,11 @@ class PageContentProcessor(HTMLParser): value = value.replace('>', '>').replace('<', '<').replace('"', '"') value = self.bare_ampersand.sub("&", value) uattrs.append((key, value)) - strattrs = ''.join(' {}="{}"'.format(k, v) for k, v in uattrs) + strattrs = ''.join(f' {k}="{v}"' for k, v in uattrs) if tag in self.elements_no_end_tag: - self.pieces.append('<{}{} />'.format(tag, strattrs)) + self.pieces.append(f'<{tag}{strattrs} />') else: - self.pieces.append('<{}{}>'.format(tag, strattrs)) + self.pieces.append(f'<{tag}{strattrs}>') def handle_endtag(self, tag): """ diff --git a/lib/galaxy/managers/workflows.py b/lib/galaxy/managers/workflows.py index 1c15d30d911..42af8ee81e0 100644 --- a/lib/galaxy/managers/workflows.py +++ b/lib/galaxy/managers/workflows.py @@ -1339,7 +1339,7 @@ class WorkflowContentsManager(UsesAnnotations): raise exceptions.MessageException(message) external_id = conn_dict['id'] if external_id not in steps_by_external_id: - raise KeyError("Failed to find external id {} in {}".format(external_id, steps_by_external_id.keys())) + raise KeyError(f"Failed to find external id {external_id} in {steps_by_external_id.keys()}") output_step = steps_by_external_id[external_id] output_name = conn_dict["output_name"] diff --git a/lib/galaxy/metadata/__init__.py b/lib/galaxy/metadata/__init__.py index 390a0ac780b..eace788a900 100644 --- a/lib/galaxy/metadata/__init__.py +++ b/lib/galaxy/metadata/__init__.py @@ -3,13 +3,12 @@ import abc import json import os +import pickle import shutil import tempfile from logging import getLogger from os.path import abspath -from six.moves import cPickle - import galaxy.model from galaxy.model import store from galaxy.model.metadata import FileParameter, MetadataTempFile @@ -94,7 +93,7 @@ class MetadataCollectionStrategy(metaclass=abc.ABCMeta): rstring = "Metadata results could not be read from '%s'" % filename_results_code if not rval: - log.debug('setting metadata externally failed for {} {}: {}'.format(dataset.__class__.__name__, dataset.id, rstring)) + log.debug(f'setting metadata externally failed for {dataset.__class__.__name__} {dataset.id}: {rstring}') return rval @@ -137,7 +136,7 @@ class PortableDirectoryMetadataGenerator(MetadataCollectionStrategy): key = name def _metadata_path(what): - return os.path.join(metadata_dir, "metadata_{}_{}".format(what, key)) + return os.path.join(metadata_dir, f"metadata_{what}_{key}") _initialize_metadata_inputs(dataset, _metadata_path, tmp_dir, kwds, real_metadata_object=real_metadata_object) @@ -343,7 +342,7 @@ class JobExternalOutputMetadataWrapper(MetadataCollectionStrategy): # is located differently, i.e. on a cluster node with a different filesystem structure def _metadata_path(what): - return abspath(tempfile.NamedTemporaryFile(dir=tmp_dir, prefix="metadata_{}_{}_".format(what, key)).name) + return abspath(tempfile.NamedTemporaryFile(dir=tmp_dir, prefix=f"metadata_{what}_{key}_").name) filename_in, filename_out, filename_results_code, filename_kwds, filename_override_metadata = _initialize_metadata_inputs(dataset, _metadata_path, tmp_dir, kwds) @@ -404,7 +403,7 @@ class JobExternalOutputMetadataWrapper(MetadataCollectionStrategy): try: os.remove(fname) except Exception as e: - log.debug('Failed to cleanup external metadata file ({}) for {}: {}'.format(key, dataset_key, e)) + log.debug(f'Failed to cleanup external metadata file ({key}) for {dataset_key}: {e}') def set_job_runner_external_pid(self, pid, sa_session): for metadata_files in sa_session.query(galaxy.model.Job).get(self.job_id).external_output_metadata: @@ -465,7 +464,7 @@ def _dump_dataset_instance_to(dataset_instance, file_path): # Touch also deferred column dataset_instance._metadata - cPickle.dump(dataset_instance, open(file_path, 'wb+')) + pickle.dump(dataset_instance, open(file_path, 'wb+')) def _get_filename_override(output_fnames, file_name): diff --git a/lib/galaxy/metadata/set_metadata.py b/lib/galaxy/metadata/set_metadata.py index 865a247c4f4..9fb3769e395 100644 --- a/lib/galaxy/metadata/set_metadata.py +++ b/lib/galaxy/metadata/set_metadata.py @@ -13,10 +13,10 @@ constructed automatically). import json import logging import os +import pickle import sys import traceback -from six.moves import cPickle from sqlalchemy.orm import clear_mappers import galaxy.model.mapping # need to load this before we unpickle, in order to setup properties assigned by the mappers @@ -182,7 +182,7 @@ def set_metadata_portable(): assert dataset is not None else: filename_in = os.path.join("metadata/metadata_in_%s" % output_name) - dataset = cPickle.load(open(filename_in, 'rb')) # load DatasetInstance + dataset = pickle.load(open(filename_in, 'rb')) # load DatasetInstance filename_kwds = os.path.join("metadata/metadata_kwds_%s" % output_name) filename_out = os.path.join("metadata/metadata_out_%s" % output_name) @@ -345,7 +345,7 @@ def set_metadata_legacy(): override_metadata = fields.pop(0) set_meta_kwds = stringify_dictionary_keys(json.load(open(filename_kwds))) # load kwds; need to ensure our keywords are not unicode try: - dataset = cPickle.load(open(filename_in, 'rb')) # load DatasetInstance + dataset = pickle.load(open(filename_in, 'rb')) # load DatasetInstance dataset.dataset.external_filename = dataset_filename_override store_by = "id" extra_files_dir_name = "dataset_%s_files" % getattr(dataset.dataset, store_by) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 2b10f722984..fcaf0ead81b 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -327,7 +327,7 @@ class JobLike: else: extra += "unflushed" - return "{}[{},tool_id={}]".format(self.__class__.__name__, extra, self.tool_id) + return f"{self.__class__.__name__}[{extra},tool_id={self.tool_id}]" @property def stdout(self): @@ -1306,7 +1306,7 @@ class Task(JobLike, RepresentById): Return an id tag suitable for identifying the task. This combines the task's job id and the task's own id. """ - return "{}_{}".format(self.job.id, self.id) + return f"{self.job.id}_{self.id}" def get_command_line(self): return self.command_line @@ -1564,7 +1564,7 @@ class JobExternalOutputMetadata(RepresentById): # uses a Dataset rather than an HDA or LDA, it's necessary to set up a # fake dataset association that provides the needed attributes for # preparing a job. -class FakeDatasetAssociation (object): +class FakeDatasetAssociation : fake_dataset_association = True def __init__(self, dataset=None): @@ -2841,7 +2841,7 @@ class DatasetInstance: """ # See if we can convert the dataset if target_ext not in self.get_converter_types(): - raise NoConverterException("Conversion from '{}' to '{}' not possible".format(self.extension, target_ext)) + raise NoConverterException(f"Conversion from '{self.extension}' to '{target_ext}' not possible") # See if converted dataset already exists, either in metadata in conversions. converted_dataset = self.get_metadata_dataset(target_ext) if converted_dataset: @@ -3421,7 +3421,7 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot @type_id.expression def type_id(cls): - return ((type_coerce(cls.content_type, types.Unicode) + u'-' + return ((type_coerce(cls.content_type, types.Unicode) + '-' + type_coerce(cls.id, types.Unicode)).label('type_id')) @@ -3866,7 +3866,7 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, RepresentById): ''').execution_options(autocommit=True) ret = object_session(self).execute(sql, {'library_dataset_id': ldda.library_dataset_id, 'ldda_id': ldda.id}) if ret.rowcount < 1: - log.warn('Attempt to updated parent folder times failed: {} records updated.'.format(ret.rowcount)) + log.warn(f'Attempt to updated parent folder times failed: {ret.rowcount} records updated.') class ExtendedMetadata(RepresentById): @@ -4170,7 +4170,7 @@ class DatasetCollection(Dictifiable, UsesAnnotations, RepresentById): for element in self.elements: if getattr(element, get_by_attribute) == key: return element - error_message = "Dataset collection has no {} with key {}.".format(get_by_attribute, key) + error_message = f"Dataset collection has no {get_by_attribute} with key {key}." raise KeyError(error_message) def copy(self, destination=None, element_destination=None, flush=True): @@ -4331,7 +4331,7 @@ class HistoryDatasetCollectionAssociation(DatasetCollectionInstance, @type_id.expression def type_id(cls): - return ((type_coerce(cls.content_type, types.Unicode) + u'-' + return ((type_coerce(cls.content_type, types.Unicode) + '-' + type_coerce(cls.id, types.Unicode)).label('type_id')) @property @@ -5295,7 +5295,7 @@ class WorkflowInvocation(UsesCreateAndUpdateTime, Dictifiable, RepresentById): if workflow_output: raise Exception("Failed to find workflow output named [%s], one was defined but none registered during execution." % label) else: - raise Exception("Failed to find workflow output named [{}], workflow doesn't define output by that name - valid names are {}.".format(label, self.workflow.workflow_output_labels)) + raise Exception(f"Failed to find workflow output named [{label}], workflow doesn't define output by that name - valid names are {self.workflow.workflow_output_labels}.") def get_input_object(self, label): for input_dataset_assoc in self.input_datasets: @@ -5458,7 +5458,7 @@ class WorkflowInvocation(UsesCreateAndUpdateTime, Dictifiable, RepresentById): extra += "id=%s" % safe_id else: extra += "unflushed" - return "{}[{}]".format(self.__class__.__name__, extra) + return f"{self.__class__.__name__}[{extra}]" class WorkflowInvocationToSubworkflowInvocationAssociation(Dictifiable, RepresentById): @@ -5974,7 +5974,7 @@ class CustosAuthnzToken(RepresentById): self.refresh_expiration_time = refresh_expiration_time -class CloudAuthz(object): +class CloudAuthz: def __init__(self, user_id, provider, config, authn_id, description=""): self.id = None self.user_id = user_id diff --git a/lib/galaxy/model/custom_types.py b/lib/galaxy/model/custom_types.py index 0419b9f2416..4370c93c300 100644 --- a/lib/galaxy/model/custom_types.py +++ b/lib/galaxy/model/custom_types.py @@ -315,7 +315,7 @@ class MetadataType(JSONType): sz = total_size(v) if sz > MAX_METADATA_VALUE_SIZE: del value[k] - log.warning('Refusing to bind metadata key {} due to size ({})'.format(k, sz)) + log.warning(f'Refusing to bind metadata key {k} due to size ({sz})') value = json_encoder.encode(value).encode() return value diff --git a/lib/galaxy/model/dataset_collections/type_description.py b/lib/galaxy/model/dataset_collections/type_description.py index ca330881f56..9b7e69da4f9 100644 --- a/lib/galaxy/model/dataset_collections/type_description.py +++ b/lib/galaxy/model/dataset_collections/type_description.py @@ -66,7 +66,7 @@ class CollectionTypeDescription: subcollection_type = subcollection_type.collection_type if not self.has_subcollections_of_type(subcollection_type): - raise ValueError("Cannot compute effective subcollection type of {} over {}".format(subcollection_type, self)) + raise ValueError(f"Cannot compute effective subcollection type of {subcollection_type} over {self}") return self.collection_type[:-(len(subcollection_type) + 1)] @@ -136,7 +136,7 @@ def map_over_collection_type(mapped_over_collection_type, target_collection_type if hasattr(target_collection_type, 'collection_type'): target_collection_type = target_collection_type.collection_type - return "{}:{}".format(mapped_over_collection_type, target_collection_type) + return f"{mapped_over_collection_type}:{target_collection_type}" COLLECTION_TYPE_DESCRIPTION_FACTORY = CollectionTypeDescriptionFactory() diff --git a/lib/galaxy/model/dataset_collections/types/paired.py b/lib/galaxy/model/dataset_collections/types/paired.py index 89095c7d36a..81469b2b79e 100644 --- a/lib/galaxy/model/dataset_collections/types/paired.py +++ b/lib/galaxy/model/dataset_collections/types/paired.py @@ -4,7 +4,7 @@ from ..types import BaseDatasetCollectionType FORWARD_IDENTIFIER = "forward" REVERSE_IDENTIFIER = "reverse" -INVALID_IDENTIFIERS_MESSAGE = "Paired instance must define '{}' and '{}' datasets .".format(FORWARD_IDENTIFIER, REVERSE_IDENTIFIER) +INVALID_IDENTIFIERS_MESSAGE = f"Paired instance must define '{FORWARD_IDENTIFIER}' and '{REVERSE_IDENTIFIER}' datasets ." class PairedDatasetCollectionType(BaseDatasetCollectionType): diff --git a/lib/galaxy/model/metadata.py b/lib/galaxy/model/metadata.py index d186c048661..66303586960 100644 --- a/lib/galaxy/model/metadata.py +++ b/lib/galaxy/model/metadata.py @@ -156,11 +156,11 @@ class MetadataCollection: def from_JSON_dict(self, filename=None, path_rewriter=None, json_dict=None): dataset = self.parent if filename is not None: - log.debug('loading metadata from file for: {} {}'.format(dataset.__class__.__name__, dataset.id)) + log.debug(f'loading metadata from file for: {dataset.__class__.__name__} {dataset.id}') with open(filename) as fh: JSONified_dict = json.load(fh) elif json_dict is not None: - log.debug('loading metadata from dict for: {} {}'.format(dataset.__class__.__name__, dataset.id)) + log.debug(f'loading metadata from dict for: {dataset.__class__.__name__} {dataset.id}') if isinstance(json_dict, str): JSONified_dict = json.loads(json_dict) elif isinstance(json_dict, dict): diff --git a/lib/galaxy/model/migrate/check.py b/lib/galaxy/model/migrate/check.py index afea334af03..d45974e2c73 100644 --- a/lib/galaxy/model/migrate/check.py +++ b/lib/galaxy/model/migrate/check.py @@ -136,7 +136,7 @@ def create_or_verify_database(url, galaxy_config_file, engine_options={}, app=No if db_schema.version > migrate_repository.versions.latest and allow_future_database: log.warning("WARNING: Database is from the future, but GALAXY_ALLOW_FUTURE_DATABASE is set, so Galaxy will continue to start.") else: - raise Exception("{}. {}{}".format(expect_msg, instructions, backup_msg)) + raise Exception(f"{expect_msg}. {instructions}{backup_msg}") else: log.info("At database version %d" % db_schema.version) @@ -150,7 +150,7 @@ def migrate_to_current_version(engine, schema): raise e for ver, change in changeset: nextver = ver + changeset.step - log.info('Migrating {} -> {}... '.format(ver, nextver)) + log.info(f'Migrating {ver} -> {nextver}... ') old_stdout = sys.stdout class FakeStdout: diff --git a/lib/galaxy/model/migrate/versions/0005_cleanup_datasets_fix.py b/lib/galaxy/model/migrate/versions/0005_cleanup_datasets_fix.py index 1ae979101de..f3d9f6c0c95 100644 --- a/lib/galaxy/model/migrate/versions/0005_cleanup_datasets_fix.py +++ b/lib/galaxy/model/migrate/versions/0005_cleanup_datasets_fix.py @@ -155,7 +155,7 @@ class Dataset: try: os.remove(self.data.file_name) except OSError as e: - log.critical('{} delete error {}'.format(self.__class__.__name__, e)) + log.critical(f'{self.__class__.__name__} delete error {e}') class DatasetInstance: @@ -750,7 +750,7 @@ def upgrade(migrate_engine): changed_associations += 1 # mark original Dataset as deleted and purged, it is no longer in use, but do not delete file_name contents dataset.deleted = True - dataset.external_filename = "Dataset was result of share before HDA, and has been replaced: {} mapped to Dataset {}".format(dataset.external_filename, guessed_dataset.id) + dataset.external_filename = f"Dataset was result of share before HDA, and has been replaced: {dataset.external_filename} mapped to Dataset {guessed_dataset.id}" dataset.purged = True # we don't really purge the file here, but we mark it as purged, since this dataset is now defunct context.flush() log.debug("%i items affected, and restored." % (changed_associations)) diff --git a/lib/galaxy/model/migrate/versions/0054_visualization_dbkey.py b/lib/galaxy/model/migrate/versions/0054_visualization_dbkey.py index 6e30b0fca5e..7500028c29a 100644 --- a/lib/galaxy/model/migrate/versions/0054_visualization_dbkey.py +++ b/lib/galaxy/model/migrate/versions/0054_visualization_dbkey.py @@ -47,8 +47,8 @@ def upgrade(migrate_engine): viz_rev_id = viz['viz_rev_id'] if viz[Visualization_revision_table.c.config]: dbkey = loads(viz[Visualization_revision_table.c.config]).get('dbkey', "").replace("'", "\\'") - migrate_engine.execute("UPDATE visualization_revision SET dbkey='{}' WHERE id={}".format(dbkey, viz_rev_id)) - migrate_engine.execute("UPDATE visualization SET dbkey='{}' WHERE id={}".format(dbkey, viz_id)) + migrate_engine.execute(f"UPDATE visualization_revision SET dbkey='{dbkey}' WHERE id={viz_rev_id}") + migrate_engine.execute(f"UPDATE visualization SET dbkey='{dbkey}' WHERE id={viz_id}") def downgrade(migrate_engine): diff --git a/lib/galaxy/model/migrate/versions/0069_rename_sequencer_form_type.py b/lib/galaxy/model/migrate/versions/0069_rename_sequencer_form_type.py index d5bf5e189bc..51175dd1441 100644 --- a/lib/galaxy/model/migrate/versions/0069_rename_sequencer_form_type.py +++ b/lib/galaxy/model/migrate/versions/0069_rename_sequencer_form_type.py @@ -16,7 +16,7 @@ def upgrade(migrate_engine): metadata.reflect() current_form_type = 'Sequencer Information Form' new_form_type = "External Service Information Form" - cmd = "update form_definition set type='{}' where type='{}'".format(new_form_type, current_form_type) + cmd = f"update form_definition set type='{new_form_type}' where type='{current_form_type}'" migrate_engine.execute(cmd) @@ -25,5 +25,5 @@ def downgrade(migrate_engine): metadata.reflect() new_form_type = 'Sequencer Information Form' current_form_type = "External Service Information Form" - cmd = "update form_definition set type='{}' where type='{}'".format(new_form_type, current_form_type) + cmd = f"update form_definition set type='{new_form_type}' where type='{current_form_type}'" migrate_engine.execute(cmd) diff --git a/lib/galaxy/model/migrate/versions/0122_grow_mysql_blobs.py b/lib/galaxy/model/migrate/versions/0122_grow_mysql_blobs.py index 5216dcd1e1f..2092764af8c 100644 --- a/lib/galaxy/model/migrate/versions/0122_grow_mysql_blobs.py +++ b/lib/galaxy/model/migrate/versions/0122_grow_mysql_blobs.py @@ -40,7 +40,7 @@ def upgrade(migrate_engine): return for (table, column) in BLOB_COLUMNS: - cmd = "ALTER TABLE {} MODIFY COLUMN {} MEDIUMBLOB;".format(table, column) + cmd = f"ALTER TABLE {table} MODIFY COLUMN {column} MEDIUMBLOB;" try: migrate_engine.execute(cmd) except Exception: diff --git a/lib/galaxy/model/migrate/versions/util.py b/lib/galaxy/model/migrate/versions/util.py index 273939147b2..52010ddc91d 100644 --- a/lib/galaxy/model/migrate/versions/util.py +++ b/lib/galaxy/model/migrate/versions/util.py @@ -31,7 +31,7 @@ def engine_true(migrate_engine): def nextval(migrate_engine, table, col='id'): if migrate_engine.name in ['postgres', 'postgresql']: - return "nextval('{}_{}_seq')".format(table, col) + return f"nextval('{table}_{col}_seq')" elif migrate_engine.name in ['mysql', 'sqlite']: return "null" else: diff --git a/lib/galaxy/model/orm/engine_factory.py b/lib/galaxy/model/orm/engine_factory.py index b07c91c7414..2fe968b2e28 100644 --- a/lib/galaxy/model/orm/engine_factory.py +++ b/lib/galaxy/model/orm/engine_factory.py @@ -45,7 +45,7 @@ def build_engine(url, engine_options, database_query_profiling_proxy=False, trac parameters, context, executemany): total = time.time() - conn.info['query_start_time'].pop(-1) if total > slow_query_log_threshold: - log.debug("Slow query: {:f}(s)\n{}\nParameters: {}".format(total, statement, parameters)) + log.debug(f"Slow query: {total:f}(s)\n{statement}\nParameters: {parameters}") if log_query_counts: try: QUERY_COUNT_LOCAL.times.append(total) @@ -55,7 +55,7 @@ def build_engine(url, engine_options, database_query_profiling_proxy=False, trac if thread_local_log is not None: try: if thread_local_log.log: - log.debug("Request query: {:f}(s)\n{}\nParameters: {}".format(total, statement, parameters)) + log.debug(f"Request query: {total:f}(s)\n{statement}\nParameters: {parameters}") except AttributeError: pass diff --git a/lib/galaxy/model/orm/scripts.py b/lib/galaxy/model/orm/scripts.py index ea63e673eed..e16fb608f82 100644 --- a/lib/galaxy/model/orm/scripts.py +++ b/lib/galaxy/model/orm/scripts.py @@ -85,13 +85,13 @@ def get_config(argv, use_argparse=True, cwd=None): Read sys.argv and parse out repository of migrations and database url. >>> import os - >>> from six.moves.configparser import SafeConfigParser + >>> from configparser import ConfigParser >>> from shutil import rmtree >>> from tempfile import mkdtemp >>> config_dir = mkdtemp() >>> os.makedirs(os.path.join(config_dir, 'config')) >>> def write_ini(path, property, value): - ... p = SafeConfigParser() + ... p = ConfigParser() ... p.add_section('app:main') ... p.set('app:main', property, value) ... with open(os.path.join(config_dir, 'config', path), 'w') as f: p.write(f) diff --git a/lib/galaxy/model/store/__init__.py b/lib/galaxy/model/store/__init__.py index d54102188e8..3d34574db1b 100644 --- a/lib/galaxy/model/store/__init__.py +++ b/lib/galaxy/model/store/__init__.py @@ -444,12 +444,12 @@ class ModelImportStore(metaclass=abc.ABCMeta): if hda_key in hdas_by_key: hda = hdas_by_key[hda_key] else: - raise KeyError("Failed to find exported hda with key [{}] of type [{}] in [{}]".format(hda_key, object_key, hdas_by_key)) + raise KeyError(f"Failed to find exported hda with key [{hda_key}] of type [{object_key}] in [{hdas_by_key}]") else: hda_id = hda_attrs["id"] hdas_by_id = object_import_tracker.hdas_by_id if hda_id not in hdas_by_id: - raise Exception("Failed to find HDA with id [{}] in [{}]".format(hda_id, hdas_by_id)) + raise Exception(f"Failed to find HDA with id [{hda_id}] in [{hdas_by_id}]") hda = hdas_by_id[hda_id] dce.hda = hda elif 'child_collection' in element_attrs: @@ -1441,7 +1441,7 @@ def get_export_dataset_filename(name, ext, hid): Builds a filename for a dataset using its name an extension. """ base = ''.join(c in FILENAME_VALID_CHARS and c or '_' for c in name) - return base + "_{}.{}".format(hid, ext) + return base + f"_{hid}.{ext}" def imported_store_for_metadata(directory, object_store=None): diff --git a/lib/galaxy/model/store/discover.py b/lib/galaxy/model/store/discover.py index 1c6d2c91c70..263aa8d794a 100644 --- a/lib/galaxy/model/store/discover.py +++ b/lib/galaxy/model/store/discover.py @@ -280,7 +280,7 @@ class ModelPersistenceContext(metaclass=abc.ABCMeta): # Associate new dataset with job element_identifier_str = ":".join(element_identifiers) - association_name = '__new_primary_file_{}|{}__'.format(name, element_identifier_str) + association_name = f'__new_primary_file_{name}|{element_identifier_str}__' self.add_output_dataset_association(association_name, dataset) self.flush() diff --git a/lib/galaxy/model/tool_shed_install/migrate/check.py b/lib/galaxy/model/tool_shed_install/migrate/check.py index 6df0a46d76f..d3265fbc37f 100644 --- a/lib/galaxy/model/tool_shed_install/migrate/check.py +++ b/lib/galaxy/model/tool_shed_install/migrate/check.py @@ -96,7 +96,7 @@ def migrate_to_current_version(engine, schema): changeset = schema.changeset(None) for ver, change in changeset: nextver = ver + changeset.step - log.info('Migrating {} -> {}... '.format(ver, nextver)) + log.info(f'Migrating {ver} -> {nextver}... ') old_stdout = sys.stdout class FakeStdout: diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index 904cacbdc8c..2166b2a8120 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -666,7 +666,7 @@ class NestedObjectStore(BaseObjectStore): try: # there are a few objects in python that don't have __class__ obj_id = self._get_object_id(obj) - return '{}({}={})'.format(obj.__class__.__name__, self.store_by, obj_id) + return f'{obj.__class__.__name__}({self.store_by}={obj_id})' except AttributeError: return str(obj) @@ -1004,7 +1004,7 @@ def build_object_store_from_config(config, fsmon=False, config_xml=None, config_ objectstore_class, objectstore_constructor_kwds = type_to_object_store_class(store, fsmon=fsmon) if objectstore_class is None: - log.error("Unrecognized object store definition: {}".format(store)) + log.error(f"Unrecognized object store definition: {store}") if from_object == 'xml': return objectstore_class.from_xml(config=config, config_xml=config_xml, **objectstore_constructor_kwds) diff --git a/lib/galaxy/objectstore/azure_blob.py b/lib/galaxy/objectstore/azure_blob.py index 63d054b7d8b..e77dedaa679 100644 --- a/lib/galaxy/objectstore/azure_blob.py +++ b/lib/galaxy/objectstore/azure_blob.py @@ -56,7 +56,7 @@ def parse_config_xml(config_xml): tag, attrs = 'extra_dir', ('type', 'path') extra_dirs = config_xml.findall(tag) if not extra_dirs: - msg = 'No {tag} element in XML tree'.format(tag=tag) + msg = f'No {tag} element in XML tree' log.error(msg) raise Exception(msg) extra_dirs = [{k: e.get(k) for k in attrs} for e in extra_dirs] diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index ed99326f199..3cd34fbbef7 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -117,7 +117,7 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): @staticmethod def _get_connection(provider, credentials): - log.debug("Configuring `{}` Connection".format(provider)) + log.debug(f"Configuring `{provider}` Connection") if provider == "aws": config = {"aws_access_key": credentials["access_key"], "aws_secret_key": credentials["secret_key"]} @@ -132,7 +132,7 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): config = {"gcp_service_creds_file": credentials["credentials_file"]} connection = CloudProviderFactory().create_provider(ProviderList.GCP, config) else: - raise Exception("Unsupported provider `{}`.".format(provider)) + raise Exception(f"Unsupported provider `{provider}`.") # Ideally it would be better to assert if the connection is # authorized to perform operations required by ObjectStore @@ -215,7 +215,7 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): elif provider == "google": cre = auth_element.get("credentials_file") if not os.path.isfile(cre): - msg = "The following file specified for GCP credentials not found: {}".format(cre) + msg = f"The following file specified for GCP credentials not found: {cre}" log.error(msg) raise OSError(msg) if cre is None: @@ -223,7 +223,7 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): config["auth"] = { "credentials_file": cre} else: - msg = "Unsupported provider `{}`.".format(provider) + msg = f"Unsupported provider `{provider}`." log.error(msg) raise Exception(msg) @@ -321,7 +321,7 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): except Exception: # These two generic exceptions will be replaced by specific exceptions # once proper exceptions are exposed by CloudBridge. - log.exception("Could not get bucket '{}'".format(bucket_name)) + log.exception(f"Could not get bucket '{bucket_name}'") raise Exception def _fix_permissions(self, rel_path): @@ -439,7 +439,7 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): log.debug("Parallel pulled key '%s' into cache to %s", rel_path, self._get_cache_path(rel_path)) ncores = multiprocessing.cpu_count() url = key.generate_url(7200) - ret_code = subprocess.call("axel -a -n {} '{}'".format(ncores, url)) + ret_code = subprocess.call(f"axel -a -n {ncores} '{url}'") if ret_code == 0: return True else: diff --git a/lib/galaxy/objectstore/irods.py b/lib/galaxy/objectstore/irods.py index 6c34673a580..e126912c3a4 100644 --- a/lib/galaxy/objectstore/irods.py +++ b/lib/galaxy/objectstore/irods.py @@ -35,7 +35,7 @@ log = logging.getLogger(__name__) def _config_xml_error(tag): - msg = 'No {tag} element in config XML tree'.format(tag=tag) + msg = f'No {tag} element in config XML tree' raise Exception(msg) @@ -136,7 +136,7 @@ def managed_session(host='localhost', port='1247', user='rods', password='rods', release_session(session) -class CloudConfigMixin(object): +class CloudConfigMixin: def _config_to_dict(self): return { diff --git a/lib/galaxy/objectstore/pithos.py b/lib/galaxy/objectstore/pithos.py index 17bb5fa576d..9cab7fffc02 100644 --- a/lib/galaxy/objectstore/pithos.py +++ b/lib/galaxy/objectstore/pithos.py @@ -62,13 +62,13 @@ def parse_config_xml(config_xml): tag, attrs = 'extra_dir', ('type', 'path') extra_dirs = config_xml.findall(tag) if not extra_dirs: - msg = 'No {tag} element in XML tree'.format(tag=tag) + msg = f'No {tag} element in XML tree' log.error(msg) raise Exception(msg) r['extra_dirs'] = [ {k: e.get(k) for k in attrs} for e in extra_dirs] if 'job_work' not in (d['type'] for d in r['extra_dirs']): - msg = 'No value for {}:type="job_work" in XML tree'.format(tag) + msg = f'No value for {tag}:type="job_work" in XML tree' log.error(msg) raise Exception(msg) except Exception: @@ -150,7 +150,7 @@ class PithosObjectStore(ConcreteObjectStore): # param extra_dir: should never be constructed from provided data but # just make sure there are no shenannigans afoot if extra_dir and extra_dir != os.path.normpath(extra_dir): - log.warning('extra_dir is not normalized: {}'.format(extra_dir)) + log.warning(f'extra_dir is not normalized: {extra_dir}') raise ObjectInvalid("The requested object is invalid") # ensure that any parent directory references in alt_name would not # result in a path not contained in the directory path constructed here @@ -178,7 +178,7 @@ class PithosObjectStore(ConcreteObjectStore): return os.path.join(base, rel_path) # Pithos+ folders are marked by having trailing '/' so add it now - rel_path = '{}/'.format(rel_path) + rel_path = f'{rel_path}/' if not dir_only: an = alt_name if alt_name else 'dataset_{}.dat'.format(self._get_object_id(obj)) @@ -339,7 +339,7 @@ class PithosObjectStore(ConcreteObjectStore): extra_dir = kwargs.get('extra_dir', False) if entire_dir and extra_dir: shutil.rmtree(cache_path) - log.debug('On Pithos: delete -r {path}/'.format(path=path)) + log.debug(f'On Pithos: delete -r {path}/') self.pithos.del_object(path, delimiter='/') return True else: @@ -430,8 +430,8 @@ class PithosObjectStore(ConcreteObjectStore): return self.pithos.publish_object(path) except ClientError as ce: log.exception( - 'Trouble generating URL for dataset "{}"'.format(path)) - log.exception('Kamaki: {}'.format(ce)) + f'Trouble generating URL for dataset "{path}"') + log.exception(f'Kamaki: {ce}') return None def _get_store_usage_percent(self): diff --git a/lib/galaxy/objectstore/s3.py b/lib/galaxy/objectstore/s3.py index 00582b52e22..1d7a74174e8 100644 --- a/lib/galaxy/objectstore/s3.py +++ b/lib/galaxy/objectstore/s3.py @@ -69,7 +69,7 @@ def parse_config_xml(config_xml): tag, attrs = 'extra_dir', ('type', 'path') extra_dirs = config_xml.findall(tag) if not extra_dirs: - msg = 'No {tag} element in XML tree'.format(tag=tag) + msg = f'No {tag} element in XML tree' log.error(msg) raise Exception(msg) extra_dirs = [{k: e.get(k) for k in attrs} for e in extra_dirs] diff --git a/lib/galaxy/openid/providers.py b/lib/galaxy/openid/providers.py index 7c6c282f2b8..5f00205518f 100644 --- a/lib/galaxy/openid/providers.py +++ b/lib/galaxy/openid/providers.py @@ -120,7 +120,7 @@ class OpenIDProviders: try: provider = OpenIDProvider.from_file(os.path.join('lib/galaxy/openid', elem.get('file'))) providers[provider.id] = provider - log.debug('Loaded OpenID provider: {} ({})'.format(provider.name, provider.id)) + log.debug(f'Loaded OpenID provider: {provider.name} ({provider.id})') except Exception as e: log.error('Failed to add OpenID provider: %s' % (e)) return cls(providers) diff --git a/lib/galaxy/queue_worker.py b/lib/galaxy/queue_worker.py index 3a9d98b965a..d13b1666c50 100644 --- a/lib/galaxy/queue_worker.py +++ b/lib/galaxy/queue_worker.py @@ -25,7 +25,6 @@ from kombu.mixins import ConsumerProducerMixin from kombu.pools import ( producers, ) -from six.moves import reload_module import galaxy.queues from galaxy import util @@ -45,7 +44,7 @@ def send_local_control_task(app, task, get_response=False, kwargs=None): log.info("Queuing {} task {} for {}.".format("sync" if get_response else "async", task, app.config.server_name)) payload = {'task': task, 'kwargs': kwargs} - routing_key = 'control.{}@{}'.format(app.config.server_name, socket.gethostname()) + routing_key = f'control.{app.config.server_name}@{socket.gethostname()}' control_task = ControlTask(app.queue_worker) return control_task.send_task(payload, routing_key, local=True, get_response=get_response) @@ -263,7 +262,7 @@ def reload_job_rules(app, **kwargs): if ((name == rules_module_name or name.startswith(rules_module_name + '.')) and ismodule(module)): log.debug("Reloading job rules module: %s", name) - reload_module(module) + importlib.reload(module) log.debug("Job rules reloaded %s", reload_timer) diff --git a/lib/galaxy/queues.py b/lib/galaxy/queues.py index 5b74c0e496d..f858b656d5f 100644 --- a/lib/galaxy/queues.py +++ b/lib/galaxy/queues.py @@ -31,7 +31,7 @@ def control_queues_from_config(config): galaxy process's config """ hostname = socket.gethostname() - process_name = "{server_name}@{hostname}".format(server_name=config.server_name, hostname=hostname) + process_name = f"{config.server_name}@{hostname}" exchange_queue = Queue("control.%s" % process_name, galaxy_exchange, routing_key='control.*') non_exchange_queue = Queue("control.%s" % process_name, routing_key='control.%s' % process_name) return exchange_queue, non_exchange_queue diff --git a/lib/galaxy/security/__init__.py b/lib/galaxy/security/__init__.py index eaf33b1a9b8..cfb538c8dcc 100644 --- a/lib/galaxy/security/__init__.py +++ b/lib/galaxy/security/__init__.py @@ -35,7 +35,7 @@ class RBACAgent: return list(self.permitted_actions.__dict__.values()) def get_item_actions(self, action, item): - raise Exception('No valid method of retrieving action ({}) for item {}.'.format(action, item)) + raise Exception(f'No valid method of retrieving action ({action}) for item {item}.') def guess_derived_permissions_for_datasets(self, datasets=[]): raise Exception("Unimplemented Method") diff --git a/lib/galaxy/selenium/cli.py b/lib/galaxy/selenium/cli.py index e46a6f3a32a..bf476c0be11 100644 --- a/lib/galaxy/selenium/cli.py +++ b/lib/galaxy/selenium/cli.py @@ -5,7 +5,7 @@ REMOTE_PORT_DESCRIPTION = "Selenium hub remote port to use (if remote driver in GALAXY_URL_DESCRIPTION = "URL of Galaxy instance to target." HEADLESS_DESCRIPTION = "Use local selenium headlessly (native in chrome, otherwise this requires pyvirtualdisplay)." -from six.moves.urllib.parse import urljoin +from urllib.parse import urljoin from .driver_factory import ( get_local_driver, diff --git a/lib/galaxy/selenium/components.py b/lib/galaxy/selenium/components.py index 4feddecdf97..3d197626c04 100644 --- a/lib/galaxy/selenium/components.py +++ b/lib/galaxy/selenium/components.py @@ -96,7 +96,7 @@ class SelectorTemplate(Target): if name in self._children: return self._children[name](**{"_": self.selector}) else: - raise AttributeError("Could not find child [{}] in {}".format(name, self._children)) + raise AttributeError(f"Could not find child [{name}] in {self._children}") __getitem__ = __getattr__ diff --git a/lib/galaxy/selenium/driver_factory.py b/lib/galaxy/selenium/driver_factory.py index 07f6d79a2ba..9a26d84b7d3 100644 --- a/lib/galaxy/selenium/driver_factory.py +++ b/lib/galaxy/selenium/driver_factory.py @@ -69,7 +69,7 @@ def get_remote_driver( assert browser in ["CHROME", "EDGE", "ANDROID", "FIREFOX", "INTERNETEXPLORER", "IPAD", "IPHONE", "OPERA", "PHANTOMJS", "SAFARI"] desired_capabilities = getattr(DesiredCapabilities, browser) desired_capabilities["loggingPrefs"] = LOGGING_PREFS - executor = 'http://{}:{}/wd/hub'.format(host, port) + executor = f'http://{host}:{port}/wd/hub' driver = webdriver.Remote( command_executor=executor, desired_capabilities=desired_capabilities, diff --git a/lib/galaxy/selenium/navigates_galaxy.py b/lib/galaxy/selenium/navigates_galaxy.py index f461a851542..56ea85ab46c 100644 --- a/lib/galaxy/selenium/navigates_galaxy.py +++ b/lib/galaxy/selenium/navigates_galaxy.py @@ -162,7 +162,7 @@ class NavigatesGalaxy(HasDriver): @contextlib.contextmanager def local_storage(self, key, value): - self.driver.execute_script('''window.localStorage.setItem("{}", {});'''.format(key, value)) + self.driver.execute_script(f'''window.localStorage.setItem("{key}", {value});''') try: yield finally: @@ -223,9 +223,9 @@ class NavigatesGalaxy(HasDriver): if history_id not in [h['id'] for h in histories]: return {} if datasets_only: - endpoint = 'histories/{}/contents?view={}'.format(history_id, view) + endpoint = f'histories/{history_id}/contents?view={view}' else: - endpoint = 'histories/{}?view={}'.format(history_id, view) + endpoint = f'histories/{history_id}?view={view}' return self.api_get(endpoint) def current_history(self): @@ -1255,7 +1255,7 @@ class NavigatesGalaxy(HasDriver): try: history_item = [d for d in contents if d["hid"] == hid][0] except IndexError: - raise Exception("Could not find history item with hid [{}] in contents [{}]".format(hid, contents)) + raise Exception(f"Could not find history item with hid [{hid}] in contents [{contents}]") history_item_selector = "#{}-{}".format(history_item["history_content_type"], history_item["id"]) if wait: self.wait_for_selector_visible(history_item_selector) @@ -1419,7 +1419,7 @@ class NavigatesGalaxy(HasDriver): if hasattr(expected, "text"): expected = expected.text text = self.get_tooltip_text(element, sleep=sleep, click_away=click_away) - assert text == expected, "Tooltip text [{}] was not expected text [{}].".format(text, expected) + assert text == expected, f"Tooltip text [{text}] was not expected text [{expected}]." def assert_error_message(self, contains=None): element = self.components._.messages["error"] @@ -1435,7 +1435,7 @@ class NavigatesGalaxy(HasDriver): if contains is not None: text = element.text if contains not in text: - message = "Text [{}] expected inside of [{}] but not found.".format(contains, text) + message = f"Text [{contains}] expected inside of [{text}] but not found." raise AssertionError(message) def assert_no_error_message(self): diff --git a/lib/galaxy/selenium/sizzle.py b/lib/galaxy/selenium/sizzle.py index d6df634048e..a392a954916 100644 --- a/lib/galaxy/selenium/sizzle.py +++ b/lib/galaxy/selenium/sizzle.py @@ -75,7 +75,7 @@ def find_element_by_sizzle(driver, sizzle_selector): return elements[0] else: raise NoSuchElementException( - "Unable to locate element by Sizzle: {selector}".format(selector=sizzle_selector) + f"Unable to locate element by Sizzle: {sizzle_selector}" ) @@ -110,7 +110,7 @@ def _inject_sizzle(driver, sizzle_url, timeout): driver.execute_script(script) wait = WebDriverWait(driver, timeout) wait.until(lambda d: _is_sizzle_loaded(d), - "Can't inject Sizzle in {timeout} seconds".format(timeout=timeout)) + f"Can't inject Sizzle in {timeout} seconds") def _is_sizzle_loaded(driver): diff --git a/lib/galaxy/tool_shed/galaxy_install/install_manager.py b/lib/galaxy/tool_shed/galaxy_install/install_manager.py index 3e76e3583aa..307c00f6731 100644 --- a/lib/galaxy/tool_shed/galaxy_install/install_manager.py +++ b/lib/galaxy/tool_shed/galaxy_install/install_manager.py @@ -658,7 +658,7 @@ class InstallRepositoryManager: for tool_guid in tool_panel_section_mapping: if tool_panel_section_mapping[tool_guid]['action'] == 'create': new_tool_panel_section_name = tool_panel_section_mapping[tool_guid]['tool_panel_section'] - log.debug('Creating tool panel section "{}" for tool {}'.format(new_tool_panel_section_name, tool_guid)) + log.debug(f'Creating tool panel section "{new_tool_panel_section_name}" for tool {tool_guid}') self.tpm.handle_tool_panel_section(self.app.toolbox, None, tool_panel_section_mapping[tool_guid]['tool_panel_section']) encoded_repository_ids = [self.app.security.encode_id(tsr.id) for tsr in created_or_updated_tool_shed_repositories] new_kwd = dict(includes_tools=includes_tools, diff --git a/lib/galaxy/tool_shed/galaxy_install/installed_repository_manager.py b/lib/galaxy/tool_shed/galaxy_install/installed_repository_manager.py index db4021bd988..0ff54fdf13e 100644 --- a/lib/galaxy/tool_shed/galaxy_install/installed_repository_manager.py +++ b/lib/galaxy/tool_shed/galaxy_install/installed_repository_manager.py @@ -161,14 +161,14 @@ class InstalledRepositoryManager: repository_dependency_tups = self.get_repository_dependency_tups_for_installed_repository(repository, status=status) # Add an entry to self.installed_repository_dependencies_of_installed_repositories. if repository_tup not in self.installed_repository_dependencies_of_installed_repositories: - debug_msg = "Adding an entry for revision {} of repository {} owned by {} ".format(installed_changeset_revision, name, owner) + debug_msg = f"Adding an entry for revision {installed_changeset_revision} of repository {name} owned by {owner} " debug_msg += "to installed_repository_dependencies_of_installed_repositories." log.debug(debug_msg) self.installed_repository_dependencies_of_installed_repositories[repository_tup] = repository_dependency_tups # Use the repository_dependency_tups to add entries to the reverse dictionary # self.installed_dependent_repositories_of_installed_repositories. for required_repository_tup in repository_dependency_tups: - debug_msg = "Appending revision {} of repository {} owned by {} ".format(installed_changeset_revision, name, owner) + debug_msg = f"Appending revision {installed_changeset_revision} of repository {name} owned by {owner} " debug_msg += "to all dependent repositories in installed_dependent_repositories_of_installed_repositories." log.debug(debug_msg) if required_repository_tup in self.installed_dependent_repositories_of_installed_repositories: @@ -181,7 +181,7 @@ class InstalledRepositoryManager: tool_dependency_tup = self.get_tool_dependency_tuple_for_installed_repository_manager(tool_dependency) if tool_dependency_tup not in self.installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies: tool_shed_repository_id, name, version, type = tool_dependency_tup - debug_msg = "Adding an entry for version {} of {} {} ".format(version, type, name) + debug_msg = f"Adding an entry for version {version} of {type} {name} " debug_msg += "to installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies." log.debug(debug_msg) status = self.install_model.ToolDependency.installation_status.INSTALLED @@ -195,7 +195,7 @@ class InstalledRepositoryManager: repository_tup = self.get_repository_tuple_for_installed_repository_manager(repository) if repository_tup not in self.installed_tool_dependencies_of_installed_repositories: tool_shed, name, owner, installed_changeset_revision = repository_tup - debug_msg = "Adding an entry for revision {} of repository {} owned by {} ".format(installed_changeset_revision, name, owner) + debug_msg = f"Adding an entry for revision {installed_changeset_revision} of repository {name} owned by {owner} " debug_msg += "to installed_tool_dependencies_of_installed_repositories." log.debug(debug_msg) installed_tool_dependency_tups = [] @@ -210,7 +210,7 @@ class InstalledRepositoryManager: repository_tup = self.get_repository_tuple_for_installed_repository_manager(repository) if repository_tup not in self.repository_dependencies_of_installed_repositories: tool_shed, name, owner, installed_changeset_revision = repository_tup - debug_msg = "Adding an entry for revision {} of repository {} owned by {} ".format(installed_changeset_revision, name, owner) + debug_msg = f"Adding an entry for revision {installed_changeset_revision} of repository {name} owned by {owner} " debug_msg += "to repository_dependencies_of_installed_repositories." log.debug(debug_msg) repository_dependency_tups = self.get_repository_dependency_tups_for_installed_repository(repository, status=None) @@ -221,7 +221,7 @@ class InstalledRepositoryManager: tool_dependency_tup = self.get_tool_dependency_tuple_for_installed_repository_manager(tool_dependency) if tool_dependency_tup not in self.runtime_tool_dependencies_of_installed_tool_dependencies: tool_shed_repository_id, name, version, type = tool_dependency_tup - debug_msg = "Adding an entry for version {} of {} {} ".format(version, type, name) + debug_msg = f"Adding an entry for version {version} of {type} {name} " debug_msg += "to runtime_tool_dependencies_of_installed_tool_dependencies." log.debug(debug_msg) runtime_dependent_tool_dependency_tups = self.get_runtime_dependent_tool_dependency_tuples(tool_dependency, @@ -234,7 +234,7 @@ class InstalledRepositoryManager: repository_tup = self.get_repository_tuple_for_installed_repository_manager(repository) if repository_tup not in self.tool_dependencies_of_installed_repositories: tool_shed, name, owner, installed_changeset_revision = repository_tup - debug_msg = "Adding an entry for revision {} of repository {} owned by {} ".format(installed_changeset_revision, name, owner) + debug_msg = f"Adding an entry for revision {installed_changeset_revision} of repository {name} owned by {owner} " debug_msg += "to tool_dependencies_of_installed_repositories." log.debug(debug_msg) tool_dependency_tups = [] @@ -730,7 +730,7 @@ class InstalledRepositoryManager: for tool_dependency in tool_dependencies_to_uninstall: uninstalled, error_message = tool_dependency_util.remove_tool_dependency(self.app, tool_dependency) if error_message: - errors = '{} {}'.format(errors, error_message) + errors = f'{errors} {error_message}' repository.deleted = True if remove_from_disk: repository.status = self.app.install_model.ToolShedRepository.installation_status.UNINSTALLED @@ -878,7 +878,7 @@ class InstalledRepositoryManager: altered_installed_dependent_repositories_of_installed_repositories # Remove this repository's entry from self.installed_repository_dependencies_of_installed_repositories. if repository_tup in self.installed_repository_dependencies_of_installed_repositories: - debug_msg = "Removing entry for revision {} of repository {} owned by {} ".format(installed_changeset_revision, name, owner) + debug_msg = f"Removing entry for revision {installed_changeset_revision} of repository {name} owned by {owner} " debug_msg += "from installed_repository_dependencies_of_installed_repositories." log.debug(debug_msg) del self.installed_repository_dependencies_of_installed_repositories[repository_tup] @@ -888,7 +888,7 @@ class InstalledRepositoryManager: tool_dependency_tup = self.get_tool_dependency_tuple_for_installed_repository_manager(tool_dependency) if tool_dependency_tup in self.installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies: tool_shed_repository_id, name, version, type = tool_dependency_tup - debug_msg = "Removing entry for version {} of {} {} ".format(version, type, name) + debug_msg = f"Removing entry for version {version} of {type} {name} " debug_msg += "from installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies." log.debug(debug_msg) del self.installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies[tool_dependency_tup] @@ -898,7 +898,7 @@ class InstalledRepositoryManager: repository_tup = self.get_repository_tuple_for_installed_repository_manager(repository) if repository_tup in self.installed_tool_dependencies_of_installed_repositories: tool_shed, name, owner, installed_changeset_revision = repository_tup - debug_msg = "Removing entry for revision {} of repository {} owned by {} ".format(installed_changeset_revision, name, owner) + debug_msg = f"Removing entry for revision {installed_changeset_revision} of repository {name} owned by {owner} " debug_msg += "from installed_tool_dependencies_of_installed_repositories." log.debug(debug_msg) del self.installed_tool_dependencies_of_installed_repositories[repository_tup] @@ -908,7 +908,7 @@ class InstalledRepositoryManager: repository_tup = self.get_repository_tuple_for_installed_repository_manager(repository) if repository_tup in self.repository_dependencies_of_installed_repositories: tool_shed, name, owner, installed_changeset_revision = repository_tup - debug_msg = "Removing entry for revision {} of repository {} owned by {} ".format(installed_changeset_revision, name, owner) + debug_msg = f"Removing entry for revision {installed_changeset_revision} of repository {name} owned by {owner} " debug_msg += "from repository_dependencies_of_installed_repositories." log.debug(debug_msg) del self.repository_dependencies_of_installed_repositories[repository_tup] diff --git a/lib/galaxy/tool_shed/galaxy_install/migrate/check.py b/lib/galaxy/tool_shed/galaxy_install/migrate/check.py index 5ffe9fff6d5..e7c654da3df 100644 --- a/lib/galaxy/tool_shed/galaxy_install/migrate/check.py +++ b/lib/galaxy/tool_shed/galaxy_install/migrate/check.py @@ -78,7 +78,7 @@ def verify_tools(app, url, galaxy_config_file=None, engine_options={}): msg += "automatically installed from the Galaxy tool shed at http://toolshed.g2.bx.psu.edu.\n\n" msg += "To skip this process, attempt to start your Galaxy server again (e.g., sh run.sh or whatever you use). If you do this,\n" msg += "be aware that these tools will no longer be available in your Galaxy tool panel, and entries for each of them should\n" - msg += "be removed from your file{} named {}.\n\n".format(plural, tool_panel_config_file_names) + msg += f"be removed from your file{plural} named {tool_panel_config_file_names}.\n\n" msg += "CRITICAL NOTE IF YOU PLAN TO INSTALL\n" msg += "The location in which the tool repositories will be installed is the value of the 'tool_path' attribute in the \n" msg += 'tag of the file named ./migrated_tool_conf.xml (i.e., ). The default location\n' @@ -126,7 +126,7 @@ def verify_tools(app, url, galaxy_config_file=None, engine_options={}): msg += "After the installation process finishes, you can start your Galaxy server. As part of this installation process,\n" msg += "entries for each of the following tool config files will be added to the file named ./migrated_tool_conf.xml, so these\n" msg += "tools will continue to be loaded into your tool panel. Because of this, existing entries for these tools have been\n" - msg += "removed from your file{} named {}.\n\n".format(plural, tool_panel_config_file_names) + msg += f"removed from your file{plural} named {tool_panel_config_file_names}.\n\n" for missing_tool_config, tool_dependencies in missing_tool_configs_dict.items(): msg += "%s\n" % missing_tool_config msg += "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n" @@ -142,7 +142,7 @@ def migrate_to_current_version(engine, schema): changeset = schema.changeset(None) for ver, change in changeset: nextver = ver + changeset.step - log.info('Installing tools from version {} -> {}... '.format(ver, nextver)) + log.info(f'Installing tools from version {ver} -> {nextver}... ') old_stdout = sys.stdout class FakeStdout: diff --git a/lib/galaxy/tool_shed/galaxy_install/migrate/common.py b/lib/galaxy/tool_shed/galaxy_install/migrate/common.py index 2875d0503b1..32644c4021a 100644 --- a/lib/galaxy/tool_shed/galaxy_install/migrate/common.py +++ b/lib/galaxy/tool_shed/galaxy_install/migrate/common.py @@ -1,8 +1,7 @@ +import configparser import os import sys -from six.moves import configparser - import galaxy.config from galaxy.tool_shed.galaxy_install import ( installed_repository_manager, diff --git a/lib/galaxy/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py b/lib/galaxy/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py index 23effb462f7..0a45b6d6fd5 100644 --- a/lib/galaxy/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py +++ b/lib/galaxy/tool_shed/galaxy_install/repository_dependencies/repository_dependency_manager.py @@ -5,10 +5,15 @@ into Galaxy from the Tool Shed. import json import logging import os - -from six.moves.urllib.error import HTTPError -from six.moves.urllib.parse import urlencode, urlparse -from six.moves.urllib.request import Request, urlopen +from urllib.error import HTTPError +from urllib.parse import ( + urlencode, + urlparse, +) +from urllib.request import ( + Request, + urlopen, +) from galaxy.tool_shed.galaxy_install.tools import tool_panel_manager from galaxy.tool_shed.util import repository_util @@ -43,7 +48,7 @@ class RepositoryDependencyInstallManager: install_model = self.app.install_model log.debug("Building repository dependency relationships...") for repo_info_dict in repo_info_dicts: - for name, repo_info_tuple in repo_info_dict.items(): + for repo_info_tuple in repo_info_dict.values(): description, \ repository_clone_url, \ changeset_revision, \ diff --git a/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/env_file_builder.py b/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/env_file_builder.py index cdeab2a1402..4b75b66e20d 100644 --- a/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/env_file_builder.py +++ b/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/env_file_builder.py @@ -26,14 +26,14 @@ class EnvFileBuilder: if env_var_action in ['prepend_to', 'set_to', 'append_to']: env_var_name = env_var_dict['name'] if env_var_action == 'prepend_to': - changed_value = '{}:${}'.format(env_var_value, env_var_name) + changed_value = f'{env_var_value}:${env_var_name}' elif env_var_action == 'set_to': changed_value = '%s' % env_var_value elif env_var_action == 'append_to': - changed_value = '${}:{}'.format(env_var_name, env_var_value) - line = "{}={}; export {}".format(env_var_name, changed_value, env_var_name) + changed_value = f'${env_var_name}:{env_var_value}' + line = f"{env_var_name}={changed_value}; export {env_var_name}" elif env_var_action == "source": - line = "if [ -f {} ] ; then . {} ; fi".format(env_var_value, env_var_value) + line = f"if [ -f {env_var_value} ] ; then . {env_var_value} ; fi" else: raise Exception("Unknown shell file action %s" % env_var_action) env_shell_file_path = os.path.join(install_dir, 'env.sh') diff --git a/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py b/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py index a8705471af3..626be903c31 100644 --- a/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py +++ b/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/install_environment.py @@ -1,5 +1,6 @@ import logging import os +import queue import shutil import subprocess import tempfile @@ -10,7 +11,6 @@ from contextlib import contextmanager # TODO: eliminate the use of fabric here. from fabric import state from fabric.operations import _AttributeString -from six.moves import queue from galaxy.tool_shed.galaxy_install.tool_dependencies.recipe import asynchronous_reader from galaxy.tool_shed.util.basic_util import ( diff --git a/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py b/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py index 8d5b9538805..b56aa617052 100644 --- a/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py +++ b/lib/galaxy/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py @@ -61,7 +61,7 @@ class Download: expected = download_url.split('#sha256#')[1].lower() if downloaded_checksum != expected: - raise Exception('Given sha256 checksum does not match with the one from the downloaded file ({} != {}).'.format(downloaded_checksum, expected)) + raise Exception(f'Given sha256 checksum does not match with the one from the downloaded file ({downloaded_checksum} != {expected}).') if 'md5sum' in checksums or '#md5#' in download_url or '#md5=' in download_url: downloaded_checksum = hashlib.md5(open(file_path, 'rb').read()).hexdigest().lower() @@ -73,7 +73,7 @@ class Download: expected = re.split('#md5[#=]', download_url)[1].lower() if downloaded_checksum != expected: - raise Exception('Given md5 checksum does not match with the one from the downloaded file ({} != {}).'.format(downloaded_checksum, expected)) + raise Exception(f'Given md5 checksum does not match with the one from the downloaded file ({downloaded_checksum} != {expected}).') if extract: if tarfile.is_tarfile(file_path) or (zipfile.is_zipfile(file_path) and not file_path.endswith('.jar')): @@ -1596,7 +1596,7 @@ class SetupVirtualEnv(Download, RecipeStep): venv_directory = os.path.join(install_environment.install_dir, "venv") python_cmd = action_dict['python'] # TODO: Consider making --no-site-packages optional. - setup_command = "{} {}/virtualenv.py --no-site-packages '{}'".format(python_cmd, venv_src_directory, venv_directory) + setup_command = f"{python_cmd} {venv_src_directory}/virtualenv.py --no-site-packages '{venv_directory}'" # POSIXLY_CORRECT forces shell commands . and source to have the same # and well defined behavior in bash/zsh. activate_command = "POSIXLY_CORRECT=1; . %s" % os.path.join(venv_directory, "bin", "activate") @@ -1621,8 +1621,8 @@ class SetupVirtualEnv(Download, RecipeStep): if not install_command: install_command = line_install_command else: - install_command = "{} && {}".format(install_command, line_install_command) - full_setup_command = "{}; {}; {}".format(setup_command, activate_command, install_command) + install_command = f"{install_command} && {line_install_command}" + full_setup_command = f"{setup_command}; {activate_command}; {install_command}" return_code = install_environment.handle_command(tool_dependency=tool_dependency, cmd=full_setup_command, return_output=False, diff --git a/lib/galaxy/tool_shed/galaxy_install/tools/data_manager.py b/lib/galaxy/tool_shed/galaxy_install/tools/data_manager.py index 23d86f2ed99..2a6552eea31 100644 --- a/lib/galaxy/tool_shed/galaxy_install/tools/data_manager.py +++ b/lib/galaxy/tool_shed/galaxy_install/tools/data_manager.py @@ -97,12 +97,12 @@ class DataManagerHandler: continue guid = data_manager_dict.get('guid', None) if guid is None: - log.error("Data manager guid '{}' is not set in metadata for '{}'.".format(guid, data_manager_id)) + log.error(f"Data manager guid '{guid}' is not set in metadata for '{data_manager_id}'.") continue elem.set('guid', guid) tool_guid = data_manager_dict.get('tool_guid', None) if tool_guid is None: - log.error("Data manager tool guid '{}' is not set in metadata for '{}'.".format(tool_guid, data_manager_id)) + log.error(f"Data manager tool guid '{tool_guid}' is not set in metadata for '{data_manager_id}'.") continue tool_dict = repository_tools_by_guid.get(tool_guid, None) if tool_dict is None: diff --git a/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py b/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py index ce227c67937..5e438bb6b2f 100644 --- a/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -101,8 +101,8 @@ class ToolPanelManager: value of config_filename. """ try: - tool_cache_data_dir = ' tool_cache_data_dir="{}"'.format(tool_cache_data_dir) if tool_cache_data_dir else '' - root = parse_xml_string('\n'.format(tool_path, tool_cache_data_dir)) + tool_cache_data_dir = f' tool_cache_data_dir="{tool_cache_data_dir}"' if tool_cache_data_dir else '' + root = parse_xml_string(f'\n') for elem in config_elems: root.append(elem) with RenamedTemporaryFile(config_filename, mode='w') as fh: diff --git a/lib/galaxy/tool_shed/metadata/metadata_generator.py b/lib/galaxy/tool_shed/metadata/metadata_generator.py index 6047447108c..e85ac511589 100644 --- a/lib/galaxy/tool_shed/metadata/metadata_generator.py +++ b/lib/galaxy/tool_shed/metadata/metadata_generator.py @@ -296,7 +296,7 @@ class MetadataGenerator: def generate_guid_for_object(self, guid_type, obj_id, version): tmp_url = remove_protocol_and_user_from_clone_url(self.repository_clone_url) - return '{}/{}/{}/{}'.format(tmp_url, guid_type, obj_id, version) + return f'{tmp_url}/{guid_type}/{obj_id}/{version}' def generate_metadata_for_changeset_revision(self): """ @@ -565,7 +565,7 @@ class MetadataGenerator: self.handle_repository_elem(repository_elem=sub_action_elem, only_if_compiling_contained_td=True) if requirements_dict: - dependency_key = '{}/{}'.format(package_name, package_version) + dependency_key = f'{package_name}/{package_version}' if repository_dependency_is_valid: valid_tool_dependencies_dict[dependency_key] = requirements_dict else: @@ -766,7 +766,7 @@ class MetadataGenerator: only_if_compiling_contained_td, message) invalid_repository_dependency_tups.append(repository_dependency_tup) - error_messages.append('{} {}'.format(error_message, message)) + error_messages.append(f'{error_message} {message}') elif elem.tag == 'set_environment': rvs.valid_tool_dependencies_dict = \ self.generate_environment_dependency_metadata(elem, rvs.valid_tool_dependencies_dict) diff --git a/lib/galaxy/tool_shed/util/container_util.py b/lib/galaxy/tool_shed/util/container_util.py index e2401335095..2960a9721bf 100644 --- a/lib/galaxy/tool_shed/util/container_util.py +++ b/lib/galaxy/tool_shed/util/container_util.py @@ -64,6 +64,6 @@ def print_folders(pad, folder): pad_str += ' ' print('{}id: {} key: {}'.format(pad_str, str(folder.id), folder.key)) for repository_dependency in folder.repository_dependencies: - print(' {}{}'.format(pad_str, repository_dependency.listify)) + print(f' {pad_str}{repository_dependency.listify}') for sub_folder in folder.folders: print_folders(pad + 5, sub_folder) diff --git a/lib/galaxy/tool_shed/util/repository_util.py b/lib/galaxy/tool_shed/util/repository_util.py index 87ae102dbb9..3adc5e2fbdd 100644 --- a/lib/galaxy/tool_shed/util/repository_util.py +++ b/lib/galaxy/tool_shed/util/repository_util.py @@ -2,9 +2,9 @@ import logging import os import re import shutil +from urllib.error import HTTPError from markupsafe import escape -from six.moves.urllib.error import HTTPError from sqlalchemy import ( and_, false, @@ -173,7 +173,7 @@ def get_absolute_path_to_file_in_repository(repo_files_dir, file_name): """Return the absolute path to a specified disk file contained in a repository.""" stripped_file_name = basic_util.strip_path(file_name) file_path = None - for root, dirs, files in os.walk(repo_files_dir): + for root, _, files in os.walk(repo_files_dir): if root.find('.hg') < 0: for name in files: if name == stripped_file_name: diff --git a/lib/galaxy/tool_shed/util/shed_util_common.py b/lib/galaxy/tool_shed/util/shed_util_common.py index d70187c9ade..6531b1080c0 100644 --- a/lib/galaxy/tool_shed/util/shed_util_common.py +++ b/lib/galaxy/tool_shed/util/shed_util_common.py @@ -88,7 +88,7 @@ def generate_tool_guid(repository_clone_url, tool): /repos//// """ tmp_url = common_util.remove_protocol_and_user_from_clone_url(repository_clone_url) - return '{}/{}/{}'.format(tmp_url, tool.id, tool.version) + return f'{tmp_url}/{tool.id}/{tool.version}' def get_ctx_rev(app, tool_shed_url, name, owner, changeset_revision): diff --git a/lib/galaxy/tool_shed/util/tool_util.py b/lib/galaxy/tool_shed/util/tool_util.py index dd6724ed0ba..c41ba1edf14 100644 --- a/lib/galaxy/tool_shed/util/tool_util.py +++ b/lib/galaxy/tool_shed/util/tool_util.py @@ -124,7 +124,7 @@ def generate_message_for_invalid_tools(app, invalid_file_tups, repository, metad correction_msg = exception_msg else: correction_msg = exception_msg.replace('
    ', new_line).replace('', bold_start).replace('', bold_end) - message += "{}{}{} - {}{}".format(bold_start, tool_file, bold_end, correction_msg, new_line) + message += f"{bold_start}{tool_file}{bold_end} - {correction_msg}{new_line}" return message diff --git a/lib/galaxy/tool_util/client/staging.py b/lib/galaxy/tool_util/client/staging.py index 895ad810b43..e1d35f9822f 100644 --- a/lib/galaxy/tool_util/client/staging.py +++ b/lib/galaxy/tool_util/client/staging.py @@ -8,7 +8,6 @@ import json import logging import os -import six import yaml from galaxy.tool_util.cwl.util import ( @@ -26,8 +25,7 @@ LOAD_TOOLS_FROM_PATH = True DEFAULT_USE_FETCH_API = True -@six.add_metaclass(abc.ABCMeta) -class StagingInterace(object): +class StagingInterace(metaclass=abc.ABCMeta): """Client that parses a job input and populates files into the Galaxy API. Abstract class that must override _post (and optionally other things such @@ -213,7 +211,7 @@ class StagingInterace(object): if job_path is not None: assert job is None - with open(job_path, "r") as f: + with open(job_path) as f: job = yaml.safe_load(f) job_dir = os.path.dirname(job_path) else: diff --git a/lib/galaxy/tool_util/cwl/parser.py b/lib/galaxy/tool_util/cwl/parser.py index b58562f8ebb..2fb9f10d941 100644 --- a/lib/galaxy/tool_util/cwl/parser.py +++ b/lib/galaxy/tool_util/cwl/parser.py @@ -509,7 +509,7 @@ class JobProxy: def stage_recursive(value): is_list = isinstance(value, list) is_dict = isinstance(value, dict) - log.info("handling value {}, is_list {}, is_dict {}".format(value, is_list, is_dict)) + log.info(f"handling value {value}, is_list {is_list}, is_dict {is_dict}") if is_list: for val in value: stage_recursive(val) @@ -582,7 +582,7 @@ class JobProxy: else: self._ok = False - log.info("Output are {}, status is {}".format(out, process_status)) + log.info(f"Output are {out}, status is {process_status}") def collect_outputs(self, tool_working_directory, rcode): if not self.is_command_line_job: @@ -636,7 +636,7 @@ class JobProxy: cwl_job = self.cwl_job() def stageFunc(resolved_path, target_path): - log.info("resolving {} to {}".format(resolved_path, target_path)) + log.info(f"resolving {resolved_path} to {target_path}") try: os.symlink(resolved_path, target_path) except OSError: @@ -868,7 +868,7 @@ def split_step_references(step_references, workflow_id=None, multiple=True): sep_on = "#" expected_prefix = workflow_id + sep_on if not step_reference.startswith(expected_prefix): - raise AssertionError("step_reference [{}] doesn't start with {}".format(step_reference, expected_prefix)) + raise AssertionError(f"step_reference [{step_reference}] doesn't start with {expected_prefix}") step_reference = step_reference[len(expected_prefix):] # Now just grab the step name and input/output name. diff --git a/lib/galaxy/tool_util/cwl/representation.py b/lib/galaxy/tool_util/cwl/representation.py index 9f2c00c09d1..3b2bfabfce5 100644 --- a/lib/galaxy/tool_util/cwl/representation.py +++ b/lib/galaxy/tool_util/cwl/representation.py @@ -149,7 +149,7 @@ def dataset_wrapper_to_file_json(inputs_dir, dataset_wrapper): for secondary_file_name in os.listdir(secondary_files_path): secondary_file_path = os.path.join(secondary_files_path, secondary_file_name) target = os.path.join(inputs_dir, secondary_file_name) - log.info("linking [{}] to [{}]".format(secondary_file_path, target)) + log.info(f"linking [{secondary_file_path}] to [{target}]") os.symlink(secondary_file_path, target) is_dir = os.path.isdir(os.path.realpath(secondary_file_path)) secondary_files.append({"class": "File" if not is_dir else "Directory", "location": target}) @@ -284,7 +284,7 @@ def to_cwl_job(tool, param_dict, local_working_directory): array_value.append(simple_value(only_input, instance[input_name[:-len("_repeat")]])) input_json[input_name[:-len("_repeat")]] = array_value elif input.type == "conditional": - assert input_name in param_dict, "No value for {} in {}".format(input_name, param_dict) + assert input_name in param_dict, f"No value for {input_name} in {param_dict}" current_case = param_dict[input_name]["_cwl__type_"] if str(current_case) != "null": # str because it is a wrapped... case_index = input.get_current_case(current_case) @@ -333,7 +333,7 @@ def to_galaxy_parameters(tool, as_dict): only_input = next(iter(input.inputs.values())) for index, value in enumerate(as_dict_value): - key = "{}_repeat_0|{}".format(input_name, only_input.name) + key = f"{input_name}_repeat_0|{only_input.name}" galaxy_value = from_simple_value(only_input, value) galaxy_request[key] = galaxy_value elif galaxy_input_type == "conditional": diff --git a/lib/galaxy/tool_util/cwl/runtime_actions.py b/lib/galaxy/tool_util/cwl/runtime_actions.py index baeeccdaf6c..56e14b85f32 100644 --- a/lib/galaxy/tool_util/cwl/runtime_actions.py +++ b/lib/galaxy/tool_util/cwl/runtime_actions.py @@ -200,7 +200,7 @@ def handle_outputs(job_directory=None): elif isinstance(output, dict): prefix = "%s|__part__|" % output_name for record_key, record_value in output.items(): - record_value_output_key = "{}{}".format(prefix, record_key) + record_value_output_key = f"{prefix}{record_key}" if isinstance(record_value, dict) and "class" in record_value: handle_known_output(record_value, record_value_output_key, output_name) else: diff --git a/lib/galaxy/tool_util/cwl/util.py b/lib/galaxy/tool_util/cwl/util.py index f7c5018e458..5badd131ab0 100644 --- a/lib/galaxy/tool_util/cwl/util.py +++ b/lib/galaxy/tool_util/cwl/util.py @@ -338,7 +338,7 @@ class FileLiteralTarget: self.path = path def __str__(self): - return "FileLiteralTarget[contents={}] with {}".format(self.contents, self.properties) + return f"FileLiteralTarget[contents={self.contents}] with {self.properties}" class FileUploadTarget: @@ -350,7 +350,7 @@ class FileUploadTarget: self.properties = kwargs def __str__(self): - return "FileUploadTarget[path={}] with {}".format(self.path, self.properties) + return f"FileUploadTarget[path={self.path}] with {self.properties}" class ObjectUploadTarget: @@ -394,7 +394,7 @@ def invocation_to_output(invocation, history_id, output_id): collection = invocation["output_collections"][output_id] galaxy_output = GalaxyOutput(history_id, "dataset_collection", collection["id"], None) else: - raise Exception("Failed to find output with label [{}] in [{}]".format(output_id, invocation)) + raise Exception(f"Failed to find output with label [{output_id}] in [{invocation}]") return galaxy_output diff --git a/lib/galaxy/tool_util/deps/__init__.py b/lib/galaxy/tool_util/deps/__init__.py index b043c543bcf..1a03f5a6bda 100644 --- a/lib/galaxy/tool_util/deps/__init__.py +++ b/lib/galaxy/tool_util/deps/__init__.py @@ -158,7 +158,7 @@ class DependencyManager: explicit_resolver_options = {} default = resolver.config_options.get(key) config_prefix = resolver.resolver_type - global_key = "{}_{}".format(config_prefix, key) + global_key = f"{config_prefix}_{key}" value = explicit_resolver_options.get(key, CONFIG_VAL_NOT_FOUND) if value is CONFIG_VAL_NOT_FOUND: value = self.get_app_option(global_key, default) @@ -299,7 +299,7 @@ class DependencyManager: return any(map(lambda r: isinstance(r, ToolShedPackageDependencyResolver), self.dependency_resolvers)) def find_dep(self, name, version=None, type='package', **kwds): - log.debug('Find dependency {} version {}'.format(name, version)) + log.debug(f'Find dependency {name} version {version}') requirements = ToolRequirements([ToolRequirement(name=name, version=version, type=type)]) dep_dict = self._requirements_to_dependencies_dict(requirements, **kwds) if len(dep_dict) > 0: diff --git a/lib/galaxy/tool_util/deps/brew_exts.py b/lib/galaxy/tool_util/deps/brew_exts.py index 3bcbf8e1b1a..f7ebda15757 100755 --- a/lib/galaxy/tool_util/deps/brew_exts.py +++ b/lib/galaxy/tool_util/deps/brew_exts.py @@ -266,7 +266,7 @@ def load_versioned_deps(cellar_path, relaxed=None): if RELAXED: return [] else: - raise OSError("Could not locate versioned receipt file: {}".format(v_metadata_path)) + raise OSError(f"Could not locate versioned receipt file: {v_metadata_path}") with open(v_metadata_path) as f: metadata = json.load(f) return metadata['deps'] diff --git a/lib/galaxy/tool_util/deps/conda_compat.py b/lib/galaxy/tool_util/deps/conda_compat.py index 77b90bc7ea3..627ec63898a 100644 --- a/lib/galaxy/tool_util/deps/conda_compat.py +++ b/lib/galaxy/tool_util/deps/conda_compat.py @@ -4,8 +4,8 @@ In general there are utilities available for Conda building and parsing that are and should be utilized when available but that are only available in conda channels and not in PyPI. This module serves as a PyPI capable interface to these utilities. """ -import collections import os +from collections.abc import Hashable import yaml @@ -28,7 +28,7 @@ class _Memoized: self.cache = {} def __call__(self, *args): - if not isinstance(args, collections.Hashable): + if not isinstance(args, Hashable): # uncacheable. a list, for instance. # better to not cache than blow up. return self.func(*args) diff --git a/lib/galaxy/tool_util/deps/conda_util.py b/lib/galaxy/tool_util/deps/conda_util.py index c538e9f58ee..b6ddde2fe06 100644 --- a/lib/galaxy/tool_util/deps/conda_util.py +++ b/lib/galaxy/tool_util/deps/conda_util.py @@ -4,12 +4,12 @@ import json import logging import os import re +import shlex import shutil import sys import tempfile import packaging.version -from six.moves import shlex_quote from galaxy.util import ( commands, @@ -226,7 +226,7 @@ class CondaContext(installable.InstallableContext): env = {} if self.condarc_override: env["CONDARC"] = self.condarc_override - cmd_string = ' '.join(map(shlex_quote, cmd)) + cmd_string = ' '.join(map(shlex.quote, cmd)) kwds = dict() try: if stdout_path: @@ -372,7 +372,7 @@ class CondaTarget: def __str__(self): attributes = "package=%s" % self.package if self.version is not None: - attributes = "{},version={}".format(self.package, self.version) + attributes = f"{self.package},version={self.version}" else: attributes = "%s,unversioned" % self.package @@ -388,7 +388,7 @@ class CondaTarget: """ Return a package specifier as consumed by conda install/create. """ if self.version: - return "{}={}".format(self.package, self.version) + return f"{self.package}={self.version}" else: return self.package @@ -399,7 +399,7 @@ class CondaTarget: a fixed and predictable name given package and version. """ if self.version: - return "__{}@{}".format(self.package, self.version) + return f"__{self.package}@{self.version}" else: return "__%s@_uv_" % (self.package) diff --git a/lib/galaxy/tool_util/deps/container_classes.py b/lib/galaxy/tool_util/deps/container_classes.py index 29dcaf766ff..bfda23afd79 100644 --- a/lib/galaxy/tool_util/deps/container_classes.py +++ b/lib/galaxy/tool_util/deps/container_classes.py @@ -84,7 +84,7 @@ class Container(metaclass=ABCMeta): self.container_info = {} def prop(self, name, default): - destination_name = "{}_{}".format(self.container_type, name) + destination_name = f"{self.container_type}_{name}" return self.destination_info.get(destination_name, default) @property @@ -268,7 +268,7 @@ class DockerContainer(Container, HasDockerLikeVolumes): def containerize_command(self, command): env_directives = [] for pass_through_var in self.tool_info.env_pass_through: - env_directives.append('"{}=${}"'.format(pass_through_var, pass_through_var)) + env_directives.append(f'"{pass_through_var}=${pass_through_var}"') # Allow destinations to explicitly set environment variables just for # docker container. Better approach is to set for destination and then @@ -276,7 +276,7 @@ class DockerContainer(Container, HasDockerLikeVolumes): for key, value in self.destination_info.items(): if key.startswith("docker_env_"): env = key[len("docker_env_"):] - env_directives.append('"{}={}"'.format(env, value)) + env_directives.append(f'"{env}={value}"') working_directory = self.job_info.working_directory if not working_directory: diff --git a/lib/galaxy/tool_util/deps/container_resolvers/mulled.py b/lib/galaxy/tool_util/deps/container_resolvers/mulled.py index b0eede03fa7..5f919ec9645 100644 --- a/lib/galaxy/tool_util/deps/container_resolvers/mulled.py +++ b/lib/galaxy/tool_util/deps/container_resolvers/mulled.py @@ -258,7 +258,7 @@ def targets_to_mulled_name(targets, hash_func, namespace, resolution_cache=None) if len(targets) == 1: target = targets[0] target_version = target.version - cache_key = "ns[{}]__single__{}__@__{}".format(namespace, target.package_name, target_version) + cache_key = f"ns[{namespace}]__single__{target.package_name}__@__{target_version}" if cache_key in unresolved_cache: return None name = cached_name(cache_key) @@ -274,7 +274,7 @@ def targets_to_mulled_name(targets, hash_func, namespace, resolution_cache=None) else: version = tag if target_version and version == target_version: - name = "{}:{}".format(target.package_name, tag) + name = f"{target.package_name}:{tag}" break else: @@ -294,7 +294,7 @@ def targets_to_mulled_name(targets, hash_func, namespace, resolution_cache=None) else: raise Exception("Unimplemented mulled hash_func [%s]" % hash_func) - cache_key = "ns[{}]__{}__{}".format(namespace, hash_func, base_image_name) + cache_key = f"ns[{namespace}]__{hash_func}__{base_image_name}" if cache_key in unresolved_cache: return None name = cached_name(cache_key) @@ -311,7 +311,7 @@ def targets_to_mulled_name(targets, hash_func, namespace, resolution_cache=None) else: # base_image_name of form , simply add build number # as tag to fully qualify image. - name = "{}:{}".format(base_image_name, tag) + name = f"{base_image_name}:{tag}" if name and mulled_resolution_cache: mulled_resolution_cache.put(cache_key, name) @@ -399,9 +399,9 @@ class MulledDockerContainerResolver(ContainerResolver): name = targets_to_mulled_name(targets=targets, hash_func=self.hash_func, namespace=self.namespace, resolution_cache=resolution_cache) if name: - container_id = "quay.io/{}/{}".format(self.namespace, name) + container_id = f"quay.io/{self.namespace}/{name}" if self.protocol: - container_id = "{}{}".format(self.protocol, container_id) + container_id = f"{self.protocol}{container_id}" container_description = ContainerDescription( container_id, type=self.container_type, diff --git a/lib/galaxy/tool_util/deps/containers.py b/lib/galaxy/tool_util/deps/containers.py index 278b7938d6a..8ae0c77f48c 100644 --- a/lib/galaxy/tool_util/deps/containers.py +++ b/lib/galaxy/tool_util/deps/containers.py @@ -140,14 +140,14 @@ class ContainerFinder: def __build_container_id_from_parts(self, container_type, destination_info, mode): repo = "" owner = "" - repo_key = "{}_repo_{}".format(container_type, mode) - owner_key = "{}_owner_{}".format(container_type, mode) + repo_key = f"{container_type}_repo_{mode}" + owner_key = f"{container_type}_owner_{mode}" if repo_key in destination_info: repo = destination_info[repo_key] + "/" if owner_key in destination_info: owner = destination_info[owner_key] + "/" - cont_id = repo + owner + destination_info["{}_image_{}".format(container_type, mode)] - tag_key = "{}_tag_{}".format(container_type, mode) + cont_id = repo + owner + destination_info[f"{container_type}_image_{mode}"] + tag_key = f"{container_type}_tag_{mode}" if tag_key in destination_info: cont_id += ":" + destination_info[tag_key] return cont_id @@ -259,7 +259,7 @@ class ContainerRegistry: continue container_description = container_resolver.resolve(enabled_container_types, tool_info, resolution_cache=resolution_cache) - log.info("Checking with container resolver [{}] found description [{}]".format(container_resolver, container_description)) + log.info(f"Checking with container resolver [{container_resolver}] found description [{container_description}]") if container_description: assert container_description.type in enabled_container_types return ResolvedContainerDescription(container_resolver, container_description) diff --git a/lib/galaxy/tool_util/deps/docker_util.py b/lib/galaxy/tool_util/deps/docker_util.py index b126fa81dad..102eb37516c 100644 --- a/lib/galaxy/tool_util/deps/docker_util.py +++ b/lib/galaxy/tool_util/deps/docker_util.py @@ -3,8 +3,7 @@ ...using common defaults and configuration mechanisms. """ import os - -from six.moves import shlex_quote +import shlex from galaxy.util.commands import argv_to_str @@ -69,7 +68,7 @@ def build_docker_cache_command( ): inspect_image_command = command_shell("inspect", [image], **kwds) pull_image_command = command_shell("pull", [image], **kwds) - cache_command = "{} > /dev/null 2>&1\n[ $? -ne 0 ] && {} > /dev/null 2>&1\n".format(inspect_image_command, pull_image_command) + cache_command = f"{inspect_image_command} > /dev/null 2>&1\n[ $? -ne 0 ] && {pull_image_command} > /dev/null 2>&1\n" return cache_command @@ -151,15 +150,15 @@ def build_docker_run_command( for volume in volumes: command_parts.extend(["-v", str(volume)]) if volumes_from: - command_parts.extend(["--volumes-from", shlex_quote(str(volumes_from))]) + command_parts.extend(["--volumes-from", shlex.quote(str(volumes_from))]) if memory: - command_parts.extend(["-m", shlex_quote(memory)]) + command_parts.extend(["-m", shlex.quote(memory)]) if name: - command_parts.extend(["--name", shlex_quote(name)]) + command_parts.extend(["--name", shlex.quote(name)]) if working_directory: - command_parts.extend(["-w", shlex_quote(working_directory)]) + command_parts.extend(["-w", shlex.quote(working_directory)]) if net: - command_parts.extend(["--net", shlex_quote(net)]) + command_parts.extend(["--net", shlex.quote(net)]) if auto_rm: command_parts.append("--rm") if run_extra_arguments: @@ -176,8 +175,8 @@ def build_docker_run_command( command_parts.extend(["--user", user]) full_image = image if tag: - full_image = "{}:{}".format(full_image, tag) - command_parts.append(shlex_quote(full_image)) + full_image = f"{full_image}:{tag}" + command_parts.append(shlex.quote(full_image)) command_parts.append(container_command) return " ".join(command_parts) diff --git a/lib/galaxy/tool_util/deps/mulled/get_tests.py b/lib/galaxy/tool_util/deps/mulled/get_tests.py index efb75c07c75..8fa919325a5 100644 --- a/lib/galaxy/tool_util/deps/mulled/get_tests.py +++ b/lib/galaxy/tool_util/deps/mulled/get_tests.py @@ -138,7 +138,7 @@ def find_anaconda_versions(name, anaconda_channel='bioconda'): """ Find a list of available anaconda versions for a given container name """ - r = requests.get("https://anaconda.org/{}/{}/files".format(anaconda_channel, name)) + r = requests.get(f"https://anaconda.org/{anaconda_channel}/{name}/files") urls = [] for line in r.text.split('\n'): if 'download/linux' in line: @@ -151,9 +151,9 @@ def open_recipe_file(file, recipes_path=None, github_repo='bioconda/bioconda-rec Open a file at a particular location and return contents as string """ if recipes_path: - return open('{}/{}'.format(recipes_path, file)).read() + return open(f'{recipes_path}/{file}').read() else: # if no clone of the repo is available locally, download from GitHub - r = requests.get('https://raw.githubusercontent.com/{}/master/{}'.format(github_repo, file)) + r = requests.get(f'https://raw.githubusercontent.com/{github_repo}/master/{file}') if r.status_code == 404: raise OSError else: @@ -165,10 +165,10 @@ def get_alternative_versions(filepath, filename, recipes_path=None, github_repo= Return files that match 'filepath/*/filename' in the bioconda-recipes repository """ if recipes_path: - return [n.replace('%s/' % recipes_path, '') for n in glob('{}/{}/*/{}'.format(recipes_path, filepath, filename))] + return [n.replace('%s/' % recipes_path, '') for n in glob(f'{recipes_path}/{filepath}/*/{filename}')] # else use the GitHub API: versions = [] - r = json.loads(requests.get('https://api.github.com/repos/{}/contents/{}'.format(github_repo, filepath)).text) + r = json.loads(requests.get(f'https://api.github.com/repos/{github_repo}/contents/{filepath}').text) for subfile in r: if subfile['type'] == 'dir': if requests.get('https://raw.githubusercontent.com/{}/master/{}/{}'.format(github_repo, subfile['path'], filename)).status_code == 200: diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_build.py b/lib/galaxy/tool_util/deps/mulled/mulled_build.py index ee9a29eaa43..b57e2a57873 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_build.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_build.py @@ -12,6 +12,7 @@ Build a mulled image with: import json import logging import os +import shlex import shutil import stat import string @@ -19,7 +20,6 @@ import subprocess import sys from sys import platform as _platform -from six.moves import shlex_quote try: import yaml except ImportError: @@ -220,8 +220,8 @@ def mull_targets( for channel in channels: if channel.startswith('file://'): - bind_path = channel.lstrip('file://') - binds.append('/{}:/{}'.format(bind_path, bind_path)) + bind_path = channel[7:] + binds.append(f'/{bind_path}:/{bind_path}') channels = ",".join(channels) target_str = ",".join(map(conda_build_target_str, targets)) @@ -252,19 +252,19 @@ def mull_targets( involucro_args.extend(["-set", "SINGULARITY=1"]) involucro_args.extend(["-set", "SINGULARITY_IMAGE_NAME=%s" % singularity_image_name]) involucro_args.extend(["-set", "SINGULARITY_IMAGE_DIR=%s" % singularity_image_dir]) - involucro_args.extend(["-set", "USER_ID={}:{}".format(os.getuid(), os.getgid())]) + involucro_args.extend(["-set", f"USER_ID={os.getuid()}:{os.getgid()}"]) if test: involucro_args.extend(["-set", "TEST=%s" % test]) if conda_version is not None: verbose = "--verbose" if verbose else "--quiet" - involucro_args.extend(["-set", "PREINSTALL=conda install {} --yes conda={}".format(verbose, conda_version)]) + involucro_args.extend(["-set", f"PREINSTALL=conda install {verbose} --yes conda={conda_version}"]) involucro_args.append(command) if test_files: test_bind = [] for test_file in test_files: if ':' not in test_file: if os.path.exists(test_file): - test_bind.append("{}:{}/{}".format(test_file, DEFAULT_WORKING_DIR, test_file)) + test_bind.append(f"{test_file}:{DEFAULT_WORKING_DIR}/{test_file}") else: if os.path.exists(test_file.split(':')[0]): test_bind.append(test_file) @@ -272,7 +272,7 @@ def mull_targets( involucro_args.insert(6, '-set') involucro_args.insert(7, "TEST_BINDS=%s" % ",".join(test_bind)) cmd = involucro_context.build_command(involucro_args) - print('Executing: ' + ' '.join(shlex_quote(_) for _ in cmd)) + print('Executing: ' + ' '.join(shlex.quote(_) for _ in cmd)) if dry_run: return 0 ensure_installed(involucro_context, True) diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_build_channel.py b/lib/galaxy/tool_util/deps/mulled/mulled_build_channel.py index 0f69eda76ad..31582cf7025 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_build_channel.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_build_channel.py @@ -40,7 +40,7 @@ def _fetch_repo_data(args): if not os.path.exists(repo_data): platform_tag = 'osx-64' if sys.platform == 'darwin' else 'linux-64' subprocess.check_call([ - 'wget', '--quiet', 'https://conda.anaconda.org/{}/{}/repodata.json.bz2'.format(channel, platform_tag), + 'wget', '--quiet', f'https://conda.anaconda.org/{channel}/{platform_tag}/repodata.json.bz2', '-O', '%s.bz2' % repo_data ]) subprocess.check_call([ diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_build_files.py b/lib/galaxy/tool_util/deps/mulled/mulled_build_files.py index 2a7106a0479..88a0c578b4e 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_build_files.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_build_files.py @@ -79,7 +79,7 @@ def generate_targets(target_source): def tuple_from_header(header): fields = header[1:].split('\t') for field in fields: - assert field in KNOWN_FIELDS, "'{}' is not one of {}".format(field, KNOWN_FIELDS) + assert field in KNOWN_FIELDS, f"'{field}' is not one of {KNOWN_FIELDS}" # Make sure tuple contains all fields for field in KNOWN_FIELDS: if field not in fields: @@ -92,7 +92,7 @@ def line_to_targets(line_str, line_tuple): line_parts = line_str.split("\t") n_fields = len(line_tuple._fields) targets_column = line_tuple._fields.index('targets') - assert len(line_parts) <= n_fields, "Too many fields in line [{}], expect at most {} - targets, image build number, and name override.".format(line_str, n_fields) + assert len(line_parts) <= n_fields, f"Too many fields in line [{line_str}], expect at most {n_fields} - targets, image build number, and name override." line_parts += [None] * (n_fields - len(line_parts)) line_parts[targets_column] = target_str_to_targets(line_parts[targets_column]) return line_tuple(*line_parts) diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_search.py b/lib/galaxy/tool_util/deps/mulled/mulled_search.py index c91030e42cd..30d86e57d1c 100755 --- a/lib/galaxy/tool_util/deps/mulled/mulled_search.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_search.py @@ -99,7 +99,7 @@ class QuaySearch(): Function downloads additional information from quay.io to get the tag-field which includes the version number. """ - url = "{}/{}/{}".format(QUAY_API_URL, self.organization, repository_string) + url = f"{QUAY_API_URL}/{self.organization}/{repository_string}" r = requests.get(url, headers={'Accept-encoding': 'gzip'}) json_decoder = json.JSONDecoder() diff --git a/lib/galaxy/tool_util/deps/mulled/util.py b/lib/galaxy/tool_util/deps/mulled/util.py index cc3d3acf650..9535c400561 100644 --- a/lib/galaxy/tool_util/deps/mulled/util.py +++ b/lib/galaxy/tool_util/deps/mulled/util.py @@ -48,7 +48,7 @@ def quay_versions(namespace, pkg_name): def quay_repository(namespace, pkg_name): assert namespace is not None assert pkg_name is not None - url = 'https://quay.io/api/v1/repository/{}/{}'.format(namespace, pkg_name) + url = f'https://quay.io/api/v1/repository/{namespace}/{pkg_name}' response = requests.get(url, timeout=QUAY_IO_TIMEOUT) data = response.json() return data @@ -184,7 +184,7 @@ def _simple_image_name(targets, image_build=None): suffix += ":%s" % target.version if build is not None: suffix += "--%s" % build - return "{}{}".format(target.package_name, suffix) + return f"{target.package_name}{suffix}" def v1_image_name(targets, image_build=None, name_override=None): @@ -221,7 +221,7 @@ def v1_image_name(targets, image_build=None, name_override=None): m = hashlib.sha1() m.update(requirements_buffer.encode()) suffix = "" if not image_build else ":%s" % image_build - return "mulled-v1-{}{}".format(m.hexdigest(), suffix) + return f"mulled-v1-{m.hexdigest()}{suffix}" def v2_image_name(targets, image_build=None, name_override=None): @@ -287,8 +287,8 @@ def v2_image_name(targets, image_build=None, name_override=None): build_suffix = image_build suffix = "" if version_hash_str or build_suffix: - suffix = ":{}{}".format(version_hash_str, build_suffix) - return "mulled-v2-{}{}".format(package_hash.hexdigest(), suffix) + suffix = f":{version_hash_str}{build_suffix}" + return f"mulled-v2-{package_hash.hexdigest()}{suffix}" def get_file_from_recipe_url(url): diff --git a/lib/galaxy/tool_util/deps/requirements.py b/lib/galaxy/tool_util/deps/requirements.py index 90f636f9ee9..679a6c37490 100644 --- a/lib/galaxy/tool_util/deps/requirements.py +++ b/lib/galaxy/tool_util/deps/requirements.py @@ -50,7 +50,7 @@ class ToolRequirement: return hash((self.name, self.type, self.version, frozenset(self.specs))) def __str__(self): - return "ToolRequirement[{},version={},type={},specs={}]".format(self.name, self.version, self.type, self.specs) + return f"ToolRequirement[{self.name},version={self.version},type={self.type},specs={self.specs}]" __repr__ = __str__ @@ -191,7 +191,7 @@ class ContainerDescription: ) def __str__(self): - return "ContainerDescription[identifier={},type={}]".format(self.identifier, self.type) + return f"ContainerDescription[identifier={self.identifier},type={self.type}]" def parse_requirements_from_dict(root_dict): diff --git a/lib/galaxy/tool_util/deps/resolvers/__init__.py b/lib/galaxy/tool_util/deps/resolvers/__init__.py index 7387d3f7899..8bdcefa14e1 100644 --- a/lib/galaxy/tool_util/deps/resolvers/__init__.py +++ b/lib/galaxy/tool_util/deps/resolvers/__init__.py @@ -269,7 +269,7 @@ class Dependency(Dictifiable, metaclass=ABCMeta): """ Return a message describing this dependency """ - return "Using dependency {} version {} of type {}".format(self.name, self.version, self.dependency_type) + return f"Using dependency {self.name} version {self.version} of type {self.dependency_type}" class ContainerDependency(Dependency): diff --git a/lib/galaxy/tool_util/deps/resolvers/brewed_tool_shed_packages.py b/lib/galaxy/tool_util/deps/resolvers/brewed_tool_shed_packages.py index af936528050..7b0d7d1e316 100644 --- a/lib/galaxy/tool_util/deps/resolvers/brewed_tool_shed_packages.py +++ b/lib/galaxy/tool_util/deps/resolvers/brewed_tool_shed_packages.py @@ -146,7 +146,7 @@ def build_recipe_name(package_name, package_version, repository_owner, repositor owner = repository_owner.replace("-", "") name = repository_name name = name.replace("_", "").replace("-", "") - base = "{}_{}".format(owner, name) + base = f"{owner}_{name}" return base diff --git a/lib/galaxy/tool_util/deps/resolvers/conda.py b/lib/galaxy/tool_util/deps/resolvers/conda.py index 82fc078c8b0..6c03873d690 100644 --- a/lib/galaxy/tool_util/deps/resolvers/conda.py +++ b/lib/galaxy/tool_util/deps/resolvers/conda.py @@ -363,7 +363,7 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li conda_target, conda_context=self.conda_context ) if not is_installed: - log.debug("Removing failed conda install of {}, version '{}'".format(name, version)) + log.debug(f"Removing failed conda install of {name}, version '{version}'") cleanup_failed_install(conda_target, conda_context=self.conda_context) return is_installed diff --git a/lib/galaxy/tool_util/deps/resolvers/galaxy_packages.py b/lib/galaxy/tool_util/deps/resolvers/galaxy_packages.py index 93e1ff0f116..7864f63f2d7 100644 --- a/lib/galaxy/tool_util/deps/resolvers/galaxy_packages.py +++ b/lib/galaxy/tool_util/deps/resolvers/galaxy_packages.py @@ -42,9 +42,9 @@ class GalaxyPackageDependency(Dependency): def shell_commands(self): base_path = self.path if self.type == 'package' and self.script is None: - commands = 'PACKAGE_BASE={}; export PACKAGE_BASE; PATH="{}/bin:$PATH"; export PATH'.format(base_path, base_path) + commands = f'PACKAGE_BASE={base_path}; export PACKAGE_BASE; PATH="{base_path}/bin:$PATH"; export PATH' else: - commands = 'PACKAGE_BASE={}; export PACKAGE_BASE; . {}'.format(base_path, self.script) + commands = f'PACKAGE_BASE={base_path}; export PACKAGE_BASE; . {self.script}' return commands diff --git a/lib/galaxy/tool_util/deps/resolvers/lmod.py b/lib/galaxy/tool_util/deps/resolvers/lmod.py index 1ecd77c244a..7c545f31a8b 100644 --- a/lib/galaxy/tool_util/deps/resolvers/lmod.py +++ b/lib/galaxy/tool_util/deps/resolvers/lmod.py @@ -7,6 +7,7 @@ LMOD @ Github: https://github.com/TACC/Lmod """ import logging +from io import StringIO from os import getenv from os.path import exists from subprocess import ( @@ -14,8 +15,6 @@ from subprocess import ( Popen ) -from six import StringIO - from . import ( Dependency, DependencyResolver, @@ -164,7 +163,7 @@ class LmodDependency(Dependency): # Get the full module name in the form "tool_name/tool_version" module_to_load = self.module_name if self.module_version: - module_to_load = '{}/{}'.format(self.module_name, self.module_version) + module_to_load = f'{self.module_name}/{self.module_version}' # Build the list of command to add to run script # Note that since "module" is actually a bash function, we are directy executing the underlying executable instead @@ -172,7 +171,7 @@ class LmodDependency(Dependency): command = 'MODULEPATH=%s; ' % (self.lmod_dependency_resolver.modulepath) command += 'export MODULEPATH; ' # - Execute the "module load" command (or rather the "/path/to/lmod load" command) - command += 'eval `{} load {}` '.format(self.lmod_dependency_resolver.lmodexec, module_to_load) + command += f'eval `{self.lmod_dependency_resolver.lmodexec} load {module_to_load}` ' # - Execute the "settarg" command in addition if needed if self.lmod_dependency_resolver.settargexec is not None: command += '&& eval `%s -s sh`' % (self.lmod_dependency_resolver.settargexec) diff --git a/lib/galaxy/tool_util/deps/resolvers/modules.py b/lib/galaxy/tool_util/deps/resolvers/modules.py index 1d3fd7eb58d..c06d950fc54 100644 --- a/lib/galaxy/tool_util/deps/resolvers/modules.py +++ b/lib/galaxy/tool_util/deps/resolvers/modules.py @@ -7,6 +7,7 @@ it, hence support for it will be minimal. The Galaxy team eagerly welcomes community contribution and maintenance however. """ import logging +from io import StringIO from os import ( environ, pathsep @@ -21,8 +22,6 @@ from subprocess import ( Popen ) -from six import StringIO - from . import ( Dependency, DependencyResolver, @@ -208,7 +207,7 @@ class ModuleDependency(Dependency): def shell_commands(self): module_to_load = self.module_name if self.module_version: - module_to_load = '{}/{}'.format(self.module_name, self.module_version) + module_to_load = f'{self.module_name}/{self.module_version}' command = 'MODULEPATH={}; export MODULEPATH; eval `{} sh load {}`'.format(self.module_dependency_resolver.modulepath, self.module_dependency_resolver.modulecmd, module_to_load) diff --git a/lib/galaxy/tool_util/deps/singularity_util.py b/lib/galaxy/tool_util/deps/singularity_util.py index 8ab2ff2a4e2..7a764caab03 100644 --- a/lib/galaxy/tool_util/deps/singularity_util.py +++ b/lib/galaxy/tool_util/deps/singularity_util.py @@ -1,6 +1,5 @@ import os - -from six.moves import shlex_quote +import shlex DEFAULT_WORKING_DIRECTORY = None DEFAULT_SINGULARITY_COMMAND = "singularity" @@ -49,7 +48,7 @@ def build_singularity_run_command( for (key, value) in env: if key == 'HOME': home = value - command_parts.extend(["SINGULARITYENV_{}={}".format(key, value)]) + command_parts.extend([f"SINGULARITYENV_{key}={value}"]) command_parts += _singularity_prefix( singularity_cmd=singularity_cmd, sudo=sudo, @@ -60,11 +59,11 @@ def build_singularity_run_command( for volume in volumes: command_parts.extend(["-B", str(volume)]) if home is not None: - command_parts.extend(["--home", "{}:{}".format(home, home)]) + command_parts.extend(["--home", f"{home}:{home}"]) if run_extra_arguments: command_parts.append(run_extra_arguments) full_image = image - command_parts.append(shlex_quote(full_image)) + command_parts.append(shlex.quote(full_image)) command_parts.append(container_command) return " ".join(command_parts) diff --git a/lib/galaxy/tool_util/fetcher.py b/lib/galaxy/tool_util/fetcher.py index eb066208b52..13d12d07a40 100644 --- a/lib/galaxy/tool_util/fetcher.py +++ b/lib/galaxy/tool_util/fetcher.py @@ -19,7 +19,7 @@ class ToolLocationFetcher: raise Exception("Invalid URI passed to get_tool_source") scheme, rest = uri_like.split(":", 2) if scheme not in self.resolver_classes: - raise Exception("Unknown tool scheme [{}] for URI [{}]".format(scheme, uri_like)) + raise Exception(f"Unknown tool scheme [{scheme}] for URI [{uri_like}]") path = self.resolver_classes[scheme]().get_tool_source_path(uri_like) return path diff --git a/lib/galaxy/tool_util/lint.py b/lib/galaxy/tool_util/lint.py index 0b5577fd269..aecf7612b23 100644 --- a/lib/galaxy/tool_util/lint.py +++ b/lib/galaxy/tool_util/lint.py @@ -92,7 +92,7 @@ class LintContext: if self.printed_linter_info: return self.printed_linter_info = True - print("Applying linter {}... {}".format(name, status)) + print(f"Applying linter {name}... {status}") for message in self.error_messages: self.found_errors = True diff --git a/lib/galaxy/tool_util/linters/inputs.py b/lib/galaxy/tool_util/linters/inputs.py index d9183edc5ef..98caf972bd5 100644 --- a/lib/galaxy/tool_util/linters/inputs.py +++ b/lib/galaxy/tool_util/linters/inputs.py @@ -83,14 +83,14 @@ def lint_inputs(tool_xml, lint_ctx): for option_id in option_ids: if option_id not in when_ids: - lint_ctx.warn("No block found for {} option '{}' inside conditional '{}'".format(first_param_type, option_id, conditional_name)) + lint_ctx.warn(f"No block found for {first_param_type} option '{option_id}' inside conditional '{conditional_name}'") for when_id in when_ids: if when_id not in option_ids: if first_param_type == 'select': - lint_ctx.warn("No