Drop support for retired Python 3.5

Upgrade syntax using `pyupgrade --py36-plus` .

Manually drop several `six` imports.

Also:
- Remove broken pr_cache in scripts/bootstrap_history.py
- Fix broken prefix removal in lib/galaxy/tool_util/deps/mulled/mulled_build.py
This commit is contained in:
Nicola Soranzo
2020-10-07 11:52:13 +01:00
parent 57d6a3857d
commit 9d74bba7fb
622 changed files with 1918 additions and 2175 deletions
+19 -19
View File
@@ -27,7 +27,7 @@ variables:
jobs:
get_code:
docker:
- image: circleci/python:3.5
- image: circleci/python:3.6
<<: *set_workdir
steps:
# Replace standard code checkout with shallow clone to speed things up.
@@ -83,34 +83,34 @@ jobs:
key: v1-repo-{{ .Environment.CIRCLE_SHA1 }}
paths:
- ~/repo
py35_docstring:
py36_docstring:
docker:
- image: circleci/python:3.5
- image: circleci/python:3.6
<<: *set_workdir
steps:
- *restore_repo_cache
- *install_tox
- run: tox -e py35-lint_docstring_include_list
py35_lint:
- run: tox -e py36-lint_docstring_include_list
py36_lint:
docker:
- image: circleci/python:3.5
- image: circleci/python:3.6
<<: *set_workdir
steps:
- *restore_repo_cache
- *install_tox
- run: tox -e py35-lint
py35_unit:
- run: tox -e py36-lint
py36_unit:
docker:
- image: circleci/python:3.5
- image: circleci/python:3.6
<<: *set_workdir
steps:
- *restore_repo_cache
- *install_tox
- *install_ffprobe
- run: tox -e py35-unit
py35_first_startup:
- run: tox -e py36-unit
py36_first_startup:
docker:
- image: circleci/python:3.5
- image: circleci/python:3.6
<<: *set_workdir
steps:
- *restore_repo_cache
@@ -118,10 +118,10 @@ jobs:
- run: wget -q https://github.com/jmchilton/galaxy-downloads/raw/master/db_gx_rev_0141.sqlite
- run: mv db_gx_rev_0141.sqlite database/universe.sqlite
- *install_tox
- run: tox -e py35-first_startup
- run: tox -e py36-first_startup
validate_test_tools:
docker:
- image: circleci/python:3.5
- image: circleci/python:3.6
<<: *set_workdir
steps:
- *restore_repo_cache
@@ -131,7 +131,7 @@ jobs:
- run: tox -e validate_test_tools
test_galaxy_packages:
docker:
- image: circleci/python:3.5
- image: circleci/python:3.6
<<: *set_workdir
steps:
- *restore_repo_cache
@@ -167,13 +167,13 @@ workflows:
get_code_and_test:
jobs:
- get_code
- py35_docstring:
- py36_docstring:
<<: *requires_get_code
- py35_lint:
- py36_lint:
<<: *requires_get_code
- py35_unit:
- py36_unit:
<<: *requires_get_code
- py35_first_startup:
- py36_first_startup:
<<: *requires_get_code
- test_galaxy_packages:
<<: *requires_get_code
+1 -1
View File
@@ -24,7 +24,7 @@ Community support is available at `Galaxy Help <https://help.galaxyproject.org/>
Galaxy Quickstart
=================
Galaxy requires Python 3.5 or 3.6 . To check your Python version, run:
Galaxy requires Python 3.6 . To check your Python version, run:
.. code:: console
-1
View File
@@ -2,7 +2,6 @@
"""
usage: %prog width height bg_color hatch_color [color alpha stop_pos] +
"""
from __future__ import division
import sys
from math import pi
@@ -8,7 +8,7 @@ from galaxy.util import Params
log = logging.getLogger(__name__)
class TourGenerator(object):
class TourGenerator:
def __init__(self, trans, tool_id, tool_version):
self._trans = trans
self._tool = self._trans.app.toolbox.get_tool(tool_id, tool_version)
@@ -177,14 +177,14 @@ class TourGenerator(object):
if name in test_inputs:
hid = self._hids[name]
dataset = self._test.inputs[name][0]
step['content'] = 'Select dataset: <b>%s: %s</b>' % (
step['content'] = 'Select dataset: <b>{}: {}</b>'.format(
hid, dataset
)
else:
step['content'] = 'Select a dataset'
elif input.type == 'conditional':
param_id = '%s|%s' % (input.name, input.test_param.name)
param_id = f'{input.name}|{input.test_param.name}'
step['title'] = input.test_param.label
step['element'] = '[tour_id="%s"]' % param_id
params = []
@@ -203,7 +203,7 @@ class TourGenerator(object):
cases[key] = value.label
for case_id, case_title in cases.items():
tour_id = '%s|%s' % (input.name, case_id)
tour_id = f'{input.name}|{case_id}'
if tour_id in self._test.inputs.keys():
if case_id in self._data_inputs.keys():
hid = self._hids[case_id]
+6 -8
View File
@@ -23,14 +23,12 @@ THE ORIGINAL WORK IS WITH YOU.
Script for merging specific local Galaxy config galaxy.ini.cri with default Galaxy galaxy.ini.sample
'''
from __future__ import print_function
import configparser
import logging
import optparse
import sys
from six.moves import configparser
def main():
# logging configuration
@@ -51,12 +49,12 @@ def main():
config_sample = configparser.RawConfigParser()
config_sample.read(options.sample)
config_sample_content = open(options.sample, 'r').read()
config_sample_content = open(options.sample).read()
config = configparser.RawConfigParser()
config.read(options.config)
logging.info("Merging your own config file %s into the sample one %s." % (options.config, options.sample))
logging.info(f"Merging your own config file {options.config} into the sample one {options.sample}.")
logging.info("---------- DIFFERENCE ANALYSIS BEGIN ----------")
for section in config.sections():
if not config_sample.has_section(section):
@@ -65,13 +63,13 @@ def main():
for (name, value) in config.items(section):
if not config_sample.has_option(section, name):
if not "#%s" % name in config_sample_content:
logging.warning("-MISSING- section [%s] option '%s' not found in sample file. It will be ignored." % (section, name))
logging.warning(f"-MISSING- section [{section}] option '{name}' not found in sample file. It will be ignored.")
else:
logging.info("-notset- section [%s] option '%s' not set in sample file. It will be added." % (section, name))
logging.info(f"-notset- section [{section}] option '{name}' not set in sample file. It will be added.")
config_sample.set(section, name, value)
else:
if not config_sample.get(section, name) == value:
logging.info("- diff - section [%s] option '%s' has different value ('%s':'%s'). It will be modified." % (section, name, config_sample.get(section, name), value))
logging.info("- diff - section [{}] option '{}' has different value ('{}':'{}'). It will be modified.".format(section, name, config_sample.get(section, name), value))
config_sample.set(section, name, value)
logging.info("---------- DIFFERENCE ANALYSIS END ----------")
+9 -11
View File
@@ -3,7 +3,6 @@
check_galaxy can be run by hand, although it is meant to run from cron
via the check_galaxy.sh script in Galaxy's cron/ directory.
"""
from __future__ import print_function
import formatter
import getopt
@@ -14,13 +13,12 @@ import socket
import sys
import time
import warnings
from user import home
from six.moves.urllib.request import (
from urllib.request import (
build_opener,
HTTPCookieProcessor,
Request
Request,
)
from user import home
with warnings.catch_warnings():
warnings.simplefilter('ignore')
@@ -52,7 +50,7 @@ handler = args[3]
warntime = 240
new_history = False
for o, a in opts:
for o, _ in opts:
if o == "-n":
if debug:
print("Specified -n, will create a new history")
@@ -73,7 +71,7 @@ tc.agent("Mozilla/5.0 (compatible; check_galaxy/0.2)")
tc.config('use_tidy', 0)
class Browser(object):
class Browser:
def __init__(self):
self.server = server
self.handler = handler
@@ -93,7 +91,7 @@ class Browser(object):
self.opener = build_opener(HTTPCookieProcessor(tc.get_browser().cj))
def get(self, path):
tc.go("%s%s" % (self.server, path))
tc.go(f"{self.server}{path}")
tc.code(200)
def req(self, path, data=None, method=None):
@@ -105,7 +103,7 @@ class Browser(object):
if method:
req.get_method = lambda: method
res = self.opener.open(req)
print('==> at %s (%s)' % (url, method or 'GET'))
print('==> at {} ({})'.format(url, method or 'GET'))
assert res.getcode() == 200, url
return res
@@ -226,7 +224,7 @@ class Browser(object):
if self.hda_state != "ok":
self.get("/datasets/%s/stderr" % self.hda_id)
print(tc.browser.get_html())
raise Exception("HDA %s NOT OK: %s" % (self.hda_id, self.hda_state))
raise Exception(f"HDA {self.hda_id} NOT OK: {self.hda_state}")
def check_hda_content(self):
self.get("/datasets/%s/display?to_ext=txt" % self.hda_id)
@@ -240,7 +238,7 @@ class Browser(object):
def delete_datasets(self):
for hda in self.undeleted_hdas:
path = '/api/histories/%s/contents/%s' % (self.history_id, hda['id'])
path = '/api/histories/{}/contents/{}'.format(self.history_id, hda['id'])
self.req(path, method='DELETE')
hdas = [hda['id'] for hda in self.undeleted_hdas]
if hdas:
-1
View File
@@ -6,7 +6,6 @@ Adds Manually created builds and chrom info to Galaxy's info tables
Usage:
python add_manual_builds.py input_file builds.txt chrom_length_dir
"""
from __future__ import print_function
import os
import sys
+2 -3
View File
@@ -11,14 +11,13 @@ All chromInfo is placed in a path with the convention
Usage:
python build_chrom_db.py dbpath/ [builds_file]
"""
from __future__ import print_function
import fileinput
import os
import sys
from urllib.parse import urlencode
import requests
from six.moves.urllib.parse import urlencode
import parse_builds # noqa: I100,I202
@@ -78,5 +77,5 @@ if __name__ == "__main__":
for chrominfo in getchrominfo("http://genome-test.gi.ucsc.edu/cgi-bin/hgTables?", build):
print("\t".join(chrominfo), file=outfile)
except Exception as e:
print("Failed to retrieve %s: %s" % (build, e))
print(f"Failed to retrieve {build}: {e}")
os.remove(outfile_name)
-1
View File
@@ -4,7 +4,6 @@ Connects to the URL specified and outputs builds available at that
DSN in tabular format. UCSC Main gateway is used as default.
build description
"""
from __future__ import print_function
import sys
import xml.etree.ElementTree as ElementTree
-1
View File
@@ -2,7 +2,6 @@
"""
Connects to sites and determines which builds are available at each.
"""
from __future__ import print_function
import xml.etree.ElementTree as ElementTree
+7 -9
View File
@@ -1,17 +1,15 @@
# coding: utf-8
# TODO: Add examples, tables and best practice links to command
# TODO: Examples of truevalue, falsevalue
# TODO: Test param extra_file
# Things dropped from schema_template.md (still documented inside schema).
# - request_parameter_translation
from __future__ import print_function
import sys
from io import StringIO
from lxml import etree
from six import StringIO
with open(sys.argv[2], "r") as f:
with open(sys.argv[2]) as f:
xmlschema_doc = etree.parse(f)
markdown_buffer = StringIO()
@@ -22,7 +20,7 @@ def main():
toc_list = []
content_list = []
found_tag = False
with open(sys.argv[1], "r") as markdown_template:
with open(sys.argv[1]) as markdown_template:
for line in markdown_template:
if line.startswith("$tag:"):
found_tag = True
@@ -41,7 +39,7 @@ def main():
print(el, end='')
class Tag(object):
class Tag:
def __init__(self, line):
assert line.startswith("$tag:")
@@ -71,7 +69,7 @@ class Tag(object):
return " > ".join("``%s``" % p for p in self.title.split("|"))
def build_toc_entry(self):
return "* [%s](%s)" % (self._pretty_title, self._anchor)
return f"* [{self._pretty_title}]({self._anchor})"
def build_help(self):
tag = xmlschema_doc.find(self.xpath)
@@ -113,7 +111,7 @@ def _build_tag(tag, hide_attributes):
doc = _doc_or_none(_type_el(element))
assert doc is not None, "Documentation for %s is empty" % element.attrib["name"]
doc = doc.strip()
assertions_buffer.write("``%s`` | %s\n" % (element.attrib["name"], doc))
assertions_buffer.write("``{}`` | {}\n".format(element.attrib["name"], doc))
text = text.replace(line, assertions_buffer.getvalue())
tag_help.write(text)
best_practices = _get_bp_link(annotation_el)
@@ -165,7 +163,7 @@ def _build_attributes_table(tag, attributes, hide_attributes=False, attribute_na
if best_practices:
details += """ Find the Intergalactic Utilities Commision suggested best practices for this element [here](%s).""" % best_practices
attribute_table.write("``%s`` | %s | %s\n" % (name, details, use))
attribute_table.write(f"``{name}`` | {details} | {use}\n")
return attribute_table.getvalue()
+9 -10
View File
@@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
#
# Galaxy documentation build configuration file, created by
# sphinx-quickstart on Tue Mar 6 10:44:44 2012.
@@ -80,8 +79,8 @@ source_suffix = ['.rst', '.md']
master_doc = 'index'
# General information about the project.
project = u'Galaxy Project'
copyright = str(datetime.datetime.now().year) + u', Galaxy Committers'
project = 'Galaxy Project'
copyright = str(datetime.datetime.now().year) + ', Galaxy Committers'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
@@ -233,8 +232,8 @@ latex_elements = {
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
('index', 'Galaxy.tex', u'Galaxy Code Documentation',
u'Galaxy Team', 'manual'),
('index', 'Galaxy.tex', 'Galaxy Code Documentation',
'Galaxy Team', 'manual'),
]
# The name of an image file (relative to this directory) to place at the top of
@@ -263,8 +262,8 @@ latex_documents = [
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
('index', 'galaxy', u'Galaxy Documentation',
[u'Galaxy Team'], 1)
('index', 'galaxy', 'Galaxy Documentation',
['Galaxy Team'], 1)
]
# If true, show URL addresses after external links.
@@ -277,8 +276,8 @@ man_pages = [
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
('index', 'Galaxy', u'Galaxy Documentation',
u'Galaxy Team', 'Galaxy', 'Data intensive biology for everyone.',
('index', 'Galaxy', 'Galaxy Documentation',
'Galaxy Team', 'Galaxy', 'Data intensive biology for everyone.',
'Miscellaneous'),
]
@@ -293,7 +292,7 @@ texinfo_documents = [
# -- ReadTheDocs.org Settings ------------------------------------------------
class Mock(object):
class Mock:
def __init__(self, *args, **kwargs):
pass
-1
View File
@@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
#
# Galaxy sphinxcontrib-simpleversioning documentation build configuration file
#
+2 -2
View File
@@ -76,7 +76,7 @@ class AdminActions:
quota.description = params.description
self.sa_session.add(quota)
self.sa_session.flush()
message = "Quota '{}' has been renamed to '{}'.".format(old_name, params.name)
message = f"Quota '{old_name}' has been renamed to '{params.name}'."
return message
def _manage_users_and_groups_for_quota(self, quota, params, decode_id=None):
@@ -122,7 +122,7 @@ class AdminActions:
else:
if params.default != 'no':
self.app.quota_agent.set_default_quota(params.default, quota)
message = "Quota '{}' is now the default for {} users.".format(quota.name, params.default)
message = f"Quota '{quota.name}' is now the default for {params.default} users."
else:
if quota.default:
message = "Quota '{}' is no longer the default for {} users.".format(quota.name, quota.default[0].type)
+6 -6
View File
@@ -109,23 +109,23 @@ class PAM(AuthProvider):
pam_service = options.get('pam-service', 'galaxy')
use_helper = string_as_bool(options.get('use-external-helper', False))
log.debug("PAM auth: will use external helper: {}".format(use_helper))
log.debug(f"PAM auth: will use external helper: {use_helper}")
authenticated = False
if use_helper:
authentication_helper = options.get('authentication-helper-script', '/bin/false').strip()
log.debug("PAM auth: external helper script: {}".format(authentication_helper))
log.debug(f"PAM auth: external helper script: {authentication_helper}")
if not authentication_helper.startswith('/'):
# don't accept relative path
authenticated = False
else:
auth_cmd = shlex.split('/usr/bin/sudo -n {}'.format(authentication_helper))
log.debug("PAM auth: external helper cmd: {}".format(auth_cmd))
message = '{}\n{}\n{}\n'.format(pam_service, pam_username, password)
auth_cmd = shlex.split(f'/usr/bin/sudo -n {authentication_helper}')
log.debug(f"PAM auth: external helper cmd: {auth_cmd}")
message = f'{pam_service}\n{pam_username}\n{password}\n'
try:
output = commands.execute(auth_cmd, input=message)
except commands.CommandLineException as e:
if e.stderr != '':
log.debug("PAM auth: external authentication script had errors: status {} error {}".format(e.returncode, e.stderr))
log.debug(f"PAM auth: external authentication script had errors: status {e.returncode} error {e.stderr}")
output = e.stdout
if output.strip() == 'True':
authenticated = True
+1 -1
View File
@@ -86,7 +86,7 @@ def parse_auth_results(trans, auth_results, options):
i += 1
else:
raise Conflict("Cannot make unique username")
log.debug("Email: {}, auto-register with username: {}".format(auto_email, auto_username))
log.debug(f"Email: {auto_email}, auto-register with username: {auto_username}")
auth_return["auto_reg"] = string_as_bool(options.get('auto-register', False))
auth_return["email"] = auto_email
auth_return["username"] = auto_username
+4 -4
View File
@@ -3,12 +3,12 @@ import hashlib
import logging
import os
from datetime import datetime, timedelta
from urllib.parse import quote
import jwt
import requests
from oauthlib.common import generate_nonce
from requests_oauthlib import OAuth2Session
from six.moves.urllib.parse import quote
from galaxy import exceptions
from galaxy import util
@@ -248,16 +248,16 @@ class CustosAuthnz(IdentityProvider):
base_url = self.config["url"]
# Remove potential trailing slash to avoid "//realms"
base_url = base_url if base_url[-1] != "/" else base_url[:-1]
return "{}/.well-known/openid-configuration".format(base_url)
return f"{base_url}/.well-known/openid-configuration"
else:
raise Exception("Unknown Custos provider name: {}".format(provider))
raise Exception(f"Unknown Custos provider name: {provider}")
def _fetch_well_known_oidc_config(self, well_known_uri):
try:
return requests.get(well_known_uri,
verify=self._get_verify_param()).json()
except Exception:
log.error("Failed to load well-known OIDC config URI: {}".format(well_known_uri))
log.error(f"Failed to load well-known OIDC config URI: {well_known_uri}")
raise
def _load_well_known_oidc_config(self, well_known_oidc_config):
+11 -11
View File
@@ -1,3 +1,4 @@
import builtins
import copy
import json
import logging
@@ -10,7 +11,6 @@ from cloudauthz import CloudAuthz
from cloudauthz.exceptions import (
CloudAuthzBaseException
)
from six.moves import builtins
from galaxy import exceptions
from galaxy import model
@@ -89,7 +89,7 @@ class AuthnzManager:
except ImportError:
raise
except etree.ParseError as e:
raise etree.ParseError("Invalid configuration at `{}`: {} -- unable to continue.".format(config_file, e))
raise etree.ParseError(f"Invalid configuration at `{config_file}`: {e} -- unable to continue.")
def _get_idp_icon(self, idp):
return self.oidc_backends_config[idp].get('icon') or DEFAULT_OIDC_IDP_ICONS.get(idp)
@@ -109,7 +109,7 @@ class AuthnzManager:
"skipping the node.".format(child.tag))
continue
if 'name' not in child.attrib:
log.error("Could not find a node attribute 'name'; skipping the node '{}'.".format(child.tag))
log.error(f"Could not find a node attribute 'name'; skipping the node '{child.tag}'.")
continue
idp = child.get('name').lower()
if idp in BACKENDS_NAME:
@@ -127,7 +127,7 @@ class AuthnzManager:
except ImportError:
raise
except etree.ParseError as e:
raise etree.ParseError("Invalid configuration at `{}`: {} -- unable to continue.".format(config_file, e))
raise etree.ParseError(f"Invalid configuration at `{config_file}`: {e} -- unable to continue.")
def _parse_idp_config(self, config_xml):
rtv = {
@@ -188,10 +188,10 @@ class AuthnzManager:
else:
return True, "", identity_provider_class(unified_provider_name, self.oidc_config, self.oidc_backends_config[unified_provider_name])
except Exception as e:
log.exception('An error occurred when loading {}'.format(identity_provider_class.__name__))
log.exception(f'An error occurred when loading {identity_provider_class.__name__}')
return False, unicodify(e), None
else:
msg = 'The requested identity provider, `{}`, is not a recognized/expected provider.'.format(provider)
msg = f'The requested identity provider, `{provider}`, is not a recognized/expected provider.'
log.debug(msg)
return False, msg, None
@@ -281,13 +281,13 @@ class AuthnzManager:
return False, message, None
elif provider in KEYCLOAK_BACKENDS:
if (self.allowed_idps and (idphint not in self.allowed_idps)):
msg = 'An error occurred when authenticating a user. Invalid EntityID: `{}`'.format(idphint)
msg = f'An error occurred when authenticating a user. Invalid EntityID: `{idphint}`'
log.exception(msg)
return False, msg, None
return True, "Redirecting to the `{}` identity provider for authentication".format(provider), backend.authenticate(trans, idphint)
return True, "Redirecting to the `{}` identity provider for authentication".format(provider), backend.authenticate(trans)
return True, f"Redirecting to the `{provider}` identity provider for authentication", backend.authenticate(trans, idphint)
return True, f"Redirecting to the `{provider}` identity provider for authentication", backend.authenticate(trans)
except Exception:
msg = 'An error occurred when authenticating a user on `{}` identity provider'.format(provider)
msg = f'An error occurred when authenticating a user on `{provider}` identity provider'
log.exception(msg)
return False, msg, None
@@ -322,7 +322,7 @@ class AuthnzManager:
# check if logout is enabled for this idp and return false if not
unified_provider_name = self._unify_provider_name(provider)
if self.oidc_backends_config[unified_provider_name]['enable_idp_logout'] is False:
return False, "IDP logout is not enabled for {}".format(provider), None
return False, f"IDP logout is not enabled for {provider}", None
success, message, backend = self._get_authnz_backend(provider)
if success is False:
+5 -5
View File
@@ -316,11 +316,11 @@ def contains_required_data(response=None, is_new=False, **kwargs):
# sent back from the identity provider. PSA internally handles such
# scenarios; however, this case is implemented to prevent uncaught
# server-side errors.
raise MalformedContents(err_msg="`response` not found. {}".format(hint_msg))
raise MalformedContents(err_msg=f"`response` not found. {hint_msg}")
if not response.get("id_token"):
# This can happen if a non-OIDC compliant backend is used;
# e.g., an OAuth2.0-based backend that only generates access token.
raise MalformedContents(err_msg="Missing identity token. {}".format(hint_msg))
raise MalformedContents(err_msg=f"Missing identity token. {hint_msg}")
if is_new and not response.get("refresh_token"):
# An identity provider (e.g., Google) sends a refresh token the first
# time user consents Galaxy's access (i.e., the first time user logs in
@@ -331,7 +331,7 @@ def contains_required_data(response=None, is_new=False, **kwargs):
# user has provided consent. This can also happen under dev efforts.
# The solution is to revoke the consent by visiting the identity provider's
# website, and then retry the login process.
raise MalformedContents(err_msg="Missing refresh token. {}".format(hint_msg))
raise MalformedContents(err_msg=f"Missing refresh token. {hint_msg}")
def verify(strategy=None, response=None, details=None, **kwargs):
@@ -344,7 +344,7 @@ def verify(strategy=None, response=None, details=None, **kwargs):
if provider.lower() == "gcp":
result = requests.post(
"https://iam.googleapis.com/v1/projects/-/serviceAccounts/{}:getIamPolicy".format(endpoint),
f"https://iam.googleapis.com/v1/projects/-/serviceAccounts/{endpoint}:getIamPolicy",
headers={
'Authorization': 'Bearer {}'.format(response.get("access_token")),
'Accept': 'application/json'})
@@ -364,7 +364,7 @@ def verify(strategy=None, response=None, details=None, **kwargs):
# sensitive information that should not be exposed to users.
raise Exception(res["error"]["message"])
else:
raise Exception("`{}` is an unsupported secondary authorization provider, contact admin.".format(provider))
raise Exception(f"`{provider}` is an unsupported secondary authorization provider, contact admin.")
def allowed_to_disconnect(name=None, user=None, user_storage=None, strategy=None,
+1 -1
View File
@@ -4,6 +4,7 @@ Universe configuration builder.
# absolute_import needed for tool_shed package.
import collections
import configparser
import errno
import ipaddress
import logging
@@ -22,7 +23,6 @@ from datetime import timedelta
import yaml
from beaker.cache import CacheManager
from beaker.util import parse_cache_config_options
from six.moves import configparser
from galaxy.config.schema import AppSchema
from galaxy.containers import parse_containers_config
+9 -9
View File
@@ -8,12 +8,12 @@ from collections import (
namedtuple,
OrderedDict
)
from io import StringIO
from textwrap import TextWrapper
import requests
import yaml
from boltons.iterutils import remap
from six import StringIO
try:
from pykwalify.core import Core
@@ -405,7 +405,7 @@ def _to_rst(args, app_desc, heading_level="~"):
def _write_option_rst(args, rst, key, heading_level, option_value):
title = "``%s``" % key
heading = heading_level * len(title)
rst.write("{}\n{}\n{}\n\n".format(heading, title, heading))
rst.write(f"{heading}\n{title}\n{heading}\n\n")
option, value = _parse_option_value(option_value)
desc = option["desc"]
rst.write(":Description:\n")
@@ -649,7 +649,7 @@ def _is_ini(path):
def _replace_file(args, f, app_desc, from_path, to_path):
_write_to_file(args, f, to_path)
backup_path = "%s.backup" % from_path
print("Moving [{}] to [{}]".format(from_path, backup_path))
print(f"Moving [{from_path}] to [{backup_path}]")
if args.dry_run:
print("... skipping because --dry-run is enabled.")
else:
@@ -661,7 +661,7 @@ def _build_sample_yaml(args, app_desc):
UWSGI_OPTIONS.update(SHED_ONLY_UWSGI_OPTIONS)
schema = app_desc.schema
f = StringIO()
for key, value in UWSGI_OPTIONS.items():
for value in UWSGI_OPTIONS.values():
for field in ["desc", "default"]:
if field not in value:
continue
@@ -693,7 +693,7 @@ def _write_to_file(args, f, path):
contents = f
if args.dry_run:
contents_indented = "\n".join(" |%s" % l for l in contents.splitlines())
print("Overwriting {} with the following contents:\n{}".format(path, contents_indented))
print(f"Overwriting {path} with the following contents:\n{contents_indented}")
print("... skipping because --dry-run is enabled.")
else:
print("Overwriting %s" % path)
@@ -743,10 +743,10 @@ def _write_option(args, f, key, option_value, as_comment=False, uwsgi_hack=False
if uwsgi_hack:
if option.get("type", "str") == "bool":
value = str(value).lower()
key_val_str = "{}: {}".format(key, value)
key_val_str = f"{key}: {value}"
else:
key_val_str = yaml.dump({key: value}, width=float("inf")).lstrip("{").rstrip("\n}")
lines = "{}{}{}".format(comment, as_comment_str, key_val_str)
lines = f"{comment}{as_comment_str}{key_val_str}"
lines_idented = "\n".join(" %s" % l for l in lines.split("\n"))
f.write("%s\n\n" % lines_idented)
@@ -774,7 +774,7 @@ def _server_paste_to_uwsgi(app_desc, server_config, applied_filters):
if server_config.get("use", "egg:Paste#http") != "egg:Paste#http":
raise Exception("Unhandled paste server 'use' value [%s], file must be manually migrate.")
uwsgi_dict["http"] = "{}:{}".format(host, port)
uwsgi_dict["http"] = f"{host}:{port}"
# default changing from 10 to 8
uwsgi_dict["threads"] = int(server_config.get("threadpool_workers", 8))
# required for static...
@@ -791,7 +791,7 @@ def _server_paste_to_uwsgi(app_desc, server_config, applied_filters):
uwsgi_dict["http-auto-gzip"] = True
if prefix:
uwsgi_dict["mount"] = "{}={}".format(prefix, app_desc.uwsgi_module)
uwsgi_dict["mount"] = f"{prefix}={app_desc.uwsgi_module}"
uwsgi_dict["manage-script-name"] = True
else:
uwsgi_dict["module"] = app_desc.uwsgi_module
+1 -1
View File
@@ -84,7 +84,7 @@ class AppSchema(Schema):
def check_type_is_str_or_any(option, key):
if option.get('type') not in ('str', 'any'):
message = "Invalid schema: property '{}' should have type 'str'".format(key)
message = f"Invalid schema: property '{key}' should have type 'str'"
raise_error(message)
def check_is_dag():
+6 -7
View File
@@ -17,7 +17,6 @@ from abc import (
from collections import namedtuple
import yaml
from six.moves import shlex_quote
from galaxy.exceptions import ContainerCLIError
from galaxy.util.submodules import import_submodules
@@ -234,14 +233,14 @@ class ContainerInterface(metaclass=ABCMeta):
def _stringify_kwopt_string(self, flag, val):
"""
"""
return '{flag} {value}'.format(flag=flag, value=shlex_quote(str(val)))
return '{flag} {value}'.format(flag=flag, value=shlex.quote(str(val)))
def _stringify_kwopt_list(self, flag, val):
"""
"""
if isinstance(val, str):
return self._stringify_kwopt_string(flag, val)
return ' '.join('{flag} {value}'.format(flag=flag, value=shlex_quote(str(v))) for v in val)
return ' '.join('{flag} {value}'.format(flag=flag, value=shlex.quote(str(v))) for v in val)
def _stringify_kwopt_list_of_kvpairs(self, flag, val):
"""
@@ -253,7 +252,7 @@ class ContainerInterface(metaclass=ABCMeta):
else:
# {'foo': 'bar', 'baz': 'quux'}
for k, v in dict(val).items():
l.append('{k}={v}'.format(k=k, v=v))
l.append(f'{k}={v}')
return self._stringify_kwopt_list(flag, l)
def _stringify_kwopt_list_of_kovtrips(self, flag, val):
@@ -263,7 +262,7 @@ class ContainerInterface(metaclass=ABCMeta):
return self._stringify_kwopt_string(flag, val)
l = []
for k, o, v in val:
l.append('{k}{o}{v}'.format(k=k, o=o, v=v))
l.append(f'{k}{o}{v}')
return self._stringify_kwopt_list(flag, l)
def _run_command(self, command, verbose=False):
@@ -275,7 +274,7 @@ class ContainerInterface(metaclass=ABCMeta):
if p.returncode == 0:
return stdout.strip()
else:
msg = "Command '{}' returned non-zero exit status {}".format(command, p.returncode)
msg = f"Command '{command}' returned non-zero exit status {p.returncode}"
log.error(msg + ': ' + stderr.strip())
raise ContainerCLIError(
msg,
@@ -324,7 +323,7 @@ class ContainerInterfaceConfig(dict):
try:
return self[name]
except KeyError:
raise AttributeError("'{}' object has no attribute '{}'".format(self.__class__.__name__, name))
raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{name}'")
def get(self, name, default=None):
try:
+7 -7
View File
@@ -4,6 +4,7 @@ Interface to Docker
import logging
import os
import shlex
from functools import partial
from itertools import cycle, repeat
from time import sleep
@@ -18,7 +19,6 @@ try:
except ImportError:
ConnectionError = None
ReadTimeout = None
from six.moves import shlex_quote
from galaxy.containers import ContainerInterface
from galaxy.containers.docker_decorators import (
@@ -130,7 +130,7 @@ class DockerCLIInterface(DockerInterface):
global_kwopts = []
if self._conf.host:
global_kwopts.append('--host')
global_kwopts.append(shlex_quote(self._conf.host))
global_kwopts.append(shlex.quote(self._conf.host))
if self._conf.force_tlsverify:
global_kwopts.append('--tlsverify')
self._docker_command = self._conf['command_template'].format(
@@ -142,9 +142,9 @@ class DockerCLIInterface(DockerInterface):
def _filter_by_id_or_name(self, id, name):
if id:
return '--filter id={}'.format(id)
return f'--filter id={id}'
elif name:
return '--filter name={}'.format(name)
return f'--filter name={name}'
return None
def _stringify_kwopt_docker_volumes(self, flag, val):
@@ -159,7 +159,7 @@ class DockerCLIInterface(DockerInterface):
for hostvol, guestopts in val.items():
if isinstance(guestopts, str):
# {'/host/vol': '/container/vol'}
l.append('{}:{}'.format(hostvol, guestopts))
l.append(f'{hostvol}:{guestopts}')
else:
# {'/host/vol': {'bind': '/container/vol'}}
# {'/host/vol': {'bind': '/container/vol', 'mode': 'rw'}}
@@ -198,7 +198,7 @@ class DockerCLIInterface(DockerInterface):
return self._run_docker(subcommand='inspect', args=container_id)[0]
except (IndexError, ContainerCLIError) as exc:
msg = "Invalid container id: %s" % container_id
if exc.stdout == '[]' and exc.stderr == 'Error: no such object: {container_id}'.format(container_id=container_id):
if exc.stdout == '[]' and exc.stderr == f'Error: no such object: {container_id}':
log.warning(msg)
return []
else:
@@ -210,7 +210,7 @@ class DockerCLIInterface(DockerInterface):
return self._run_docker(subcommand='image inspect', args=image)[0]
except (IndexError, ContainerCLIError) as exc:
msg = "%s not pulled, cannot get digest" % image
if exc.stdout == '[]' and exc.stderr == 'Error: no such image: {image}'.format(image=image):
if exc.stdout == '[]' and exc.stderr == f'Error: no such image: {image}':
log.warning(msg, image)
return []
else:
+10 -10
View File
@@ -3,13 +3,13 @@ Model objects for docker objects
"""
import logging
import shlex
try:
import docker
except ImportError:
from galaxy.util.bunch import Bunch
docker = Bunch(errors=Bunch(NotFound=None))
from six.moves import shlex_quote
from galaxy.containers import (
Container,
@@ -106,7 +106,7 @@ class DockerVolume(ContainerVolume):
def __str__(self):
volume_str = ":".join(filter(lambda x: x is not None, (self.host_path, self.path, self.mode)))
if "$" not in volume_str:
volume_for_cmd_line = shlex_quote(volume_str)
volume_for_cmd_line = shlex.quote(volume_str)
else:
# e.g. $_GALAXY_JOB_TMP_DIR:$_GALAXY_JOB_TMP_DIR:rw so don't single quote.
volume_for_cmd_line = '"%s"' % volume_str
@@ -393,10 +393,10 @@ class DockerServiceConstraint:
return hash((self._name, self._op, self._value))
def __repr__(self):
return '{}({}{}{})'.format(self.__class__.__name__, self._name, self._op, self._value)
return f'{self.__class__.__name__}({self._name}{self._op}{self._value})'
def __str__(self):
return '{}{}{}'.format(self._name, self._op, self._value)
return f'{self._name}{self._op}{self._value}'
@staticmethod
def split_constraint_string(constraint_str):
@@ -508,7 +508,7 @@ class DockerNode:
@property
def state(self):
return ('{}-{}'.format(self._status, self._availability)).lower()
return (f'{self._status}-{self._availability}').lower()
@property
def cpus(self):
@@ -597,10 +597,10 @@ class DockerNodeLabel:
return hash((self._name, self._value))
def __repr__(self):
return '{}({}: {})'.format(self.__class__.__name__, self._name, self._value)
return f'{self.__class__.__name__}({self._name}: {self._value})'
def __str__(self):
return '{}: {}'.format(self._name, self._value)
return f'{self._name}: {self._value}'
@property
def name(self):
@@ -612,12 +612,12 @@ class DockerNodeLabel:
@property
def constraint_string(self):
return 'node.labels.{name}=={value}'.format(name=self.name, value=self.value)
return f'node.labels.{self.name}=={self.value}'
@property
def constraint(self):
return DockerServiceConstraint(
name='node.labels.{name}'.format(name=self.name),
name=f'node.labels.{self.name}',
op='==',
value=self.value
)
@@ -744,7 +744,7 @@ class DockerTask:
@property
def state(self):
return ('{}-{}'.format(self._desired_state, self._state)).lower()
return (f'{self._desired_state}-{self._state}').lower()
@property
def current_state(self):
+2 -2
View File
@@ -331,7 +331,7 @@ class DockerSwarmCLIInterface(DockerSwarmInterface, DockerCLIInterface):
@docker_columns
def service_ps(self, service_id):
return self._run_docker(subcommand='service ps', args='--no-trunc {}'.format(service_id))
return self._run_docker(subcommand='service ps', args=f'--no-trunc {service_id}')
def service_rm(self, service_ids):
service_ids = ' '.join(service_ids)
@@ -347,7 +347,7 @@ class DockerSwarmCLIInterface(DockerSwarmInterface, DockerCLIInterface):
@docker_columns
def node_ps(self, node_id):
return self._run_docker(subcommand='node ps', args='--no-trunc {}'.format(node_id))
return self._run_docker(subcommand='node ps', args=f'--no-trunc {node_id}')
def node_update(self, node_id, **kwopts):
return self._run_docker(subcommand='node update', args='{kwopts} {node_id}'.format(
+2 -2
View File
@@ -37,7 +37,7 @@ class AnvioComposite(Html):
missing_text = ''
if not os.path.exists(os.path.join(dataset.extra_files_path, composite_name)):
missing_text = ' (missing)'
rval.append('<li><a href="{}">{}</a>{}{}</li>'.format(composite_name, composite_name, opt_text, missing_text))
rval.append(f'<li><a href="{composite_name}">{composite_name}</a>{opt_text}{missing_text}</li>')
rval.append("</ul>")
defined_files = map(lambda x: x[0], defined_files)
extra_files = []
@@ -49,7 +49,7 @@ class AnvioComposite(Html):
if extra_files:
rval.append("<p/>This composite dataset contains these undefined files:<p/><ul>")
for rel_path in extra_files:
rval.append('<li><a href="{}">{}</a></li>'.format(rel_path, rel_path))
rval.append(f'<li><a href="{rel_path}">{rel_path}</a></li>')
rval.append('</ul>')
if not (defined_files or extra_files):
rval.append("<p/>This composite dataset does not contain any files!<p/><ul>")
+2 -2
View File
@@ -157,7 +157,7 @@ class Velvet(Html):
if composite_file.get('description'):
rval.append('<li><a href="{}" type="text/plain">{} ({})</a>{}</li>'.format(fn, fn, composite_file.get('description'), opt_text))
else:
rval.append('<li><a href="{}" type="text/plain">{}</a>{}</li>'.format(fn, fn, opt_text))
rval.append(f'<li><a href="{fn}" type="text/plain">{fn}</a>{opt_text}</li>')
rval.append('</ul></div></html>')
return "\n".join(rval)
@@ -204,7 +204,7 @@ class Velvet(Html):
if composite_file.get('description'):
rval.append('<li><a href="{}" type="text/plain">{} ({})</a>{}</li>'.format(fn, fn, composite_file.get('description'), opt_text))
else:
rval.append('<li><a href="{}" type="text/plain">{}</a>{}</li>'.format(fn, fn, opt_text))
rval.append(f'<li><a href="{fn}" type="text/plain">{fn}</a>{opt_text}</li>')
rval.append('</ul></div></html>')
with open(dataset.file_name, 'w') as f:
f.write("\n".join(rval))
+5 -5
View File
@@ -403,7 +403,7 @@ class BamNative(CompressedArchive):
# Galaxy display each tag as separate column because 'tostring()' funcition put tabs in between each tag of tags column.
# Below code will remove spaces between each tag.
bamline_modified = ('\t').join(bamline.split()[:11] + [(' ').join(bamline.split()[11:])])
ck_data = "{}\n{}".format(ck_data, bamline_modified)
ck_data = f"{ck_data}\n{bamline_modified}"
else:
# Nothing to enumerate; we've either offset to the end
# of the bamfile, or there is no data. (possible with
@@ -489,9 +489,9 @@ class Bam(BamNative):
# we start another process and discard stderr.
if index_flag == '-b':
# IOError: No such file or directory: '-b' if index_flag is set to -b (pysam 0.15.4)
cmd = ['python', '-c', "import pysam; pysam.set_verbosity(0); pysam.index('{}', '{}')".format(file_name, index_name)]
cmd = ['python', '-c', f"import pysam; pysam.set_verbosity(0); pysam.index('{file_name}', '{index_name}')"]
else:
cmd = ['python', '-c', "import pysam; pysam.set_verbosity(0); pysam.index('{}', '{}', '{}')".format(index_flag, file_name, index_name)]
cmd = ['python', '-c', f"import pysam; pysam.set_verbosity(0); pysam.index('{index_flag}', '{file_name}', '{index_name}')"]
with open(os.devnull, 'w') as devnull:
subprocess.check_call(cmd, stderr=devnull, shell=False)
needs_sorting = False
@@ -1711,7 +1711,7 @@ class CuffDiffSQlite(SQlite):
for gene_id, gene_name in result:
if gene_name is None:
continue
gene = '{}: {}'.format(gene_id, gene_name)
gene = f'{gene_id}: {gene_name}'
if gene not in genes:
genes.append(gene)
samples_query = 'SELECT DISTINCT(sample_name) as sample_name FROM samples ORDER BY sample_name'
@@ -1859,7 +1859,7 @@ class BlibSQlite(SQlite):
c = conn.cursor()
tables_query = "SELECT majorVersion,minorVersion FROM LibInfo"
(majorVersion, minorVersion) = c.execute(tables_query).fetchall()[0]
dataset.metadata.blib_version = '{}.{}'.format(majorVersion, minorVersion)
dataset.metadata.blib_version = f'{majorVersion}.{minorVersion}'
except Exception as e:
log.warning('%s, set_meta Exception: %s', self, e)
+2 -2
View File
@@ -147,7 +147,7 @@ class BlastXml(GenericXml):
raise ValueError("The header in BLAST XML file %s is too long" % f)
if "<BlastOutput>" not in header:
h.close()
raise ValueError("{} is not a BLAST XML file:\n{}\n...".format(f, header))
raise ValueError(f"{f} is not a BLAST XML file:\n{header}\n...")
if f == split_files[0]:
out.write(header)
old_header = header
@@ -223,7 +223,7 @@ class _BlastDb(Data):
if not msg:
msg = title
# Galaxy assumes HTML for the display of composite datatypes,
return smart_str("<html><head><title>{}</title></head><body><pre>{}</pre></body></html>".format(title, msg))
return smart_str(f"<html><head><title>{title}</title></head><body><pre>{msg}</pre></body></html>")
def merge(split_files, output_file):
"""Merge BLAST databases (not implemented for now)."""
@@ -34,7 +34,7 @@ def main():
current_line += 1
if 0 == current_line % lines_per_chunk:
chunk_end = in_file.tell()
out_file.write('{{"start":"{}","end":"{}","sequences":"{}"}},'.format(chunk_begin, chunk_end, sequences))
out_file.write(f'{{"start":"{chunk_begin}","end":"{chunk_end}","sequences":"{sequences}"}},')
chunk_begin = chunk_end
line = in_file.readline()
@@ -41,7 +41,7 @@ def rgConv(inpedfilepath, outhtmlname, outfilepath):
try:
mf = open(inmap)
except Exception:
sys.exit('{} cannot open inmap file {} - do you have permission?\n'.format(prog, inmap))
sys.exit(f'{prog} cannot open inmap file {inmap} - do you have permission?\n')
try:
rsl = [x.split()[1] for x in mf]
except Exception:
@@ -94,8 +94,8 @@ def main():
flist = os.listdir(outfilepath)
with open(outhtmlname, 'w') as f:
f.write(galhtmlprefix % prog)
print('## Rgenetics: http://rgenetics.org Galaxy Tools {} {}'.format(prog, timenow())) # becomes info
f.write('<div>## Rgenetics: http://rgenetics.org Galaxy Tools {} {}\n<ol>'.format(prog, timenow()))
print(f'## Rgenetics: http://rgenetics.org Galaxy Tools {prog} {timenow()}') # becomes info
f.write(f'<div>## Rgenetics: http://rgenetics.org Galaxy Tools {prog} {timenow()}\n<ol>')
for i, data in enumerate(flist):
f.write('<li><a href="{}">{}</a></li>\n'.format(os.path.split(data)[-1], os.path.split(data)[-1]))
f.write("</div></body></html>")
@@ -99,7 +99,7 @@ def main():
flist = os.listdir(outfilepath)
with open(outhtmlname, 'w') as f:
f.write(galhtmlprefix % prog)
s = '## Rgenetics: http://rgenetics.org Galaxy Tools {} {}'.format(prog, timenow()) # becomes info
s = f'## Rgenetics: http://rgenetics.org Galaxy Tools {prog} {timenow()}' # becomes info
print(s)
f.write('<div>%s\n<ol>' % (s))
for i, data in enumerate(flist):
@@ -60,8 +60,8 @@ def makeLDreduced(basename, infpath=None, outfpath=None, plinke='plink', forcere
inbase = os.path.join(infpath)
plinktasks = []
vclbase = [plinke, '--noweb']
plinktasks += [['--bfile', inbase, '--indep-pairwise {} {} {}'.format(winsize, winmove, r2thresh), '--out %s' % outbase],
['--bfile', inbase, '--extract {}.prune.in --make-bed --out {}'.format(outbase, outbase)]]
plinktasks += [['--bfile', inbase, f'--indep-pairwise {winsize} {winmove} {r2thresh}', '--out %s' % outbase],
['--bfile', inbase, f'--extract {outbase}.prune.in --make-bed --out {outbase}']]
vclbase = [plinke, '--noweb']
pruneLD(plinktasks=plinktasks, cd=outfpath, vclbase=vclbase)
@@ -99,10 +99,10 @@ def main():
flist = os.listdir(outfilepath)
with open(outhtmlname, 'w') as f:
f.write(galhtmlprefix % prog)
s1 = '## Rgenetics: http://rgenetics.org Galaxy Tools {} {}'.format(prog, timenow()) # becomes info
s2 = 'Input {}, winsize={}, winmove={}, r2thresh={}'.format(base_name, winsize, winmove, r2thresh)
print('{} {}'.format(s1, s2))
f.write('<div>{}\n{}\n<ol>'.format(s1, s2))
s1 = f'## Rgenetics: http://rgenetics.org Galaxy Tools {prog} {timenow()}' # becomes info
s2 = f'Input {base_name}, winsize={winsize}, winmove={winmove}, r2thresh={r2thresh}'
print(f'{s1} {s2}')
f.write(f'<div>{s1}\n{s2}\n<ol>')
for i, data in enumerate(flist):
f.write('<li><a href="{}">{}</a></li>\n'.format(os.path.split(data)[-1], os.path.split(data)[-1]))
f.write("</div></body></html>")
@@ -65,7 +65,7 @@ def main():
flist = os.listdir(outfilepath)
with open(outhtmlname, 'w') as f:
f.write(galhtmlprefix % prog)
s = '## Rgenetics: http://bitbucket.org/rgalaxy Galaxy Tools {} {}'.format(prog, timenow()) # becomes info
s = f'## Rgenetics: http://bitbucket.org/rgalaxy Galaxy Tools {prog} {timenow()}' # becomes info
print(s)
f.write('<div>%s\n<ol>' % (s))
for i, data in enumerate(flist):
+9 -9
View File
@@ -65,7 +65,7 @@ class DatatypeValidation:
return DatatypeValidation("unknown", "Dataset validation unimplemented for this datatype.")
def __repr__(self):
return "DatatypeValidation[state={},message={}]".format(self.state, self.message)
return f"DatatypeValidation[state={self.state},message={self.message}]"
def validate(dataset_instance):
@@ -313,7 +313,7 @@ class Data(metaclass=DataMeta):
display_name = os.path.splitext(outfname)[0]
if not display_name.endswith(ext):
display_name = '{}_{}'.format(display_name, ext)
display_name = f'{display_name}_{ext}'
error, msg = self._archive_main_file(archive, display_name, path)[:2]
if not error:
@@ -339,7 +339,7 @@ class Data(metaclass=DataMeta):
outext = 'tgz'
if do_action == 'tbz':
outext = 'tbz'
trans.response.headers["Content-Disposition"] = 'attachment; filename="{}.{}"'.format(outfname, outext)
trans.response.headers["Content-Disposition"] = f'attachment; filename="{outfname}.{outext}"'
archive.wsgi_status = trans.response.wsgi_status()
archive.wsgi_headeritems = trans.response.wsgi_headeritems()
return archive.stream
@@ -438,7 +438,7 @@ class Data(metaclass=DataMeta):
self._clean_and_set_mime_type(trans, mime)
return self._yield_user_file_content(trans, data, file_path)
else:
return webob.exc.HTTPNotFound("Could not find '{}' on the extra files path {}.".format(filename, file_path))
return webob.exc.HTTPNotFound(f"Could not find '{filename}' on the extra files path {file_path}.")
self._clean_and_set_mime_type(trans, data.get_mime())
trans.log_event("Display dataset id: %s" % str(data.id))
@@ -627,7 +627,7 @@ class Data(metaclass=DataMeta):
return getattr(self, self.supported_display_apps[type]['file_function'])(dataset, **kwd)
except Exception:
log.exception('Function %s is referred to in datatype %s for displaying as type %s, but is not accessible', self.supported_display_apps[type]['file_function'], self.__class__.__name__, type)
return "This display type ({}) is not implemented for this datatype ({}).".format(type, dataset.ext)
return f"This display type ({type}) is not implemented for this datatype ({dataset.ext})."
def get_display_links(self, dataset, type, app, base_url, target_frame='_blank', **kwd):
"""
@@ -657,7 +657,7 @@ class Data(metaclass=DataMeta):
converter = trans.app.datatypes_registry.get_converter_by_target_type(original_dataset.ext, target_type)
if converter is None:
raise Exception("A converter does not exist for {} to {}.".format(original_dataset.ext, target_type))
raise Exception(f"A converter does not exist for {original_dataset.ext} to {target_type}.")
# Generate parameter dictionary
params = {}
# determine input parameter name and add to params
@@ -683,7 +683,7 @@ class Data(metaclass=DataMeta):
value.visible = False
if return_output:
return converted_dataset
return "The file conversion of {} on data {} has been added to the Queue.".format(converter.name, original_dataset.hid)
return f"The file conversion of {converter.name} on data {original_dataset.hid} has been added to the Queue."
# We need to clear associated files before we set metadata
# so that as soon as metadata starts to be set, e.g. implicitly converted datasets are deleted and no longer available 'while' metadata is being set, not just after
@@ -856,7 +856,7 @@ class Text(Data):
sample_lines = dataset_read.count('\n')
est_lines = int(sample_lines * (float(dataset.get_size()) / float(sample_size)))
except UnicodeDecodeError:
log.error('Unable to estimate lines in file {}'.format(dataset.file_name))
log.error(f'Unable to estimate lines in file {dataset.file_name}')
est_lines = None
return est_lines
@@ -880,7 +880,7 @@ class Text(Data):
if line and not line.startswith('#'):
data_lines += 1
except UnicodeDecodeError:
log.error('Unable to count lines in file {}'.format(dataset.file_name))
log.error(f'Unable to count lines in file {dataset.file_name}')
data_lines = None
return data_lines
+1 -2
View File
@@ -4,8 +4,7 @@ is further subdivided into multiple data (e.g. columns from a line).
"""
import logging
import re
from six.moves.urllib.parse import unquote_plus
from urllib.parse import unquote_plus
from . import line
@@ -17,8 +17,7 @@ DataProvider related decorators.
import copy
import logging
from functools import wraps
from six.moves.urllib.parse import unquote
from urllib.parse import unquote
log = logging.getLogger(__name__)
@@ -104,8 +103,8 @@ def dataprovider_factory(name, settings=None):
def named_dataprovider_factory(func):
setattr(func, _DATAPROVIDER_METHOD_NAME_KEY, name)
setattr(func, 'parse_query_string_settings', parse_query_string_settings)
setattr(func, 'settings', settings)
func.parse_query_string_settings = parse_query_string_settings
func.settings = settings
# TODO: I want a way to inherit settings from the previous provider( this_name ) instead of defining over and over
@wraps(func)
@@ -6,9 +6,11 @@ import gzip
import logging
import subprocess
import tempfile
from six.moves.urllib.parse import urlencode, urlparse
from six.moves.urllib.request import urlopen
from urllib.parse import (
urlencode,
urlparse,
)
from urllib.request import urlopen
from . import (
base,
@@ -2,8 +2,7 @@
import logging
from collections import OrderedDict
from copy import deepcopy
from six.moves.urllib.parse import quote_plus
from urllib.parse import quote_plus
from galaxy.util import (
parse_xml,
@@ -192,7 +191,7 @@ class DynamicDisplayApplicationBuilder:
# now populate
links.append(DisplayApplicationLink.from_elem(new_elem, display_application, other_values=dynamic_values))
else:
log.warning('Invalid dynamic display application link specified in {}: "{}"'.format(filename, line))
log.warning(f'Invalid dynamic display application link specified in {filename}: "{line}"')
self.links = links
def __iter__(self):
@@ -1,7 +1,6 @@
# Contains parameters that are used in Display Applications
import mimetypes
from six.moves.urllib.parse import quote_plus
from urllib.parse import quote_plus
from galaxy.util import string_as_bool
from galaxy.util.bunch import Bunch
@@ -83,7 +82,7 @@ class DisplayApplicationDataParameter(DisplayApplicationParameter):
data = data.value
if self.metadata:
rval = getattr(data.metadata, self.metadata, None)
assert rval, 'Unknown metadata name ({}) provided for dataset type ({}).'.format(self.metadata, data.datatype.__class__.name)
assert rval, f'Unknown metadata name ({self.metadata}) provided for dataset type ({data.datatype.__class__.name}).'
return Bunch(file_name=rval.file_name, state=data.state, states=data.states, extension='data')
elif self.extensions and (self.force_conversion or not isinstance(data.datatype, self.formats)):
for ext in self.extensions:
+10 -10
View File
@@ -15,9 +15,9 @@ import logging
import os
import re
import sys
from urllib.parse import quote_plus
from markupsafe import escape
from six.moves.urllib.parse import quote_plus
from galaxy.datatypes import metadata
from galaxy.datatypes.data import (
@@ -107,7 +107,7 @@ class GenomeGraphs(Tabular):
display_url = quote_plus(display_url)
# was display_url = quote_plus( "%s/display_as?id=%i&display_app=%s" % (base_url, dataset.id, type) )
# redirect_url = quote_plus( "%sdb=%s&position=%s:%s-%s&hgt.customText=%%s" % (site_url, dataset.dbkey, chrom, start, stop) )
sl = ["{}db={}".format(site_url, dataset.dbkey), ]
sl = [f"{site_url}db={dataset.dbkey}", ]
# sl.append("&hgt.customText=%s")
sl.append("&hgGenome_dataSetName={}&hgGenome_dataSetDescription={}".format(dataset.name, 'GalaxyGG_data'))
sl.append("&hgGenome_formatType=best guess&hgGenome_markerType=best guess")
@@ -117,7 +117,7 @@ class GenomeGraphs(Tabular):
s = ''.join(sl)
s = quote_plus(s)
redirect_url = s
link = '{}?redirect_url={}&display_url={}'.format(internal_url, redirect_url, display_url)
link = f'{internal_url}?redirect_url={redirect_url}&display_url={display_url}'
ret_val.append((site_name, link))
return ret_val
@@ -286,7 +286,7 @@ class Rgenetics(Html):
if composite_file.get('description'):
rval.append('<li><a href="{}" type="application/binary">{} ({})</a>{}</li>'.format(fn, fn, composite_file.get('description'), opt_text))
else:
rval.append('<li><a href="{}" type="application/binary">{}</a>{}</li>'.format(fn, fn, opt_text))
rval.append(f'<li><a href="{fn}" type="application/binary">{fn}</a>{opt_text}</li>')
rval.append('</ul></div></html>')
return "\n".join(rval)
@@ -296,11 +296,11 @@ class Rgenetics(Html):
"""
efp = dataset.extra_files_path
flist = os.listdir(efp)
rval = ['<html><head><title>Files for Composite Dataset {}</title></head><body><p/>Composite {} contains:<p/><ul>'.format(dataset.name, dataset.name)]
rval = [f'<html><head><title>Files for Composite Dataset {dataset.name}</title></head><body><p/>Composite {dataset.name} contains:<p/><ul>']
for fname in flist:
sfname = os.path.split(fname)[-1]
f, e = os.path.splitext(fname)
rval.append('<li><a href="{}">{}</a></li>'.format(sfname, sfname))
rval.append(f'<li><a href="{sfname}">{sfname}</a></li>')
rval.append('</ul></body></html>')
with open(dataset.file_name, 'w') as f:
f.write("\n".join(rval))
@@ -334,7 +334,7 @@ class Rgenetics(Html):
return False
if len(flist) == 0:
if verbose:
gal_Log.debug('@@@rgenetics set_meta failed - {} efp {} is empty?'.format(dataset.name, efp))
gal_Log.debug(f'@@@rgenetics set_meta failed - {dataset.name} efp {efp} is empty?')
return False
self.regenerate_primary_file(dataset)
if not dataset.info:
@@ -555,7 +555,7 @@ class IdeasPre(Html):
rval.append('<ul>')
for composite_name in self.get_composite_files(dataset=dataset).keys():
fn = composite_name
rval.append('<li><a href="{}>{}</a></li>'.format(fn, fn))
rval.append(f'<li><a href="{fn}>{fn}</a></li>')
rval.append('</ul></body></html>\n')
return "\n".join(rval)
@@ -566,7 +566,7 @@ class IdeasPre(Html):
rval.append('<ul>')
for fname in os.listdir(dataset.extra_files_path):
fn = os.path.split(fname)[-1]
rval.append('<li><a href="{}">{}</a></li>'.format(fn, fn))
rval.append(f'<li><a href="{fn}">{fn}</a></li>')
rval.append('</ul></body></html>')
with open(dataset.file_name, 'w') as f:
f.write("\n".join(rval))
@@ -752,7 +752,7 @@ class RexpBase(Html):
rval = ['<html><head><title>Files for Composite Dataset %s</title></head><p/>Comprises the following files:<p/><ul>' % (bn)]
for fname in flist:
sfname = os.path.split(fname)[-1]
rval.append('<li><a href="{}">{}</a>'.format(sfname, sfname))
rval.append(f'<li><a href="{sfname}">{sfname}</a>')
rval.append('</ul></html>')
with open(dataset.file_name, 'w') as f:
f.write("\n".join(rval))
+1 -1
View File
@@ -42,7 +42,7 @@ class Shapefile(Binary):
if composite_file.get('description'):
rval.append('<li><a href="{}" type="application/binary">{} ({})</a>{}</li>'.format(fn, fn, composite_file.get('description'), opt_text))
else:
rval.append('<li><a href="{}" type="application/binary">{}</a>{}</li>'.format(fn, fn, opt_text))
rval.append(f'<li><a href="{fn}" type="application/binary">{fn}</a>{opt_text}</li>')
rval.append('</ul></div></html>\n')
return "\n".join(rval)
+5 -6
View File
@@ -4,8 +4,7 @@ Image classes
import base64
import logging
import zipfile
from six.moves.urllib.parse import quote_plus
from urllib.parse import quote_plus
from galaxy.datatypes.text import Html as HtmlFromText
from galaxy.util import nice_size
@@ -52,7 +51,7 @@ class Image(data.Data):
name = hda.name or ''
with open(dataset.file_name, "rb") as f:
base64_image_data = base64.b64encode(f.read()).decode("utf-8")
return "![{}](data:image/{};base64,{})".format(name, self.file_ext, base64_image_data)
return f"![{name}](data:image/{self.file_ext};base64,{base64_image_data})"
class Jpg(Image):
@@ -177,14 +176,14 @@ def create_applet_tag_peek(class_name, archive, params):
height="30" width="200" align="center" >
<param name="archive" value="{}"/>""".format(class_name, archive)
for name, value in params.items():
text += """<param name="{}" value="{}"/>""".format(name, value)
text += f"""<param name="{name}" value="{value}"/>"""
text += """
<object classid="clsid:8AD9C840-044E-11D1-B3E9-00805F499D93"
height="30" width="200" >
<param name="code" value="{}" />
<param name="archive" value="{}"/>""".format(class_name, archive)
for name, value in params.items():
text += """<param name="{}" value="{}"/>""".format(name, value)
text += f"""<param name="{name}" value="{value}"/>"""
text += """<div class="errormessage">You must install and enable Java in your browser in order to access this applet.<div></object>
</object>
"""
@@ -265,7 +264,7 @@ class Laj(data.Text):
"alignfile1": "display?id=%s" % dataset.id,
"buttonlabel": "Launch LAJ",
"title": "LAJ in Galaxy",
"posturl": quote_plus("history_add_to?%s" % "&".join("{}={}".format(key, value) for key, value in {'history_id': dataset.history_id, 'ext': 'lav', 'name': 'LAJ Output', 'info': 'Added by LAJ', 'dbkey': dataset.dbkey, 'copy_access_from': dataset.id}.items())),
"posturl": quote_plus("history_add_to?%s" % "&".join(f"{key}={value}" for key, value in {'history_id': dataset.history_id, 'ext': 'lav', 'name': 'LAJ Output', 'info': 'Added by LAJ', 'dbkey': dataset.dbkey, 'copy_access_from': dataset.id}.items())),
"noseq": "true"
}
class_name = "edu.psu.cse.bio.laj.LajApplet.class"
+9 -9
View File
@@ -5,9 +5,9 @@ import logging
import math
import sys
import tempfile
from urllib.parse import quote_plus
from bx.intervals.io import GenomicIntervalReader, ParseError
from six.moves.urllib.parse import quote_plus
from galaxy import util
from galaxy.datatypes import metadata
@@ -264,7 +264,7 @@ class Interval(Tabular):
(base_url, app.url_for(controller='root'), dataset.id, type))
redirect_url = quote_plus("%sdb=%s&position=%s:%s-%s&hgt.customText=%%s" %
(site_url, dataset.dbkey, chrom, start, stop))
link = '{}?redirect_url={}&display_url={}'.format(internal_url, redirect_url, display_url)
link = f'{internal_url}?redirect_url={redirect_url}&display_url={display_url}'
ret_val.append((site_name, link))
return ret_val
@@ -604,11 +604,11 @@ class _RemoteCallMixin:
the data available, followed by redirecting to the remote site with a
link back to the available information.
"""
internal_url = "%s" % app.url_for(controller='dataset', dataset_id=dataset.id, action='display_at', filename='{}_{}'.format(type, site_name))
internal_url = "%s" % app.url_for(controller='dataset', dataset_id=dataset.id, action='display_at', filename=f'{type}_{site_name}')
base_url = app.config.get("display_at_callback", base_url)
display_url = quote_plus("%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" %
(base_url, app.url_for(controller='root'), dataset.id, type))
link = '{}?redirect_url={}&display_url={}'.format(internal_url, redirect_url, display_url)
link = f'{internal_url}?redirect_url={redirect_url}&display_url={display_url}'
return link
@@ -794,7 +794,7 @@ class Gff(Tabular, _RemoteCallMixin):
if site_name in app.datatypes_registry.get_display_sites('gbrowse'):
if seqid.startswith('chr') and len(seqid) > 3:
seqid = seqid[3:]
redirect_url = quote_plus("{}/?q={}:{}..{}&eurl=%s".format(site_url, seqid, start, stop))
redirect_url = quote_plus(f"{site_url}/?q={seqid}:{start}..{stop}&eurl=%s")
link = self._get_remote_call_url(redirect_url, site_name, dataset, type, app, base_url)
ret_val.append((site_name, link))
return ret_val
@@ -1146,7 +1146,7 @@ class Wiggle(Tabular, _RemoteCallMixin):
if site_name in app.datatypes_registry.get_display_sites('gbrowse'):
if chrom.startswith('chr') and len(chrom) > 3:
chrom = chrom[3:]
redirect_url = quote_plus("{}/?q={}:{}..{}&eurl=%s".format(site_url, chrom, start, stop))
redirect_url = quote_plus(f"{site_url}/?q={chrom}:{start}..{stop}&eurl=%s")
link = self._get_remote_call_url(redirect_url, site_name, dataset, type, app, base_url)
ret_val.append((site_name, link))
return ret_val
@@ -1157,7 +1157,7 @@ class Wiggle(Tabular, _RemoteCallMixin):
if chrom is not None:
for site_name, site_url in app.datatypes_registry.get_legacy_sites_by_build('ucsc', dataset.dbkey):
if site_name in app.datatypes_registry.get_display_sites('ucsc'):
redirect_url = quote_plus("{}db={}&position={}:{}-{}&hgt.customText=%s".format(site_url, dataset.dbkey, chrom, start, stop))
redirect_url = quote_plus(f"{site_url}db={dataset.dbkey}&position={chrom}:{start}-{stop}&hgt.customText=%s")
link = self._get_remote_call_url(redirect_url, site_name, dataset, type, app, base_url)
ret_val.append((site_name, link))
return ret_val
@@ -1324,8 +1324,8 @@ class CustomTrack(Tabular):
if site_name in app.datatypes_registry.get_display_sites('ucsc'):
internal_url = "%s" % app.url_for(controller='dataset', dataset_id=dataset.id, action='display_at', filename='ucsc_' + site_name)
display_url = quote_plus("%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" % (base_url, app.url_for(controller='root'), dataset.id, type))
redirect_url = quote_plus("{}db={}&position={}:{}-{}&hgt.customText=%s".format(site_url, dataset.dbkey, chrom, start, stop))
link = '{}?redirect_url={}&display_url={}'.format(internal_url, redirect_url, display_url)
redirect_url = quote_plus(f"{site_url}db={dataset.dbkey}&position={chrom}:{start}-{stop}&hgt.customText=%s")
link = f'{internal_url}?redirect_url={redirect_url}&display_url={display_url}'
ret_val.append((site_name, link))
return ret_val
+1 -1
View File
@@ -274,7 +274,7 @@ class _Isa(data.Data):
</body></html>"""
else:
html = '<html><body>'
html += '<h1>{} {}</h1>'.format(investigation.title, investigation.identifier)
html += f'<h1>{investigation.title} {investigation.identifier}</h1>'
# Loop on all studies
for study in investigation.studies:
+2 -2
View File
@@ -38,9 +38,9 @@ class GenericMicroarrayFile(data.Text):
def set_peek(self, dataset, is_multi_byte=False):
if not dataset.dataset.purged:
if dataset.metadata.block_count == 1:
dataset.blurb = "{} {}: Format {}, 1 block, {} headers and {} columns".format(dataset.metadata.file_type, dataset.metadata.version_number, dataset.metadata.file_format, dataset.metadata.number_of_optional_header_records, dataset.metadata.number_of_data_columns)
dataset.blurb = f"{dataset.metadata.file_type} {dataset.metadata.version_number}: Format {dataset.metadata.file_format}, 1 block, {dataset.metadata.number_of_optional_header_records} headers and {dataset.metadata.number_of_data_columns} columns"
else:
dataset.blurb = "{} {}: Format {}, {} blocks, {} headers and {} columns".format(dataset.metadata.file_type, dataset.metadata.version_number, dataset.metadata.file_format, dataset.metadata.block_count, dataset.metadata.number_of_optional_header_records, dataset.metadata.number_of_data_columns)
dataset.blurb = f"{dataset.metadata.file_type} {dataset.metadata.version_number}: Format {dataset.metadata.file_format}, {dataset.metadata.block_count} blocks, {dataset.metadata.number_of_optional_header_records} headers and {dataset.metadata.number_of_data_columns} columns"
dataset.peek = get_file_peek(dataset.file_name)
else:
dataset.peek = 'file does not exist'
+2 -2
View File
@@ -514,7 +514,7 @@ class PDB(GenericMolFile):
hetatm_numbers = count_special_lines("^HETATM", dataset.file_name)
chain_ids = ','.join(dataset.metadata.chain_ids) if len(dataset.metadata.chain_ids) > 0 else 'None'
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "{} atoms and {} HET-atoms\nchain_ids: {}".format(atom_numbers, hetatm_numbers, chain_ids)
dataset.blurb = f"{atom_numbers} atoms and {hetatm_numbers} HET-atoms\nchain_ids: {chain_ids}"
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disk'
@@ -565,7 +565,7 @@ class PDBQT(GenericMolFile):
root_numbers = count_special_lines("^ROOT", dataset.file_name)
branch_numbers = count_special_lines("^BRANCH", dataset.file_name)
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "{} roots and {} branches".format(root_numbers, branch_numbers)
dataset.blurb = f"{root_numbers} roots and {branch_numbers} branches"
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disk'
+1 -1
View File
@@ -36,7 +36,7 @@ class BowtieIndex(Html):
rval = ['<html><head><title>Files for Composite Dataset %s</title></head><p/>Comprises the following files:<p/><ul>' % (bn)]
for fname in flist:
sfname = os.path.split(fname)[-1]
rval.append('<li><a href="{}">{}</a>'.format(sfname, sfname))
rval.append(f'<li><a href="{sfname}">{sfname}</a>')
rval.append('</ul></html>')
with open(dataset.file_name, 'w') as f:
f.write("\n".join(rval))
+4 -4
View File
@@ -48,7 +48,7 @@ class Wiff(Binary):
if composite_file.get('description'):
rval.append('<li><a href="{}" type="text/plain">{} ({})</a>{}</li>'.format(fn, fn, composite_file.get('description'), opt_text))
else:
rval.append('<li><a href="{}" type="text/plain">{}</a>{}</li>'.format(fn, fn, opt_text))
rval.append(f'<li><a href="{fn}" type="text/plain">{fn}</a>{opt_text}</li>')
rval.append('</ul></div></html>')
return "\n".join(rval)
@@ -900,7 +900,7 @@ class SPLib(Msp):
if composite_file.get('description'):
rval.append('<li><a href="{}" type="text/plain">{} ({})</a>{}</li>'.format(fn, fn, composite_file.get('description'), opt_text))
else:
rval.append('<li><a href="{}" type="text/plain">{}</a>{}</li>'.format(fn, fn, opt_text))
rval.append(f'<li><a href="{fn}" type="text/plain">{fn}</a>{opt_text}</li>')
rval.append('</ul></div></html>')
return "\n".join(rval)
@@ -995,7 +995,7 @@ class ImzML(Binary):
if composite_file.get('description'):
rval.append('<li><a href="{}" type="text/plain">{} ({})</a>{}</li>'.format(fn, fn, composite_file.get('description'), opt_text))
else:
rval.append('<li><a href="{}" type="text/plain">{}</a>{}</li>'.format(fn, fn, opt_text))
rval.append(f'<li><a href="{fn}" type="text/plain">{fn}</a>{opt_text}</li>')
rval.append('</ul></div></html>')
return "\n".join(rval)
@@ -1041,6 +1041,6 @@ class Analyze75(Binary):
if composite_file.get('description'):
rval.append('<li><a href="{}" type="text/plain">{} ({})</a>{}</li>'.format(fn, fn, composite_file.get('description'), opt_text))
else:
rval.append('<li><a href="{}" type="text/plain">{}</a>{}</li>'.format(fn, fn, opt_text))
rval.append(f'<li><a href="{fn}" type="text/plain">{fn}</a>{opt_text}</li>')
rval.append('</ul></div></html>')
return "\n".join(rval)
+8 -8
View File
@@ -330,7 +330,7 @@ class Registry:
self.datatype_info_dicts.append(datatype_info_dict)
for auto_compressed_type in auto_compressed_types:
compressed_extension = "{}.{}".format(extension, auto_compressed_type)
compressed_extension = f"{extension}.{auto_compressed_type}"
upper_compressed_type = auto_compressed_type[0].upper() + auto_compressed_type[1:]
auto_compressed_type_name = datatype_class_name + upper_compressed_type
attributes = {}
@@ -372,7 +372,7 @@ class Registry:
if not override:
# Do not load the datatype since it conflicts with an existing datatype which we are not supposed
# to override.
self.log.debug("Ignoring conflicting datatype with extension '{}' from {}.".format(extension, config))
self.log.debug(f"Ignoring conflicting datatype with extension '{extension}' from {config}.")
# Load datatype sniffers from the config - we'll do this even if one or more datatypes were not properly processed in the config
# since sniffers are not tightly coupled with datatypes.
self.load_datatype_sniffers(root,
@@ -413,7 +413,7 @@ class Registry:
if not os.path.exists(path):
sample_path = "%s.sample" % path
if os.path.exists(sample_path):
self.log.debug("Build site file [{}] not found using sample [{}].".format(path, sample_path))
self.log.debug(f"Build site file [{path}] not found using sample [{sample_path}].")
path = sample_path
self.build_sites[site_type] = path
@@ -703,13 +703,13 @@ class Registry:
del self.datatypes_by_extension[extension].display_applications[display_app.id]
if inherit and (self.datatypes_by_extension[extension], display_app) in self.inherit_display_application_by_class:
self.inherit_display_application_by_class.remove((self.datatypes_by_extension[extension], display_app))
self.log.debug("Deactivated display application '{}' for datatype '{}'.".format(display_app.id, extension))
self.log.debug(f"Deactivated display application '{display_app.id}' for datatype '{extension}'.")
else:
self.display_applications[display_app.id] = display_app
self.datatypes_by_extension[extension].add_display_application(display_app)
if inherit and (self.datatypes_by_extension[extension], display_app) not in self.inherit_display_application_by_class:
self.inherit_display_application_by_class.append((self.datatypes_by_extension[extension], display_app))
self.log.debug("Loaded display application '{}' for datatype '{}', inherit={}.".format(display_app.id, extension, inherit))
self.log.debug(f"Loaded display application '{display_app.id}' for datatype '{extension}', inherit={inherit}.")
except Exception:
if deactivate:
self.log.exception("Error deactivating display application (%s)" % config_path)
@@ -720,7 +720,7 @@ class Registry:
for d_type2, display_app in self.inherit_display_application_by_class:
current_app = d_type1.get_display_application(display_app.id, None)
if current_app is None and isinstance(d_type1, type(d_type2)):
self.log.debug("Adding inherited display application '{}' to datatype '{}'".format(display_app.id, extension))
self.log.debug(f"Adding inherited display application '{display_app.id}' to datatype '{extension}'")
d_type1.add_display_application(display_app)
def reload_display_applications(self, display_application_ids=None):
@@ -894,7 +894,7 @@ class Registry:
for convert_ext in self.get_converters_by_datatype(ext):
convert_ext_datatype = self.get_datatype_by_extension(convert_ext)
if convert_ext_datatype is None:
self.log.warning("Datatype class not found for extension '{}', which is used as target for conversion from datatype '{}'".format(convert_ext, dataset.ext))
self.log.warning(f"Datatype class not found for extension '{convert_ext}', which is used as target for conversion from datatype '{dataset.ext}'")
elif convert_ext_datatype.matches_any(accepted_formats):
converted_dataset = dataset and dataset.get_converted_files_by_type(convert_ext)
if converted_dataset:
@@ -919,7 +919,7 @@ class Registry:
help_txt = meta_spec.desc
if not help_txt or help_txt == meta_name:
help_txt = ""
inputs.append('<param type="text" name="{}" label="Set metadata value for &quot;{}&quot;" value="{}" help="{}"/>'.format(meta_name, meta_name, meta_spec.default, help_txt))
inputs.append(f'<param type="text" name="{meta_name}" label="Set metadata value for &quot;{meta_name}&quot;" value="{meta_spec.default}" help="{help_txt}"/>')
rval[ext] = "\n".join(inputs)
if 'auto' not in rval and 'txt' in rval: # need to manually add 'auto' datatype
rval['auto'] = rval['txt']
+6 -6
View File
@@ -186,7 +186,7 @@ class Sequence(data.Text):
ds = input_datasets[ds_no]
base_name = os.path.basename(ds.file_name)
part_path = os.path.join(dir, base_name)
split_data = dict(class_name='{}.{}'.format(cls.__module__, cls.__name__),
split_data = dict(class_name=f'{cls.__module__}.{cls.__name__}',
output_name=part_path,
input_name=ds.file_name,
args=dict(start_sequence=start_sequence, num_sequences=sequences_per_file[part_no]))
@@ -431,7 +431,7 @@ class Fasta(Sequence):
part_dir = subdir_generator_function()
part_path = os.path.join(part_dir, os.path.basename(input_file))
part_file = open(part_path, 'w')
log.debug("Writing {} part to {}".format(input_file, part_path))
log.debug(f"Writing {input_file} part to {part_path}")
start_offset = 0
while True:
offset = f.tell()
@@ -444,7 +444,7 @@ class Fasta(Sequence):
part_dir = subdir_generator_function()
part_path = os.path.join(part_dir, os.path.basename(input_file))
part_file = open(part_path, 'w')
log.debug("Writing {} part to {}".format(input_file, part_path))
log.debug(f"Writing {input_file} part to {part_path}")
start_offset = f.tell()
part_file.write(line)
except Exception as e:
@@ -467,7 +467,7 @@ class Fasta(Sequence):
part_dir = subdir_generator_function()
part_path = os.path.join(part_dir, os.path.basename(input_file))
part_file = open(part_path, 'w')
log.debug("Writing {} part to {}".format(input_file, part_path))
log.debug(f"Writing {input_file} part to {part_path}")
rec_count = 0
while True:
line = f.readline()
@@ -481,7 +481,7 @@ class Fasta(Sequence):
part_dir = subdir_generator_function()
part_path = os.path.join(part_dir, os.path.basename(input_file))
part_file = open(part_path, 'w')
log.debug("Writing {} part to {}".format(input_file, part_path))
log.debug(f"Writing {input_file} part to {part_path}")
rec_count = 1
part_file.write(line)
except Exception as e:
@@ -633,7 +633,7 @@ class Fastg(Sequence):
dataset.blurb += '\nversion=%s' % dataset.metadata.version
for k, v in dataset.metadata.properties.items():
if k != 'version':
dataset.blurb += '\n{}={}'.format(k, v)
dataset.blurb += f'\n{k}={v}'
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disk'
+1 -1
View File
@@ -88,7 +88,7 @@ class _SpalnDb(Data):
for i, fname in enumerate(flist):
sfname = os.path.split(fname)[-1]
f, e = os.path.splitext(fname)
rval.append('<li><a href="{}">{}</a></li>'.format(sfname, sfname))
rval.append(f'<li><a href="{sfname}">{sfname}</a></li>')
rval.append("</ul></body></html>")
with open(dataset.file_name, "w") as f:
f.write("\n".join(rval))
+2 -2
View File
@@ -25,7 +25,7 @@ class TextGrid(Text):
def sniff(self, filename):
with open(filename, 'r') as fd:
with open(filename) as fd:
text = fd.read(len(self.header))
return text == self.header
@@ -56,7 +56,7 @@ class BPF(Text):
def set_meta(self, dataset, overwrite=True, **kwd):
"""Set the metadata for this dataset from the file contents"""
types = set()
with open(dataset.dataset.file_name, 'r') as fd:
with open(dataset.dataset.file_name) as fd:
for line in fd:
# Split the line on a colon rather than regexing it
parts = line.split(':')
+1 -1
View File
@@ -305,7 +305,7 @@ class Tabular(TabularData):
if column_type2 == column_type:
return False
# neither column type was found in our ordered list, this cannot happen
raise ValueError("Tried to compare unknown column types: {} and {}".format(column_type1, column_type2))
raise ValueError(f"Tried to compare unknown column types: {column_type1} and {column_type2}")
def is_int(column_text):
# Don't allow underscores in numeric literals (PEP 515)
+3 -4
View File
@@ -6,11 +6,10 @@ import json
import logging
import os
import re
import shlex
import subprocess
import tempfile
from six.moves import shlex_quote
from galaxy.datatypes.data import get_file_peek, Text
from galaxy.datatypes.metadata import MetadataElement, MetadataParameter
from galaxy.datatypes.sniff import build_sniff_from_prefix, iter_headers
@@ -181,7 +180,7 @@ class Ipynb(Json):
ofilename = '%s.html' % ofilename
except subprocess.CalledProcessError:
ofilename = dataset.file_name
log.exception('Command "%s" failed. Could not convert the Jupyter Notebook to HTML, defaulting to plain text.', ' '.join(map(shlex_quote, cmd)))
log.exception('Command "%s" failed. Could not convert the Jupyter Notebook to HTML, defaulting to plain text.', ' '.join(map(shlex.quote, cmd)))
return open(ofilename, mode='rb')
def set_meta(self, dataset, **kwd):
@@ -465,7 +464,7 @@ class Arff(Text):
if not dataset.dataset.purged:
dataset.peek = get_file_peek(dataset.file_name)
dataset.blurb = "Attribute-Relation File Format (ARFF)"
dataset.blurb += ", {} comments, {} attributes".format(dataset.metadata.comment_lines, dataset.metadata.columns)
dataset.blurb += f", {dataset.metadata.comment_lines} comments, {dataset.metadata.columns} attributes"
else:
dataset.peek = 'file does not exist'
dataset.blurb = 'file purged from disc'
+1 -1
View File
@@ -38,7 +38,7 @@ class UCSCTrackHub(Html):
opt_text = ''
if composite_file.optional:
opt_text = ' (optional)'
rval.append('<li><a href="{}">{}</a>{}'.format(composite_name, composite_name, opt_text))
rval.append(f'<li><a href="{composite_name}">{composite_name}</a>{opt_text}')
rval.append('</ul></html>')
return "\n".join(rval)
+9 -10
View File
@@ -16,7 +16,6 @@ from errno import EMFILE
import bx.align.maf
import bx.interval_index_file
import bx.intervals
from six.moves import xrange
try:
maketrans = str.maketrans
@@ -132,7 +131,7 @@ class TempFileHandler:
self.files[index].flush()
def __del__(self):
for i in xrange(len(self.files)):
for i in range(len(self.files)):
self.close(i, delete=True)
@@ -295,7 +294,7 @@ def maf_index_by_uid(maf_uid, index_location_file):
maf_files = fields[4].replace("\n", "").replace("\r", "").split(",")
return bx.align.maf.MultiIndexed(maf_files, keep_open=True, parse_e_rows=False)
except Exception as e:
raise Exception('MAF UID ({}) found, but configuration appears to be malformed: {}'.format(maf_uid, e))
raise Exception(f'MAF UID ({maf_uid}) found, but configuration appears to be malformed: {e}')
except Exception:
pass
return None
@@ -347,7 +346,7 @@ def build_maf_index_species_chromosomes(filename, index_species=None):
indexes.add(c.src, forward_strand_start, forward_strand_end, pos, max=c.src_size)
except Exception as e:
# most likely a bad MAF
log.debug('Building MAF index on {} failed: {}'.format(filename, e))
log.debug(f'Building MAF index on {filename} failed: {e}')
return (None, [], {}, 0)
return (indexes, species, species_chromosomes, blocks)
@@ -427,7 +426,7 @@ def orient_block_by_region(block, src, region, force_strand=None):
def get_oriented_chopped_blocks_for_region(index, src, region, species=None, mincols=0, force_strand=None):
for block, idx, offset in get_oriented_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols, force_strand):
for block, _, _ in get_oriented_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols, force_strand):
yield block
@@ -488,7 +487,7 @@ def iter_blocks_split_by_species(block, species=None):
# generator yielding only chopped and valid blocks for a specified region
def get_chopped_blocks_for_region(index, src, region, species=None, mincols=0):
for block, idx, offset in get_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols):
for block, _, _ in get_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols):
yield block
@@ -512,7 +511,7 @@ def get_region_alignment(index, primary_species, chrom, start, end, strand='+',
def reduce_block_by_primary_genome(block, species, chromosome, region_start):
# returns ( startIndex, {species:texts}
# where texts' contents are reduced to only positions existing in the primary genome
src = "{}.{}".format(species, chromosome)
src = f"{species}.{chromosome}"
ref = block.get_component_by_src(src)
start_offset = ref.start - region_start
species_texts = {}
@@ -533,7 +532,7 @@ def fill_region_alignment(alignment, index, primary_species, chrom, start, end,
region = bx.intervals.Interval(start, end)
region.chrom = chrom
region.strand = strand
primary_src = "{}.{}".format(primary_species, chrom)
primary_src = f"{primary_species}.{chrom}"
# Order blocks overlaping this position by score, lowest first
blocks = []
@@ -684,9 +683,9 @@ def remove_temp_index_file(index_filename):
def get_fasta_header(component, attributes={}, suffix=None):
header = ">%s(%s):%i-%i|" % (component.src, component.strand, component.get_forward_strand_start(), component.get_forward_strand_end())
for key, value in attributes.items():
header = "{}{}={}|".format(header, key, value)
header = f"{header}{key}={value}|"
if suffix:
header = "{}{}".format(header, suffix)
header = f"{header}{suffix}"
else:
header = "{}{}".format(header, src_split(component.src)[0])
return header
@@ -11,7 +11,6 @@ name = "pypi"
[dev-packages]
gunicorn = "*"
lxml = "!=4.2.2"
mock = "*"
NoseHTML = "*"
pygithub = "*"
pytest = "*"
@@ -61,7 +60,6 @@ Beaker = "*"
dictobj = "*"
nose = "*"
Parsley = "*"
six = "*"
sortedcontainers = "*"
Whoosh = "*"
galaxy_sequence_utils = "*"
@@ -88,4 +86,4 @@ gxformat2 = "*"
refgenconf = ">=0.7.0"
[requires]
python_version = "3.5"
python_version = "3.6"
+4 -4
View File
@@ -12,7 +12,7 @@ log = logging.getLogger(__name__)
FileSourcePath = namedtuple('FileSourcePath', ['file_source', 'path'])
class ConfiguredFileSources(object):
class ConfiguredFileSources:
"""Load plugins and resolve Galaxy URIs to FileSource objects."""
def __init__(self, file_sources_config, conf_file=None, conf_dict=None, load_stock_plugins=False):
@@ -161,7 +161,7 @@ class ConfiguredFileSources(object):
return ConfiguredFileSources(file_sources_config, conf_dict=sources_as_dict)
class ConfiguredFileSourcesConfig(object):
class ConfiguredFileSourcesConfig:
def __init__(self, symlink_allowlist=[], library_import_dir=None, user_library_import_dir=None, ftp_upload_dir=None, ftp_upload_purge=True):
self.symlink_allowlist = symlink_allowlist
@@ -202,7 +202,7 @@ class ConfiguredFileSourcesConfig(object):
)
class ProvidesUserFileSourcesUserContext(object):
class ProvidesUserFileSourcesUserContext:
"""Implement a FileSourcesUserContext from a Galaxy ProvidesUserContext (e.g. trans)."""
def __init__(self, trans):
@@ -228,7 +228,7 @@ class ProvidesUserFileSourcesUserContext(object):
return user and user.extra_preferences
class DictFileSourcesUserContext(object):
class DictFileSourcesUserContext:
def __init__(self, **kwd):
self._kwd = kwd
+1 -4
View File
@@ -2,16 +2,13 @@ import abc
import os
import time
import six
from galaxy.util.template import fill_template
DEFAULT_SCHEME = "gxfiles"
DEFAULT_WRITABLE = False
@six.add_metaclass(abc.ABCMeta)
class FilesSource(object):
class FilesSource(metaclass=abc.ABCMeta):
"""
"""
+3 -3
View File
@@ -18,7 +18,7 @@ class UserFtpFilesSource(PosixFilesSource):
if "delete_on_realize" not in posix_kwds:
file_sources_config = kwd.get("file_sources_config")
posix_kwds["delete_on_realize"] = file_sources_config.ftp_upload_purge
super(UserFtpFilesSource, self).__init__(**posix_kwds)
super().__init__(**posix_kwds)
def get_prefix(self):
return None
@@ -38,7 +38,7 @@ class LibraryImportFilesSource(PosixFilesSource):
doc=doc,
)
posix_kwds.update(kwd)
super(LibraryImportFilesSource, self).__init__(**posix_kwds)
super().__init__(**posix_kwds)
def get_prefix(self):
return None
@@ -58,7 +58,7 @@ class UserLibraryImportFilesSource(PosixFilesSource):
doc=doc,
)
posix_kwds.update(kwd)
super(UserLibraryImportFilesSource, self).__init__(**posix_kwds)
super().__init__(**posix_kwds)
def get_prefix(self):
return None
+2 -2
View File
@@ -21,7 +21,7 @@ class FormDefinitionFactory:
"""
Return new FormDefinition.
"""
assert form_type in self.form_types, 'Invalid FormDefinition type ( {} not in {} )'.format(form_type, self.form_types.keys())
assert form_type in self.form_types, f'Invalid FormDefinition type ( {form_type} not in {self.form_types.keys()} )'
assert name, 'FormDefinition requires a name'
if description is None:
description = ''
@@ -113,7 +113,7 @@ class FormDefinitionFieldFactory:
visible = string_as_bool(elem.get('visible', 'true'))
field_layout = elem.get('layout', None)
if field_layout:
assert layout and field_layout in layout, 'Invalid layout specified: {} not in {}'.format(field_layout, layout)
assert layout and field_layout in layout, f'Invalid layout specified: {field_layout} not in {layout}'
field_layout = str(layout.index(field_layout)) # existing behavior: integer indexes are stored as strings. why?
return self.new(name=name, label=label, required=required, helptext=helptext, default=default, visible=visible, layout=field_layout)
+4 -4
View File
@@ -386,7 +386,7 @@ def collect_primary_datasets(job_context, output, input_ext):
if dbkey == INPUT_DBKEY_TOKEN:
dbkey = job_context.input_dbkey
if filename_index == 0 and extra_file_collector.assign_primary_output and output_index == 0:
new_outdata_name = fields_match.name or "{} ({})".format(outdata.name, designation)
new_outdata_name = fields_match.name or f"{outdata.name} ({designation})"
outdata.change_datatype(ext)
outdata.dbkey = dbkey
outdata.designation = designation
@@ -399,7 +399,7 @@ def collect_primary_datasets(job_context, output, input_ext):
primary_datasets[name] = OrderedDict()
visible = fields_match.visible
# Create new primary dataset
new_primary_name = fields_match.name or "{} ({})".format(outdata.name, designation)
new_primary_name = fields_match.name or f"{outdata.name} ({designation})"
info = outdata.info
# TODO: should be able to disambiguate files in different directories...
@@ -418,7 +418,7 @@ def collect_primary_datasets(job_context, output, input_ext):
dataset_attributes=new_primary_datasets_attributes,
)
# Associate new dataset with job
job_context.add_output_dataset_association('__new_primary_file_{}|{}__'.format(name, designation), primary_data)
job_context.add_output_dataset_association(f'__new_primary_file_{name}|{designation}__', primary_data)
if new_primary_datasets_attributes:
extra_files_path = new_primary_datasets_attributes.get('extra_files', None)
@@ -577,7 +577,7 @@ def read_exit_code_from(exit_code_file, id_tag):
exit_code = int(exit_code_str)
except ValueError:
galaxy_id_tag = id_tag
log.warning("({}) Exit code '{}' invalid. Using 0.".format(galaxy_id_tag, exit_code_str))
log.warning(f"({galaxy_id_tag}) Exit code '{exit_code_str}' invalid. Using 0.")
exit_code = 0
return exit_code
+2 -2
View File
@@ -110,13 +110,13 @@ class CollectlCli:
if not interval2:
return interval_arg
self.__validate_interval_arg(interval2, multiple_of=int(interval))
interval_arg = "{}:{}".format(interval_arg, interval2)
interval_arg = f"{interval_arg}:{interval2}"
interval3 = kwargs.get("interval3", None)
if not interval3:
return interval_arg
self.__validate_interval_arg(interval3, multiple_of=int(interval))
interval_arg = "{}:{}".format(interval_arg, interval3)
interval_arg = f"{interval_arg}:{interval3}"
return interval_arg
def __validate_interval_arg(self, value, multiple_of=None):
@@ -49,7 +49,7 @@ class InstrumentPlugin(metaclass=ABCMeta):
""" Provide a common pattern for naming files used by instrumentation
plugins - to ease their staging out of remote job directories.
"""
return "{}_{}_{}".format(INSTRUMENT_FILE_PREFIX, self.plugin_type, name)
return f"{INSTRUMENT_FILE_PREFIX}_{self.plugin_type}_{name}"
def _instrument_file_path(self, job_directory, name):
return os.path.join(job_directory, self._instrument_file_name(name))
@@ -48,7 +48,7 @@ class CollectlFormatter(formatting.JobMetricFormatter):
else:
value_str = str(value)
resource_title = FORMATTED_RESOURCE_TITLES.get(resource_type, resource_type)
return ("{} ({})".format(resource_title, stat_type), value_str)
return (f"{resource_title} ({stat_type})", value_str)
class CollectlPlugin(InstrumentPlugin):
@@ -102,7 +102,7 @@ class CollectlPlugin(InstrumentPlugin):
rel_path = filter(self._is_instrumented_collectl_log, contents)[0]
path = os.path.join(job_directory, rel_path)
except IndexError:
message = "Failed to find collectl log in directory {}, files were {}".format(job_directory, contents)
message = f"Failed to find collectl log in directory {job_directory}, files were {contents}"
raise Exception(message)
properties = dict(
@@ -51,7 +51,7 @@ class CpuInfoPlugin(InstrumentPlugin):
# If verbose, dump information about each processor
# into database...
key, value = line.split(":", 1)
key = "processor_{}_{}".format(current_processor, key.strip())
key = f"processor_{current_processor}_{key.strip()}"
value = value
properties["processor_count"] = processor_count
return properties
+15 -15
View File
@@ -803,7 +803,7 @@ class JobConfiguration(ConfiguresHandlers):
log.warning("A non-class name was found in __all__, ignoring: %s" % id)
continue
except AssertionError:
log.warning("Job runner classes must be subclassed from BaseJobRunner, {} has bases: {}".format(id, runner_class.__bases__))
log.warning(f"Job runner classes must be subclassed from BaseJobRunner, {id} has bases: {runner_class.__bases__}")
continue
try:
rval[id] = runner_class(self.app, runner.get('workers', JobConfiguration.DEFAULT_NWORKERS), **runner.get('kwds', {}))
@@ -811,7 +811,7 @@ class JobConfiguration(ConfiguresHandlers):
log.exception("Job runner '%s:%s' has not been converted to a new-style runner or encountered TypeError on load",
module_name, class_name)
rval[id] = runner_class(self.app)
log.debug("Loaded job runner '{}:{}' as '{}'".format(module_name, class_name, id))
log.debug(f"Loaded job runner '{module_name}:{class_name}' as '{id}'")
return rval
def is_id(self, collection):
@@ -847,13 +847,13 @@ class JobConfiguration(ConfiguresHandlers):
destination.params = job_runners[destination.runner].url_to_destination(destination.url).params
destination.converted = True
if destination.params:
log.debug("Legacy destination with id '{}', url '{}' converted, got params:".format(id, destination.url))
log.debug(f"Legacy destination with id '{id}', url '{destination.url}' converted, got params:")
for k, v in destination.params.items():
log.debug(" {}: {}".format(k, v))
log.debug(f" {k}: {v}")
else:
log.debug("Legacy destination with id '{}', url '{}' converted, got params:".format(id, destination.url))
log.debug(f"Legacy destination with id '{id}', url '{destination.url}' converted, got params:")
else:
log.warning("Legacy destination with id '{}' could not be converted: Unknown runner plugin: {}".format(id, destination.runner))
log.warning(f"Legacy destination with id '{id}' could not be converted: Unknown runner plugin: {destination.runner}")
class HasResourceParameters:
@@ -1129,7 +1129,7 @@ class JobWrapper(HasResourceParameters):
if version_string_cmd_raw:
version_command_template = string.Template(version_string_cmd_raw)
version_string_cmd = version_command_template.safe_substitute({"__tool_directory__": compute_environment.tool_directory()})
self.write_version_cmd = "{} > {} 2>&1".format(version_string_cmd, compute_environment.version_path())
self.write_version_cmd = f"{version_string_cmd} > {compute_environment.version_path()} 2>&1"
else:
self.write_version_cmd = None
return self.extra_filenames
@@ -1412,7 +1412,7 @@ class JobWrapper(HasResourceParameters):
"""
if job is None:
job = self.get_job()
log.debug('({}) Persisting job destination (destination id: {})'.format(job.id, job_destination.id))
log.debug(f'({job.id}) Persisting job destination (destination id: {job_destination.id})')
job.destination_id = job_destination.id
job.destination_params = job_destination.params
job.job_runner_name = job_destination.runner
@@ -1594,7 +1594,7 @@ class JobWrapper(HasResourceParameters):
try:
self.reclaim_ownership()
except Exception:
log.exception('({}) Failed to change ownership of {}, failing'.format(job.id, self.working_directory))
log.exception(f'({job.id}) Failed to change ownership of {self.working_directory}, failing')
return fail()
# if the job was deleted, don't finish it
@@ -1641,7 +1641,7 @@ class JobWrapper(HasResourceParameters):
for dataset_path in self.get_output_fnames():
try:
shutil.move(dataset_path.false_path, dataset_path.real_path)
log.debug("finish(): Moved {} to {}".format(dataset_path.false_path, dataset_path.real_path))
log.debug(f"finish(): Moved {dataset_path.false_path} to {dataset_path.real_path}")
except OSError:
# this can happen if Galaxy is restarted during the job's
# finish method - the false_path file has already moved,
@@ -1661,7 +1661,7 @@ class JobWrapper(HasResourceParameters):
import_model_store = store.get_import_model_store_for_directory(os.path.join(self.working_directory, 'metadata', 'outputs_populated'), app=self.app, import_options=import_options)
import_model_store.perform_import(history=job.history)
except Exception:
log.exception("problem importing job outputs. stdout [{}] stderr [{}]".format(job.stdout, job.stderr))
log.exception(f"problem importing job outputs. stdout [{job.stdout}] stderr [{job.stderr}]")
raise
output_dataset_associations = job.output_datasets + job.output_library_datasets
for dataset_assoc in output_dataset_associations:
@@ -1883,14 +1883,14 @@ class JobWrapper(HasResourceParameters):
def get_env_setup_clause(self):
if self.app.config.environment_setup_file is None:
return ''
return '[ -f "{}" ] && . {}'.format(self.app.config.environment_setup_file, self.app.config.environment_setup_file)
return f'[ -f "{self.app.config.environment_setup_file}" ] && . {self.app.config.environment_setup_file}'
def get_input_dataset_fnames(self, ds):
filenames = []
filenames = [ds.file_name]
# we will need to stage in metadata file names also
# TODO: would be better to only stage in metadata files that are actually needed (found in command line, referenced in config files, etc.)
for key, value in ds.metadata.items():
for _, value in ds.metadata.items():
if isinstance(value, model.MetadataFile):
filenames.append(value.file_name)
return filenames
@@ -1938,7 +1938,7 @@ class JobWrapper(HasResourceParameters):
for (hda, dataset_path) in self.output_hdas_and_paths.values():
if hda == dataset:
return dataset_path
raise KeyError("Couldn't find job output for [{}] in [{}]".format(dataset, self.output_hdas_and_paths.values()))
raise KeyError(f"Couldn't find job output for [{dataset}] in [{self.output_hdas_and_paths.values()}]")
def get_mutable_output_fnames(self):
if self.output_paths is None:
@@ -2100,7 +2100,7 @@ class JobWrapper(HasResourceParameters):
dependency_shell_commands = metadata_tool.build_dependency_shell_commands(job_directory=self.working_directory, metadata=True)
if dependency_shell_commands:
dependency_shell_commands = "; ".join(dependency_shell_commands)
command = "{}; {}".format(dependency_shell_commands, command)
command = f"{dependency_shell_commands}; {command}"
return command
def check_for_entry_points(self, check_already_configured=True):
+4 -4
View File
@@ -364,14 +364,14 @@ class DeleteIntermediatesAction(DefaultJobAction):
if wfi_step_job:
jobs_to_check.append(wfi_step_job)
else:
log.debug("No job found yet for wfi_step {}, (step {})".format(wfi_step, wfi_step.workflow_step))
log.debug(f"No job found yet for wfi_step {wfi_step}, (step {wfi_step.workflow_step})")
for j2c in jobs_to_check:
creating_jobs = []
for input_dataset in j2c.input_datasets:
if not input_dataset.dataset:
log.debug("PJA Async Issue: No dataset attached to input_dataset {} during handling of workflow invocation {}".format(input_dataset.id, wfi))
log.debug(f"PJA Async Issue: No dataset attached to input_dataset {input_dataset.id} during handling of workflow invocation {wfi}")
elif not input_dataset.dataset.creating_job:
log.debug("PJA Async Issue: No creating job attached to dataset {} during handling of workflow invocation {}".format(input_dataset.dataset.id, wfi))
log.debug(f"PJA Async Issue: No creating job attached to dataset {input_dataset.dataset.id} during handling of workflow invocation {wfi}")
else:
creating_jobs.append((input_dataset, input_dataset.dataset.creating_job))
for (input_dataset, creating_job) in creating_jobs:
@@ -383,7 +383,7 @@ class DeleteIntermediatesAction(DefaultJobAction):
safe_to_delete = True
for job_to_check in [d_j.job for d_j in input_dataset.dependent_jobs]:
if job_to_check != job and job_to_check.state not in [job.states.OK, job.states.DELETED]:
log.trace("Workflow Intermediates cleanup attempted, but non-terminal state '{}' detected for job {}".format(job_to_check.state, job_to_check.id))
log.trace(f"Workflow Intermediates cleanup attempted, but non-terminal state '{job_to_check.state}' detected for job {job_to_check.id}")
safe_to_delete = False
if safe_to_delete:
# Support purging here too.
+4 -4
View File
@@ -155,7 +155,7 @@ def __externalize_commands(job_wrapper, shell, commands_builder, remote_command_
tool_commands,
)
write_script(local_container_script, script_contents, config)
commands = "{} {}".format(shell, local_container_script)
commands = f"{shell} {local_container_script}"
# TODO: Cleanup for_pulsar hack.
# - Integrate Pulsar sending tool_stdout/tool_stderr back
# https://github.com/galaxyproject/pulsar/pull/202
@@ -170,7 +170,7 @@ def __externalize_commands(job_wrapper, shell, commands_builder, remote_command_
for_pulsar = True
if not for_pulsar:
commands += " > ../outputs/tool_stdout 2> ../outputs/tool_stderr"
log.info("Built script [{}] for tool command [{}]".format(local_container_script, tool_commands))
log.info(f"Built script [{local_container_script}] for tool command [{tool_commands}]")
return commands
@@ -236,14 +236,14 @@ def __handle_metadata(commands_builder, job_wrapper, runner, remote_command_para
metadata_command = metadata_command.strip()
if metadata_command:
# Place Galaxy and its dependencies in environment for metadata regardless of tool.
metadata_command = "{}{}".format(SETUP_GALAXY_FOR_METADATA, metadata_command)
metadata_command = f"{SETUP_GALAXY_FOR_METADATA}{metadata_command}"
commands_builder.capture_return_code()
commands_builder.append_command(metadata_command)
def __copy_if_exists_command(work_dir_output):
source_file, destination = work_dir_output
return "if [ -f {} ] ; then cp {} {} ; fi".format(source_file, source_file, destination)
return f"if [ -f {source_file} ] ; then cp {source_file} {destination} ; fi"
class CommandsBuilder:
+14 -14
View File
@@ -5,11 +5,11 @@ import datetime
import os
import time
from collections import defaultdict
from six.moves.queue import (
from queue import (
Empty,
Queue
Queue,
)
from sqlalchemy.exc import OperationalError
from sqlalchemy.sql.expression import (
and_,
@@ -217,7 +217,7 @@ class JobHandlerQueue(Monitors):
for job in jobs_at_startup:
if not self.app.toolbox.has_tool(job.tool_id, job.tool_version, exact=True):
log.warning("({}) Tool '{}' removed from tool config, unable to recover job".format(job.id, job.tool_id))
log.warning(f"({job.id}) Tool '{job.tool_id}' removed from tool config, unable to recover job")
self.job_wrapper(job).fail('This tool was disabled before the job completed. Please contact your Galaxy administrator.')
elif job.job_runner_name is not None and job.job_runner_external_id is None:
# This could happen during certain revisions of Galaxy where a runner URL was persisted before the job was dispatched to a runner.
@@ -238,7 +238,7 @@ class JobHandlerQueue(Monitors):
log.info('(%s) Converted job from a URL to a destination and recovered' % (job.id))
elif job.job_runner_name is None:
# Never (fully) dispatched
log.debug("({}) No job runner assigned and job still in '{}' state, adding to the job handler queue".format(job.id, job.state))
log.debug(f"({job.id}) No job runner assigned and job still in '{job.state}' state, adding to the job handler queue")
if self.track_jobs_in_database:
job.set_state(model.Job.states.NEW)
else:
@@ -486,7 +486,7 @@ class JobHandlerQueue(Monitors):
elif dataset_state == model.Dataset.states.ERROR:
jobs_to_pause[job_id].append("Input dataset '%s' is in error state" % hda_name)
elif dataset_state != model.Dataset.states.OK:
jobs_to_ignore[job_id].append("Input dataset '{}' is in {} state".format(hda_name, dataset_state))
jobs_to_ignore[job_id].append(f"Input dataset '{hda_name}' is in {dataset_state} state")
for job_id in sorted(jobs_to_pause):
pause_message = ", ".join(jobs_to_pause[job_id])
pause_message = "%s. To resume this job fix the input dataset(s)." % pause_message
@@ -557,7 +557,7 @@ class JobHandlerQueue(Monitors):
# Cause the job_destination to be set and cached by the mapper
job_destination = job_wrapper.job_destination
except AssertionError as e:
log.warning("({}) Tool '{}' removed from tool config, unable to run job".format(job.id, job.tool_id))
log.warning(f"({job.id}) Tool '{job.tool_id}' removed from tool config, unable to run job")
job_wrapper.fail(e)
return JOB_ERROR, job_destination
except JobNotReadyException as e:
@@ -631,7 +631,7 @@ class JobHandlerQueue(Monitors):
continue
# don't run jobs for which the input dataset was deleted
if idata.deleted:
self.job_wrappers.pop(job.id, self.job_wrapper(job)).fail("input data {} (file: {}) was deleted before the job started".format(idata.hid, idata.file_name))
self.job_wrappers.pop(job.id, self.job_wrapper(job)).fail(f"input data {idata.hid} (file: {idata.file_name}) was deleted before the job started")
return JOB_INPUT_DELETED
# an error in the input data causes us to bail immediately
elif idata.state == idata.states.ERROR:
@@ -1000,12 +1000,12 @@ class DefaultJobDispatcher:
try:
if isinstance(job_wrapper, TaskWrapper):
# DBTODO Refactor
log.debug("({}) Dispatching task {} to {} runner".format(job_wrapper.job_id, job_wrapper.task_id, runner_name))
log.debug(f"({job_wrapper.job_id}) Dispatching task {job_wrapper.task_id} to {runner_name} runner")
else:
log.debug("({}) Dispatching to {} runner".format(job_wrapper.job_id, runner_name))
log.debug(f"({job_wrapper.job_id}) Dispatching to {runner_name} runner")
self.job_runners[runner_name].put(job_wrapper)
except KeyError:
log.error('put(): ({}) Invalid job runner: {}'.format(job_wrapper.job_id, runner_name))
log.error(f'put(): ({job_wrapper.job_id}) Invalid job runner: {runner_name}')
job_wrapper.fail(DEFAULT_JOB_PUT_FAILURE_MESSAGE)
def stop(self, job, job_wrapper):
@@ -1023,11 +1023,11 @@ class DefaultJobDispatcher:
job_runner_name = job.get_job_runner_name()
if job_runner_name is not None:
runner_name = job_runner_name.split(":", 1)[0]
log.debug("Stopping job {} in {} runner".format(job_wrapper.get_id_tag(), runner_name))
log.debug(f"Stopping job {job_wrapper.get_id_tag()} in {runner_name} runner")
try:
self.job_runners[runner_name].stop_job(job_wrapper)
except KeyError:
log.error('stop(): ({}) Invalid job runner: {}'.format(job_wrapper.get_id_tag(), runner_name))
log.error(f'stop(): ({job_wrapper.get_id_tag()}) Invalid job runner: {runner_name}')
# Job and output dataset states have already been updated, so nothing is done here.
def recover(self, job, job_wrapper):
@@ -1036,7 +1036,7 @@ class DefaultJobDispatcher:
try:
self.job_runners[runner_name].recover(job, job_wrapper)
except KeyError:
log.error('recover(): ({}) Invalid job runner: {}'.format(job_wrapper.job_id, runner_name))
log.error(f'recover(): ({job_wrapper.job_id}) Invalid job runner: {runner_name}')
job_wrapper.fail(DEFAULT_JOB_PUT_FAILURE_MESSAGE)
def shutdown(self):
+1 -1
View File
@@ -187,7 +187,7 @@ class RuleHelper:
invocation for jobs outside of workflows.
"""
if hash_by not in VALID_JOB_HASH_STRATEGIES:
message = "Do not know how to hash jobs by {}, must be one of {}".format(hash_by, VALID_JOB_HASH_STRATEGIES)
message = f"Do not know how to hash jobs by {hash_by}, must be one of {VALID_JOB_HASH_STRATEGIES}"
raise Exception(message)
if hash_by == "workflow_invocation":
+12 -13
View File
@@ -9,10 +9,9 @@ import sys
import threading
import time
import traceback
from six.moves.queue import (
from queue import (
Empty,
Queue
Queue,
)
import galaxy.jobs
@@ -90,7 +89,7 @@ class BaseJobRunner:
"""
self.work_queue = Queue()
self.work_threads = []
log.debug('Starting {} {} workers'.format(self.nworkers, self.runner_name))
log.debug(f'Starting {self.nworkers} {self.runner_name} workers')
for i in range(self.nworkers):
worker = threading.Thread(name="%s.work_thread-%d" % (self.runner_name, i), target=self.run_next)
worker.daemon = True
@@ -129,7 +128,7 @@ class BaseJobRunner:
except Exception:
name = 'unknown'
try:
action_str = 'galaxy.jobs.runners.{}.{}'.format(self.__class__.__name__.lower(), name)
action_str = f'galaxy.jobs.runners.{self.__class__.__name__.lower()}.{name}'
action_timer = self.app.execution_timer_factory.get_timer(
'internals.%s' % action_str,
'job runner action %s for job ${job_id} executed' % (action_str)
@@ -137,7 +136,7 @@ class BaseJobRunner:
method(arg)
log.trace(action_timer.to_str(job_id=job_id))
except Exception:
log.exception("({}) Unhandled exception calling {}".format(job_id, name))
log.exception(f"({job_id}) Unhandled exception calling {name}")
if not isinstance(arg, JobState):
job_state = JobState(job_wrapper=arg, job_destination={})
else:
@@ -151,7 +150,7 @@ class BaseJobRunner:
put_timer = ExecutionTimer()
job_wrapper.enqueue()
self.mark_as_queued(job_wrapper)
log.debug("Job [{}] queued {}".format(job_wrapper.job_id, put_timer))
log.debug(f"Job [{job_wrapper.job_id}] queued {put_timer}")
def mark_as_queued(self, job_wrapper):
self.work_queue.put((self.queue_job, job_wrapper))
@@ -222,12 +221,12 @@ class BaseJobRunner:
# Make sure the job hasn't been deleted
if job_state == model.Job.states.DELETED:
log.debug("({}) Job deleted by user before it entered the {} queue".format(job_id, self.runner_name))
log.debug(f"({job_id}) Job deleted by user before it entered the {self.runner_name} queue")
if self.app.config.cleanup_job in ("always", "onsuccess"):
job_wrapper.cleanup()
return False
elif job_state != model.Job.states.QUEUED:
log.info("({}) Job is in state {}, skipping execution".format(job_id, job_state))
log.info(f"({job_id}) Job is in state {job_state}, skipping execution")
# cleanup may not be safe in all states
return False
@@ -352,13 +351,13 @@ class BaseJobRunner:
tmp_dir=job_wrapper.working_directory,
# We don't want to overwrite metadata that was copied over in init_meta(), as per established behavior
kwds={'overwrite' : False})
external_metadata_script = "{} {} {}".format(lib_adjust, venv, external_metadata_script)
external_metadata_script = f"{lib_adjust} {venv} {external_metadata_script}"
if resolve_requirements:
dependency_shell_commands = self.app.datatypes_registry.set_external_metadata_tool.build_dependency_shell_commands(job_directory=job_wrapper.working_directory)
if dependency_shell_commands:
if isinstance(dependency_shell_commands, list):
dependency_shell_commands = "&&".join(dependency_shell_commands)
external_metadata_script = "{}&&{}".format(dependency_shell_commands, external_metadata_script)
external_metadata_script = f"{dependency_shell_commands}&&{external_metadata_script}"
log.debug('executing external set_meta script for job %d: %s' % (job_wrapper.job_id, external_metadata_script))
external_metadata_proc = subprocess.Popen(args=external_metadata_script,
shell=True,
@@ -396,7 +395,7 @@ class BaseJobRunner:
# Additional logging to enable if debugging from_work_dir handling, metadata
# commands, etc... (or just peak in the job script.)
job_id = job_wrapper.job_id
log.debug('({}) command is: {}'.format(job_id, command_line))
log.debug(f'({job_id}) command is: {command_line}')
options.update(**kwds)
return job_script(**options)
@@ -600,7 +599,7 @@ class JobState:
if not hasattr(self, "job_id"):
prefix = "(%s)" % self.job_wrapper.get_id_tag()
else:
prefix = "({}/{})".format(self.job_wrapper.get_id_tag(), self.job_id)
prefix = f"({self.job_wrapper.get_id_tag()}/{self.job_id})"
log.debug("{} Unable to cleanup {}: {}".format(prefix, file, unicodify(e)))
+2 -2
View File
@@ -196,7 +196,7 @@ class ChronosJobRunner(AsynchronousJobRunner):
msg = 'Job {name!r} failed more than {retries!s} times'
reason = msg.format(name=job_name, retries=str(max_retries))
return self._mark_as_failed(job_state, reason)
reason = 'Job {name!r} not found'.format(name=job_name)
reason = f'Job {job_name!r} not found'
return self._mark_as_failed(job_state, reason)
def _mark_as_successful(self, job_state):
@@ -264,7 +264,7 @@ class ChronosJobRunner(AsynchronousJobRunner):
jobs = self._chronos_client.list()
job = [x for x in jobs if x['name'] == job_id]
if len(job) > 1:
msg = 'Multiple jobs found with name {name!r}'.format(name=job_id)
msg = f'Multiple jobs found with name {job_id!r}'
LOGGER.error(msg)
raise ChronosRunnerException(msg)
return job[0] if job else None
+13 -13
View File
@@ -47,7 +47,7 @@ class ShellJobRunner(AsynchronousJobRunner):
job_params = {'job_' + k: v for k, v in [kv.split('=', 1) for kv in job_params.split('&')]}
params.update(shell_params)
params.update(job_params)
log.debug("Converted URL '{}' to destination runner=cli, params={}".format(url, params))
log.debug(f"Converted URL '{url}' to destination runner=cli, params={params}")
# Create a dynamic JobDestination
return JobDestination(runner='cli', params=params)
@@ -94,7 +94,7 @@ class ShellJobRunner(AsynchronousJobRunner):
job_wrapper.cleanup()
return
log.debug("({}) submitting file: {}".format(galaxy_id_tag, ajs.job_file))
log.debug(f"({galaxy_id_tag}) submitting file: {ajs.job_file}")
returncode, stdout = self.submit(shell, job_interface, ajs.job_file, galaxy_id_tag, retry=MAX_SUBMIT_RETRY)
if returncode != 0:
@@ -108,7 +108,7 @@ class ShellJobRunner(AsynchronousJobRunner):
job_wrapper.fail("failure submitting job")
return
log.info("({}) queued with identifier: {}".format(galaxy_id_tag, external_job_id))
log.info(f"({galaxy_id_tag}) queued with identifier: {external_job_id}")
# store runner information for tracking if Galaxy restarts
job_wrapper.set_external_id(external_job_id)
@@ -131,8 +131,8 @@ class ShellJobRunner(AsynchronousJobRunner):
cmd_out = shell.execute(job_interface.submit(job_file))
if cmd_out.returncode == 0:
return cmd_out.returncode, cmd_out.stdout
stdout = '({}) submission failed (stdout): {}'.format(galaxy_id_tag, cmd_out.stdout)
stderr = '({}) submission failed (stderr): {}'.format(galaxy_id_tag, cmd_out.stderr)
stdout = f'({galaxy_id_tag}) submission failed (stdout): {cmd_out.stdout}'
stderr = f'({galaxy_id_tag}) submission failed (stderr): {cmd_out.stderr}'
if retry > 0:
log.debug("%s, retrying in %s seconds", stdout, timeout)
log.debug("%s, retrying in %s seconds", stderr, timeout)
@@ -161,15 +161,15 @@ class ShellJobRunner(AsynchronousJobRunner):
if ajs.job_wrapper.get_state() == model.Job.states.DELETED:
continue
log.debug("({}/{}) job not found in batch state check".format(id_tag, external_job_id))
log.debug(f"({id_tag}/{external_job_id}) job not found in batch state check")
shell_params, job_params = self.parse_destination_params(ajs.job_destination.params)
shell, job_interface = self.get_cli_plugins(shell_params, job_params)
cmd_out = shell.execute(job_interface.get_single_status(external_job_id))
state = job_interface.parse_single_status(cmd_out.stdout, external_job_id)
if not state == model.Job.states.OK:
log.warning('({}/{}) job not found in batch state check, but found in individual state check'.format(id_tag, external_job_id))
log.warning(f'({id_tag}/{external_job_id}) job not found in batch state check, but found in individual state check')
if state != old_state:
log.debug("({}/{}) state change: from {} to {}".format(id_tag, external_job_id, old_state, state))
log.debug(f"({id_tag}/{external_job_id}) state change: from {old_state} to {state}")
if not state == model.Job.states.OK:
# No need to change_state when the state is OK, this will be handled by `self.finish_job`
ajs.job_wrapper.change_state(state)
@@ -186,7 +186,7 @@ class ShellJobRunner(AsynchronousJobRunner):
external_metadata = not asbool(ajs.job_wrapper.job_destination.params.get("embed_metadata_in_job", DEFAULT_EMBED_METADATA_IN_JOB))
if external_metadata:
self.work_queue.put((self.handle_metadata_externally, ajs))
log.debug('({}/{}) job execution finished, running job wrapper finish method'.format(id_tag, external_job_id))
log.debug(f'({id_tag}/{external_job_id}) job execution finished, running job wrapper finish method')
self.work_queue.put((self.finish_job, ajs))
else:
new_watched.append(ajs)
@@ -234,9 +234,9 @@ class ShellJobRunner(AsynchronousJobRunner):
shell, job_interface = self.get_cli_plugins(shell_params, job_params)
cmd_out = shell.execute(job_interface.delete(job.job_runner_external_id))
assert cmd_out.returncode == 0, cmd_out.stderr
log.debug("({}/{}) Terminated at user's request".format(job.id, job.job_runner_external_id))
log.debug(f"({job.id}/{job.job_runner_external_id}) Terminated at user's request")
except Exception as e:
log.debug("({}/{}) User killed running job, but error encountered during termination: {}".format(job.id, job.job_runner_external_id, e))
log.debug(f"({job.id}/{job.job_runner_external_id}) User killed running job, but error encountered during termination: {e}")
def recover(self, job, job_wrapper):
"""Recovers jobs stuck in the queued/running state when Galaxy started"""
@@ -250,12 +250,12 @@ class ShellJobRunner(AsynchronousJobRunner):
ajs.job_wrapper = job_wrapper
ajs.job_destination = job_wrapper.job_destination
if job.state == model.Job.states.RUNNING:
log.debug("({}/{}) is still in running state, adding to the runner monitor queue".format(job.id, job.job_runner_external_id))
log.debug(f"({job.id}/{job.job_runner_external_id}) is still in running state, adding to the runner monitor queue")
ajs.old_state = model.Job.states.RUNNING
ajs.running = True
self.monitor_queue.put(ajs)
elif job.state == model.Job.states.QUEUED:
log.debug("({}/{}) is still in queued state, adding to the runner monitor queue".format(job.id, job.job_runner_external_id))
log.debug(f"({job.id}/{job.job_runner_external_id}) is still in queued state, adding to the runner monitor queue")
ajs.old_state = model.Job.states.QUEUED
ajs.running = False
self.monitor_queue.put(ajs)
+19 -19
View File
@@ -63,7 +63,7 @@ class CondorJobRunner(AsynchronousJobRunner):
Remove this function in 21.01
"""
if cjs.job_wrapper is not None:
job_file = "{}/galaxy_{}.sh".format(self.app.config.cluster_files_directory, cjs.job_wrapper.job_id)
job_file = f"{self.app.config.cluster_files_directory}/galaxy_{cjs.job_wrapper.job_id}.sh"
if not os.path.exists(cjs.job_file) and os.path.exists(job_file):
cluster_files_dir_and_id = (self.app.config.cluster_files_directory, cjs.job_wrapper.get_id_tag())
cjs.output_file = "%s/galaxy_%s.o" % cluster_files_dir_and_id
@@ -155,11 +155,11 @@ class CondorJobRunner(AsynchronousJobRunner):
job_wrapper.cleanup()
return
log.debug("({}) submitting file {}".format(galaxy_id_tag, executable))
log.debug(f"({galaxy_id_tag}) submitting file {executable}")
external_job_id, message = condor_submit(submit_file)
if external_job_id is None:
log.debug("condor_submit failed for job {}: {}".format(job_wrapper.get_id_tag(), message))
log.debug(f"condor_submit failed for job {job_wrapper.get_id_tag()}: {message}")
if self.app.config.cleanup_job == "always":
os.unlink(submit_file)
cjs.cleanup()
@@ -168,7 +168,7 @@ class CondorJobRunner(AsynchronousJobRunner):
os.unlink(submit_file)
log.info("({}) queued as {}".format(galaxy_id_tag, external_job_id))
log.info(f"({galaxy_id_tag}) queued as {external_job_id}")
# store runner information for tracking if Galaxy restarts
job_wrapper.set_external_id(external_job_id)
@@ -201,8 +201,8 @@ class CondorJobRunner(AsynchronousJobRunner):
cjs.user_log_size = log_size
except Exception:
# so we don't kill the monitor thread
log.exception("({}/{}) Unable to check job status".format(galaxy_id_tag, job_id))
log.warning("({}/{}) job will now be errored".format(galaxy_id_tag, job_id))
log.exception(f"({galaxy_id_tag}/{job_id}) Unable to check job status")
log.warning(f"({galaxy_id_tag}/{job_id}) job will now be errored")
cjs.fail_message = "Cluster could not complete job"
self.work_queue.put((self.fail_job, cjs))
continue
@@ -212,10 +212,10 @@ class CondorJobRunner(AsynchronousJobRunner):
cjs.job_wrapper.check_for_entry_points()
if job_running and not cjs.running:
log.debug("({}/{}) job is now running".format(galaxy_id_tag, job_id))
log.debug(f"({galaxy_id_tag}/{job_id}) job is now running")
cjs.job_wrapper.change_state(model.Job.states.RUNNING)
if not job_running and cjs.running:
log.debug("({}/{}) job has stopped running".format(galaxy_id_tag, job_id))
log.debug(f"({galaxy_id_tag}/{job_id}) job has stopped running")
# Will switching from RUNNING to QUEUED confuse Galaxy?
# cjs.job_wrapper.change_state( model.Job.states.QUEUED )
if job_complete:
@@ -223,11 +223,11 @@ class CondorJobRunner(AsynchronousJobRunner):
external_metadata = not asbool(cjs.job_wrapper.job_destination.params.get("embed_metadata_in_job", True))
if external_metadata:
self._handle_metadata_externally(cjs.job_wrapper, resolve_requirements=True)
log.debug("({}/{}) job has completed".format(galaxy_id_tag, job_id))
log.debug(f"({galaxy_id_tag}/{job_id}) job has completed")
self.work_queue.put((self.finish_job, cjs))
continue
if job_failed:
log.debug("({}/{}) job failed".format(galaxy_id_tag, job_id))
log.debug(f"({galaxy_id_tag}/{job_id}) job failed")
cjs.failed = True
self.work_queue.put((self.finish_job, cjs))
continue
@@ -243,7 +243,7 @@ class CondorJobRunner(AsynchronousJobRunner):
galaxy_id_tag = job_wrapper.get_id_tag()
if job.container:
try:
log.info("stop_job(): {}: trying to stop container .... ({})".format(job.id, external_id))
log.info(f"stop_job(): {job.id}: trying to stop container .... ({external_id})")
# self.watched = [cjs for cjs in self.watched if cjs.job_id != external_id]
new_watch_list = list()
cjs = None
@@ -260,21 +260,21 @@ class CondorJobRunner(AsynchronousJobRunner):
external_metadata = not asbool(cjs.job_wrapper.job_destination.params.get("embed_metadata_in_job", True))
if external_metadata:
self._handle_metadata_externally(cjs.job_wrapper, resolve_requirements=True)
log.debug("({}/{}) job has completed".format(galaxy_id_tag, external_id))
log.debug(f"({galaxy_id_tag}/{external_id}) job has completed")
self.work_queue.put((self.finish_job, cjs))
except Exception as e:
log.warning("stop_job(): {}: trying to stop container failed. ({})".format(job.id, e))
log.warning(f"stop_job(): {job.id}: trying to stop container failed. ({e})")
try:
self._kill_container(job_wrapper)
except Exception as e:
log.warning("stop_job(): {}: trying to kill container failed. ({})".format(job.id, e))
log.warning(f"stop_job(): {job.id}: trying to kill container failed. ({e})")
failure_message = condor_stop(external_id)
if failure_message:
log.debug("({}). Failed to stop condor {}".format(external_id, failure_message))
log.debug(f"({external_id}). Failed to stop condor {failure_message}")
else:
failure_message = condor_stop(external_id)
if failure_message:
log.debug("({}). Failed to stop condor {}".format(external_id, failure_message))
log.debug(f"({external_id}). Failed to stop condor {failure_message}")
def recover(self, job, job_wrapper):
"""Recovers jobs stuck in the queued/running state when Galaxy started"""
@@ -293,11 +293,11 @@ class CondorJobRunner(AsynchronousJobRunner):
cjs.register_cleanup_file_attribute('user_log')
self.__old_state_paths(cjs) # remove in 21.01
if job.state == model.Job.states.RUNNING:
log.debug("({}/{}) is still in running state, adding to the DRM queue".format(job.id, job.job_runner_external_id))
log.debug(f"({job.id}/{job.job_runner_external_id}) is still in running state, adding to the DRM queue")
cjs.running = True
self.monitor_queue.put(cjs)
elif job.state == model.Job.states.QUEUED:
log.debug("({}/{}) is still in DRM queued state, adding to the DRM queue".format(job.id, job.job_runner_external_id))
log.debug(f"({job.id}/{job.job_runner_external_id}) is still in DRM queued state, adding to the DRM queue")
cjs.running = False
self.monitor_queue.put(cjs)
@@ -317,7 +317,7 @@ class CondorJobRunner(AsynchronousJobRunner):
return self._run_command(cont.container_info['commands'][command], external_id)[0]
def _run_command(self, command, external_job_id):
command = 'condor_ssh_to_job {} {}'.format(external_job_id, command)
command = f'condor_ssh_to_job {external_job_id} {command}'
p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, close_fds=True, preexec_fn=os.setpgrp)
stdout, stderr = p.communicate()
+12 -12
View File
@@ -106,7 +106,7 @@ class DRMAAJobRunner(AsynchronousJobRunner):
native_spec = url.split('/')[2]
if native_spec:
params = dict(nativeSpecification=native_spec)
log.debug("Converted URL '{}' to destination runner=drmaa, params={}".format(url, params))
log.debug(f"Converted URL '{url}' to destination runner=drmaa, params={params}")
return JobDestination(runner='drmaa', params=params)
else:
log.debug("Converted URL '%s' to destination runner=drmaa" % url)
@@ -218,7 +218,7 @@ class DRMAAJobRunner(AsynchronousJobRunner):
if external_job_id is None:
job_wrapper.fail("(%s) could not queue job" % galaxy_id_tag)
return
log.info("({}) queued as {}".format(galaxy_id_tag, external_job_id))
log.info(f"({galaxy_id_tag}) queued as {external_job_id}")
# store runner information for tracking if Galaxy restarts
job_wrapper.set_external_id(external_job_id)
@@ -274,7 +274,7 @@ class DRMAAJobRunner(AsynchronousJobRunner):
galaxy_id_tag = ajs.job_wrapper.get_id_tag()
state = None
try:
assert external_job_id not in (None, 'None'), '({}/{}) Invalid job id'.format(galaxy_id_tag, external_job_id)
assert external_job_id not in (None, 'None'), f'({galaxy_id_tag}/{external_job_id}) Invalid job id'
state = self.ds.job_status(external_job_id)
# Reset exception retries
for retry_exception in RETRY_EXCEPTIONS_LOWER:
@@ -306,8 +306,8 @@ class DRMAAJobRunner(AsynchronousJobRunner):
return None
except Exception:
# so we don't kill the monitor thread
log.exception("({}/{}) unable to check job status".format(galaxy_id_tag, external_job_id))
log.warning("({}/{}) job will now be errored".format(galaxy_id_tag, external_job_id))
log.exception(f"({galaxy_id_tag}/{external_job_id}) unable to check job status")
log.warning(f"({galaxy_id_tag}/{external_job_id}) job will now be errored")
ajs.fail_message = "Cluster could not complete job"
self.work_queue.put((self.fail_job, ajs))
return None
@@ -361,13 +361,13 @@ class DRMAAJobRunner(AsynchronousJobRunner):
cmd = shlex.split(kill_script)
cmd.extend([str(ext_id), str(self.userid)])
commands.execute(cmd)
log.info("({}/{}) Removed from DRM queue at user's request".format(job.id, ext_id))
log.info(f"({job.id}/{ext_id}) Removed from DRM queue at user's request")
except drmaa.InvalidJobException:
log.exception("({}/{}) User killed running job, but it was already dead".format(job.id, ext_id))
log.exception(f"({job.id}/{ext_id}) User killed running job, but it was already dead")
except commands.CommandLineException as e:
log.error("({}/{}) User killed running job, but command execution failed: {}".format(job.id, ext_id, unicodify(e)))
except Exception:
log.exception("({}/{}) User killed running job, but error encountered removing from DRM queue".format(job.id, ext_id))
log.exception(f"({job.id}/{ext_id}) User killed running job, but error encountered removing from DRM queue")
def recover(self, job, job_wrapper):
"""Recovers jobs stuck in the queued/running state when Galaxy started"""
@@ -381,12 +381,12 @@ class DRMAAJobRunner(AsynchronousJobRunner):
ajs.job_wrapper = job_wrapper
ajs.job_destination = job_wrapper.job_destination
if job.state == model.Job.states.RUNNING:
log.debug("({}/{}) is still in running state, adding to the DRM queue".format(job.id, job.get_job_runner_external_id()))
log.debug(f"({job.id}/{job.get_job_runner_external_id()}) is still in running state, adding to the DRM queue")
ajs.old_state = drmaa.JobState.RUNNING
ajs.running = True
self.monitor_queue.put(ajs)
elif job.get_state() == model.Job.states.QUEUED:
log.debug("({}/{}) is still in DRM queued state, adding to the DRM queue".format(job.id, job.get_job_runner_external_id()))
log.debug(f"({job.id}/{job.get_job_runner_external_id()}) is still in DRM queued state, adding to the DRM queue")
ajs.old_state = drmaa.JobState.QUEUED_ACTIVE
ajs.running = False
self.monitor_queue.put(ajs)
@@ -395,10 +395,10 @@ class DRMAAJobRunner(AsynchronousJobRunner):
""" Stores the content of a DRMAA JobTemplate object in a file as a JSON string.
Path is hard-coded, but it's no worse than other path in this module.
Uses Galaxy's JobID, so file is expected to be unique."""
filename = "{}/{}.jt_json".format(self.app.config.cluster_files_directory, job_wrapper.get_id_tag())
filename = f"{self.app.config.cluster_files_directory}/{job_wrapper.get_id_tag()}.jt_json"
with open(filename, 'w+') as fp:
json.dump(jt, fp)
log.debug('({}) Job script for external submission is: {}'.format(job_wrapper.job_id, filename))
log.debug(f'({job_wrapper.job_id}) Job script for external submission is: {filename}')
return filename
def external_runjob(self, external_runjob_script, jobtemplate_filename, username):
+2 -2
View File
@@ -253,13 +253,13 @@ class GodockerJobRunner(AsynchronousJobRunner):
job_wrapper.command_line = job.command_line
ajs.job_wrapper = job_wrapper
if job.state == model.Job.states.RUNNING:
log.debug("({}/{}) is still in running state, adding to the god queue".format(job.id, job.get_job_runner_external_id()))
log.debug(f"({job.id}/{job.get_job_runner_external_id()}) is still in running state, adding to the god queue")
ajs.old_state = 'R'
ajs.running = True
self.monitor_queue.put(ajs)
elif job.state == model.Job.states.QUEUED:
log.debug("({}/{}) is still in god queued state, adding to the god queue".format(job.id, job.get_job_runner_external_id()))
log.debug(f"({job.id}/{job.get_job_runner_external_id()}) is still in god queued state, adding to the god queue")
ajs.old_state = 'Q'
ajs.running = False
self.monitor_queue.put(ajs)
+1 -1
View File
@@ -527,7 +527,7 @@ class KubernetesJobRunner(AsynchronousJobRunner):
self.__cleanup_k8s_job(job_to_delete)
# TODO assert whether job parallelism == 0
# assert not job_to_delete.exists(), "Could not delete job,"+job.job_runner_external_id+" it still exists"
log.debug("({}/{}) Terminated at user's request".format(job.id, job.job_runner_external_id))
log.debug(f"({job.id}/{job.job_runner_external_id}) Terminated at user's request")
except Exception as e:
log.exception("({}/{}) User killed running job, but error encountered during termination: {}".format(
job.id, job.job_runner_external_id, e))
+1 -1
View File
@@ -94,7 +94,7 @@ class LocalJobRunner(BaseJobRunner):
try:
stdout_file = tempfile.NamedTemporaryFile(mode='wb+', suffix='_stdout', dir=job_wrapper.working_directory)
stderr_file = tempfile.NamedTemporaryFile(mode='wb+', suffix='_stderr', dir=job_wrapper.working_directory)
log.debug('({}) executing job script: {}'.format(job_id, job_file))
log.debug(f'({job_id}) executing job script: {job_file}')
# The preexec_fn argument of Popen() is used to call os.setpgrp() in
# the child process just before the child is executed. This will set
# the PGID of the child process to its PID (i.e. ensures that it is
+28 -28
View File
@@ -130,7 +130,7 @@ class PBSJobRunner(AsynchronousJobRunner):
pbs_destination = '@%s' % server
pbs_queue = url_split[3] or None
if pbs_queue is not None:
pbs_destination = '{}{}'.format(pbs_queue, pbs_destination)
pbs_destination = f'{pbs_queue}{pbs_destination}'
params = dict(destination=pbs_destination)
@@ -147,7 +147,7 @@ class PBSJobRunner(AsynchronousJobRunner):
param, value = opt.split(None, 1)
params[param] = value
log.debug("Converted URL '{}' to destination runner=pbs, params={}".format(url, params))
log.debug(f"Converted URL '{url}' to destination runner=pbs, params={params}")
# Create a dynamic JobDestination
return JobDestination(runner='pbs', params=params)
@@ -186,7 +186,7 @@ class PBSJobRunner(AsynchronousJobRunner):
try:
rval.append(dict(name=getattr(pbs, 'ATTR_' + arg), value=value))
except AttributeError as e:
raise Exception("Invalid parameter '{}': {}".format(arg, e))
raise Exception(f"Invalid parameter '{arg}': {e}")
return rval
def __get_pbs_server(self, job_destination_params):
@@ -230,13 +230,13 @@ class PBSJobRunner(AsynchronousJobRunner):
if c <= 0:
errno, text = pbs.error()
job_wrapper.fail("Unable to queue job for execution. Resubmitting the job may succeed.")
log.error("Connection to PBS server for submit failed: {}: {}".format(errno, text))
log.error(f"Connection to PBS server for submit failed: {errno}: {text}")
return
# define job attributes
ofile = "{}/{}.o".format(self.app.config.cluster_files_directory, job_wrapper.job_id)
efile = "{}/{}.e".format(self.app.config.cluster_files_directory, job_wrapper.job_id)
ecfile = "{}/{}.ec".format(self.app.config.cluster_files_directory, job_wrapper.job_id)
ofile = f"{self.app.config.cluster_files_directory}/{job_wrapper.job_id}.o"
efile = f"{self.app.config.cluster_files_directory}/{job_wrapper.job_id}.e"
ecfile = f"{self.app.config.cluster_files_directory}/{job_wrapper.job_id}.ec"
output_fnames = job_wrapper.get_output_fnames()
@@ -262,7 +262,7 @@ class PBSJobRunner(AsynchronousJobRunner):
]
# define PBS job options
attrs.append(dict(name=pbs.ATTR_N, value=str("{}_{}_{}".format(job_wrapper.job_id, job_wrapper.tool.id, job_wrapper.user))))
attrs.append(dict(name=pbs.ATTR_N, value=str(f"{job_wrapper.job_id}_{job_wrapper.tool.id}_{job_wrapper.user}")))
job_attrs = pbs.new_attropl(len(attrs) + len(pbs_options))
for i, attr in enumerate(attrs + pbs_options):
job_attrs[i].name = attr['name']
@@ -287,7 +287,7 @@ class PBSJobRunner(AsynchronousJobRunner):
env_setup_commands = [stage_commands]
script = self.get_job_file(job_wrapper, exit_code_path=ecfile, env_setup_commands=env_setup_commands, shell=job_wrapper.shell)
job_file = "{}/{}.sh".format(self.app.config.cluster_files_directory, job_wrapper.job_id)
job_file = f"{self.app.config.cluster_files_directory}/{job_wrapper.job_id}.sh"
self.write_executable_script(job_file, script)
# job was deleted while we were preparing it
if job_wrapper.get_state() == model.Job.states.DELETED:
@@ -302,7 +302,7 @@ class PBSJobRunner(AsynchronousJobRunner):
# The job tag includes the job and the task identifier
# (if a TaskWrapper was passed in):
galaxy_job_id = job_wrapper.get_id_tag()
log.debug("({}) submitting file {}".format(galaxy_job_id, job_file))
log.debug(f"({galaxy_job_id}) submitting file {job_file}")
tries = 0
while tries < 5:
@@ -320,9 +320,9 @@ class PBSJobRunner(AsynchronousJobRunner):
return
if pbs_queue_name is None:
log.debug("({}) queued in default queue as {}".format(galaxy_job_id, job_id))
log.debug(f"({galaxy_job_id}) queued in default queue as {job_id}")
else:
log.debug("({}) queued in {} queue as {}".format(galaxy_job_id, pbs_queue_name, job_id))
log.debug(f"({galaxy_job_id}) queued in {pbs_queue_name} queue as {job_id}")
# persist destination
job_wrapper.set_job_destination(job_destination, job_id)
@@ -356,7 +356,7 @@ class PBSJobRunner(AsynchronousJobRunner):
old_state = pbs_job_state.old_state
pbs_server_name = self.__get_pbs_server(pbs_job_state.job_destination.params)
if pbs_server_name in failures:
log.debug("({}/{}) Skipping state check because PBS server connection failed".format(galaxy_job_id, job_id))
log.debug(f"({galaxy_job_id}/{job_id}) Skipping state check because PBS server connection failed")
new_watched.append(pbs_job_state)
continue
try:
@@ -367,13 +367,13 @@ class PBSJobRunner(AsynchronousJobRunner):
try:
# Recheck to make sure it wasn't a communication problem
self.check_single_job(pbs_server_name, job_id)
log.warning("({}/{}) PBS job was not in state check list, but was found with individual state check".format(galaxy_job_id, job_id))
log.warning(f"({galaxy_job_id}/{job_id}) PBS job was not in state check list, but was found with individual state check")
new_watched.append(pbs_job_state)
except Exception:
errno, text = pbs.error()
if errno == 15001:
# 15001 == job not in queue
log.debug("({}/{}) PBS job has left queue".format(galaxy_job_id, job_id))
log.debug(f"({galaxy_job_id}/{job_id}) PBS job has left queue")
self.work_queue.put((self.finish_job, pbs_job_state))
else:
# Unhandled error, continue to monitor
@@ -381,7 +381,7 @@ class PBSJobRunner(AsynchronousJobRunner):
new_watched.append(pbs_job_state)
continue
if status.job_state != old_state:
log.debug("({}/{}) PBS job state changed from {} to {}".format(galaxy_job_id, job_id, old_state, status.job_state))
log.debug(f"({galaxy_job_id}/{job_id}) PBS job state changed from {old_state} to {status.job_state}")
if status.job_state == "R" and not pbs_job_state.running:
pbs_job_state.running = True
pbs_job_state.job_wrapper.change_state(model.Job.states.RUNNING)
@@ -396,18 +396,18 @@ class PBSJobRunner(AsynchronousJobRunner):
# "keep_completed" is enabled in PBS, so try to check exit status
try:
assert int(status.exit_status) == 0
log.debug("({}/{}) PBS job has completed successfully".format(galaxy_job_id, job_id))
log.debug(f"({galaxy_job_id}/{job_id}) PBS job has completed successfully")
except AssertionError:
exit_status = int(status.exit_status)
error_message = JOB_EXIT_STATUS.get(exit_status, 'Unknown error: %s' % status.exit_status)
pbs_job_state.fail_message = CLUSTER_ERROR_MESSAGE % error_message
log.error('({}/{}) PBS job failed: {}'.format(galaxy_job_id, job_id, error_message))
log.error(f'({galaxy_job_id}/{job_id}) PBS job failed: {error_message}')
pbs_job_state.stop_job = False
self.work_queue.put((self.fail_job, pbs_job_state))
continue
except AttributeError:
# No exit_status, can't verify proper completion so we just have to assume success.
log.debug("({}/{}) PBS job has completed".format(galaxy_job_id, job_id))
log.debug(f"({galaxy_job_id}/{job_id}) PBS job has completed")
self.work_queue.put((self.finish_job, pbs_job_state))
continue
pbs_job_state.old_state = status.job_state
@@ -497,14 +497,14 @@ class PBSJobRunner(AsynchronousJobRunner):
stage_name = os.path.join(self.app.config.pbs_stage_path, os.path.split(fname)[1])
else:
stage_name = fname
stage += "{}@{}:{}".format(stage_name, self.app.config.pbs_dataset_server, fname)
stage += f"{stage_name}@{self.app.config.pbs_dataset_server}:{fname}"
return stage
def stop_job(self, job_wrapper):
"""Attempts to delete a job from the PBS queue"""
job = job_wrapper.get_job()
job_id = job.get_job_runner_external_id().encode('utf-8')
job_tag = "({}/{})".format(job.get_id_tag(), job_id)
job_tag = f"({job.get_id_tag()}/{job_id})"
log.debug("%s Stopping PBS job" % job_tag)
# Declare the connection handle c so that it can be cleaned up:
@@ -526,7 +526,7 @@ class PBSJobRunner(AsynchronousJobRunner):
% job_tag)
except Exception:
e = traceback.format_exc()
log.debug("{} Unable to stop job: {}".format(job_tag, e))
log.debug(f"{job_tag} Unable to stop job: {e}")
finally:
# Cleanup: disconnect from the server.
if (None is not c):
@@ -536,22 +536,22 @@ class PBSJobRunner(AsynchronousJobRunner):
"""Recovers jobs stuck in the queued/running state when Galaxy started"""
job_id = job.get_job_runner_external_id()
pbs_job_state = AsynchronousJobState()
pbs_job_state.output_file = "{}/{}.o".format(self.app.config.cluster_files_directory, job.id)
pbs_job_state.error_file = "{}/{}.e".format(self.app.config.cluster_files_directory, job.id)
pbs_job_state.exit_code_file = "{}/{}.ec".format(self.app.config.cluster_files_directory, job.id)
pbs_job_state.job_file = "{}/{}.sh".format(self.app.config.cluster_files_directory, job.id)
pbs_job_state.output_file = f"{self.app.config.cluster_files_directory}/{job.id}.o"
pbs_job_state.error_file = f"{self.app.config.cluster_files_directory}/{job.id}.e"
pbs_job_state.exit_code_file = f"{self.app.config.cluster_files_directory}/{job.id}.ec"
pbs_job_state.job_file = f"{self.app.config.cluster_files_directory}/{job.id}.sh"
pbs_job_state.job_id = str(job_id)
pbs_job_state.runner_url = job_wrapper.get_job_runner_url()
pbs_job_state.job_destination = job_wrapper.job_destination
job_wrapper.command_line = job.command_line
pbs_job_state.job_wrapper = job_wrapper
if job.state == model.Job.states.RUNNING:
log.debug("({}/{}) is still in running state, adding to the PBS queue".format(job.id, job.get_job_runner_external_id()))
log.debug(f"({job.id}/{job.get_job_runner_external_id()}) is still in running state, adding to the PBS queue")
pbs_job_state.old_state = 'R'
pbs_job_state.running = True
self.monitor_queue.put(pbs_job_state)
elif job.state == model.Job.states.QUEUED:
log.debug("({}/{}) is still in PBS queued state, adding to the PBS queue".format(job.id, job.get_job_runner_external_id()))
log.debug(f"({job.id}/{job.get_job_runner_external_id()}) is still in PBS queued state, adding to the PBS queue")
pbs_job_state.old_state = 'Q'
pbs_job_state.running = False
self.monitor_queue.put(pbs_job_state)
+3 -3
View File
@@ -539,7 +539,7 @@ class PulsarJobRunner(AsynchronousJobRunner):
def get_client_from_wrapper(self, job_wrapper):
job_id = job_wrapper.job_id
if hasattr(job_wrapper, 'task_id'):
job_id = "{}_{}".format(job_id, job_wrapper.task_id)
job_id = f"{job_id}_{job_wrapper.task_id}"
params = job_wrapper.job_destination.params.copy()
user = job_wrapper.get_job().user
if user:
@@ -685,7 +685,7 @@ class PulsarJobRunner(AsynchronousJobRunner):
# Remote kill
pulsar_url = job.job_runner_name
job_id = job.job_runner_external_id
log.debug("Attempt remote Pulsar kill of job with url {} and id {}".format(pulsar_url, job_id))
log.debug(f"Attempt remote Pulsar kill of job with url {pulsar_url} and id {job_id}")
client = self.get_client(job.destination_params, job_id)
client.kill()
@@ -1010,7 +1010,7 @@ class PulsarComputeEnvironment(ComputeEnvironment):
# first but that adds untested logic that wouln't ever be used.
remote_input_path = self.path_mapper.remote_input_path_rewrite(metadata_val, client_input_path_type=CLIENT_INPUT_PATH_TYPES.INPUT_METADATA_PATH)
if remote_input_path:
log.info("input_metadata_rewrite is {} from {}".format(remote_input_path, metadata_val))
log.info(f"input_metadata_rewrite is {remote_input_path} from {metadata_val}")
self.path_rewrites_input_metadata[metadata_val] = remote_input_path
return remote_input_path
@@ -93,7 +93,7 @@ def _handle_resubmit_definitions(resubmit_definitions, app, job_runner, job_stat
external_id = getattr(job_state, "job_id", None)
if external_id:
job_log_prefix = "({}/{})".format(job_state.job_wrapper.job_id, job_state.job_id)
job_log_prefix = f"({job_state.job_wrapper.job_id}/{job_state.job_id})"
else:
job_log_prefix = "(%s)" % (job_state.job_wrapper.job_id)
+13 -13
View File
@@ -91,13 +91,13 @@ class UnivaJobRunner(DRMAAJobRunner):
ajs.fail_message = "This job failed because it was cancelled."
drmaa_state = self.drmaa.JobState.FAILED
elif ("signal" in extinfo and extinfo["signal"] == "SIGKILL") and time_wasted > time_granted:
log.error('({tag}/{jobid}) Job hit walltime'.format(tag=ajs.job_wrapper.get_id_tag(), jobid=ajs.job_id))
log.error(f'({ajs.job_wrapper.get_id_tag()}/{ajs.job_id}) Job hit walltime')
ajs.fail_message = "This job was terminated because it ran longer than the maximum allowed job run time."
ajs.runner_state = ajs.runner_states.WALLTIME_REACHED
drmaa_state = self.drmaa.JobState.FAILED
# test wasted>granted memory only if failed != 0 and exit_status != 0, ie if marked as failed
elif state == self.drmaa.JobState.FAILED and mem_wasted > mem_granted * slots:
log.error('({idtag}/{jobid}) Job hit memory limit ({used}>{limit})'.format(idtag=ajs.job_wrapper.get_id_tag(), jobid=ajs.job_id, used=mem_wasted, limit=mem_granted))
log.error(f'({ajs.job_wrapper.get_id_tag()}/{ajs.job_id}) Job hit memory limit ({mem_wasted}>{mem_granted})')
ajs.fail_message = "This job was terminated because it used more than the maximum allowed memory."
ajs.runner_state = ajs.runner_states.MEMORY_LIMIT_REACHED
drmaa_state = self.drmaa_job_states.FAILED
@@ -106,10 +106,10 @@ class UnivaJobRunner(DRMAAJobRunner):
# TODO return True?
return True # job was not actually terminal
elif state == self.drmaa.JobState.UNDETERMINED:
log.warning('({tag}/{jobid}) Job state could not be determined'.format(tag=ajs.job_wrapper.get_id_tag(), jobid=ajs.job_id))
log.warning(f'({ajs.job_wrapper.get_id_tag()}/{ajs.job_id}) Job state could not be determined')
drmaa_state = self.drmaa_job_states.FAILED
else:
log.error("DRMAAUniva: job {job_id} determined unknown state {state}".format(job_id=ajs.job_id, state=state))
log.error(f"DRMAAUniva: job {ajs.job_id} determined unknown state {state}")
drmaa_state = self.drmaa_job_states.FAILED
# by default, finish the job with the state from drmaa
return super()._complete_terminal_job(ajs, drmaa_state=drmaa_state)
@@ -203,7 +203,7 @@ class UnivaJobRunner(DRMAAJobRunner):
try:
stdout = commands.execute(cmd).strip()
except commands.CommandLineException as e:
if slp <= 32 and "job id {jobid} not found".format(jobid=job_id) in e.stderr:
if slp <= 32 and f"job id {job_id} not found" in e.stderr:
time.sleep(slp)
slp *= 2
continue
@@ -397,7 +397,7 @@ class UnivaJobRunner(DRMAAJobRunner):
extinfo["slots"] = float(rv.resourceUsage['slots'])
# log.debug("wait -> \texitStatus {0}\thasCoreDump {1}\thasExited {2}\thasSignal {3}\tjobId {4}\t\tterminatedSignal {5}\twasAborted {6}\tresourceUsage {7}".format(rv.exitStatus, rv.hasCoreDump, rv.hasExited, rv.hasSignal, rv.jobId, rv.terminatedSignal, rv.wasAborted, rv.resourceUsage))
if rv.wasAborted:
log.error("DRMAAUniva: job {job_id} was aborted according to wait()".format(job_id=job_id))
log.error(f"DRMAAUniva: job {job_id} was aborted according to wait()")
extinfo["deleted"] = True
return self.drmaa.JobState.FAILED
@@ -405,19 +405,19 @@ class UnivaJobRunner(DRMAAJobRunner):
# but also violation of scheduler constraints
state = self.drmaa.JobState.DONE
if rv.exitStatus != 0:
log.error("DRMAAUniva: job {job_id} has exit status {status}".format(job_id=job_id, status=rv.exitStatus))
log.error(f"DRMAAUniva: job {job_id} has exit status {rv.exitStatus}")
extinfo["state"] = self.drmaa.JobState.FAILED
if not rv.hasExited or rv.hasSignal:
if rv.hasCoreDump != 0:
log.error("DRMAAUniva: job {job_id} has core dump".format(job_id=job_id))
log.error(f"DRMAAUniva: job {job_id} has core dump")
extinfo["state"] = self.drmaa.JobState.FAILED
elif len(rv.terminatedSignal) > 0:
log.error("DRMAAUniva: job {job_id} was kill by signal {signal}".format(job_id=job_id, signal=rv.terminatedSignal))
log.error(f"DRMAAUniva: job {job_id} was kill by signal {rv.terminatedSignal}")
state = self.drmaa.JobState.FAILED
extinfo["signal"] = rv.terminatedSignal
elif rv.wasAborted == 0:
log.error("DRMAAUniva: job {job_id} has finished in unclear condition".format(job_id=job_id))
log.error(f"DRMAAUniva: job {job_id} has finished in unclear condition")
state = self.drmaa.JobState.FAILED
# log.debug("UnivaJobRunner._get_drmaa_state_wait ({jobid}) -> {state}".format(jobid=job_id, state=self.drmaa_job_state_strings[state]))
return state
@@ -535,7 +535,7 @@ class UnivaJobRunner(DRMAAJobRunner):
elif "w" in state:
return self.drmaa.JobState.QUEUED_ACTIVE
else:
log.error("DRMAAUniva: job {job_id} unknown state from qstat: {state}".format(job_id=job_id, state=state))
log.error(f"DRMAAUniva: job {job_id} unknown state from qstat: {state}")
return self.drmaa.JobState.UNDETERMINED
@@ -569,12 +569,12 @@ def _parse_native_specs(job_id, native_spec):
if m is not None:
tme = _parse_time(m.group(1))
if tme is None:
log.error("DRMAAUniva: job {job_id} has unparsable time native spec {spec}".format(job_id=job_id, spec=native_spec))
log.error(f"DRMAAUniva: job {job_id} has unparsable time native spec {native_spec}")
# parse memory
m = re.search(r"mem=([\d.]+[KGMT]?)[\s,]*", native_spec)
if m is not None:
mem = size_to_bytes(m.group(1))
# mem = _parse_mem(m.group(1))
if mem is None:
log.error("DRMAAUniva: job {job_id} has unparsable memory native spec {spec}".format(job_id=job_id, spec=native_spec))
log.error(f"DRMAAUniva: job {job_id} has unparsable memory native spec {native_spec}")
return tme, mem
+1 -1
View File
@@ -39,7 +39,7 @@ class CliInterface:
def __load_from_path(module_path):
base_module = importlib.import_module(module_path)
for module_info in pkgutil.iter_modules(base_module.__path__):
module = importlib.import_module('{}.{}'.format(module_path, module_info.name))
module = importlib.import_module(f'{module_path}.{module_info.name}')
yield module
def __load(module_path, d):
+1 -1
View File
@@ -52,7 +52,7 @@ class LSF(BaseJobExec):
# Generated template.
template_scriptargs = ''
for k, v in scriptargs.items():
template_scriptargs += '#BSUB {} {}\n'.format(k, v)
template_scriptargs += f'#BSUB {k} {v}\n'
return dict(headers=template_scriptargs)
def submit(self, script_file):
@@ -47,7 +47,7 @@ class Slurm(BaseJobExec):
# Generated template.
template_scriptargs = ''
for k, v in scriptargs.items():
template_scriptargs += '#SBATCH {} {}\n'.format(k, v)
template_scriptargs += f'#SBATCH {k} {v}\n'
return dict(headers=template_scriptargs)
def submit(self, script_file):
@@ -59,7 +59,7 @@ class Torque(BaseJobExec):
log.warning(ERROR_MESSAGE_UNRECOGNIZED_ARG % k)
template_pbsargs = ''
for k, v in pbsargs.items():
template_pbsargs += '#PBS {} {}\n'.format(k, v)
template_pbsargs += f'#PBS {k} {v}\n'
return dict(headers=template_pbsargs)
def submit(self, script_file):
@@ -58,7 +58,7 @@ def build_submit_description(executable, output, error, user_log, query_params):
submit_description = []
for key, value in all_query_params.items():
submit_description.append('{} = {}'.format(key, value))
submit_description.append(f'{key} = {value}')
submit_description.append('executable = ' + executable)
submit_description.append('output = ' + output)
submit_description.append('error = ' + error)
+1 -1
View File
@@ -29,7 +29,7 @@ def env_to_statement(env):
return execute
name = env['name']
value = __escape(env['value'], env)
return '{}={}; export {}'.format(name, value, name)
return f'{name}={value}; export {name}'
def __escape(value, env):
+4 -5
View File
@@ -5,12 +5,11 @@ IPC with multiple process configurations.
import json
import logging
import os
import shlex
import socket
import subprocess
import threading
from six.moves import shlex_quote
from galaxy.util import (
listify,
sleeper,
@@ -75,12 +74,12 @@ class TransferManager:
# not the case, this process will need to be moved to a
# non-blocking method.
cmd = self.command + [tj.id]
log.debug('Transfer command is: %s', ' '.join(map(shlex_quote, cmd)))
log.debug('Transfer command is: %s', ' '.join(map(shlex.quote, cmd)))
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
p.wait()
output = p.stdout.read(32768)
if p.returncode != 0:
log.error('Spawning transfer job failed: {}: {}'.format(tj.id, output))
log.error(f'Spawning transfer job failed: {tj.id}: {output}')
tj.state = tj.states.ERROR
tj.info = 'Spawning transfer job failed: %s' % output.splitlines()[-1]
self.sa_session.add(tj)
@@ -154,7 +153,7 @@ class TransferManager:
except Exception:
self.sa_session.refresh(tj)
if tj.state == tj.states.RUNNING:
log.error('Transfer job {} is marked as running but pid {} appears to be dead.'.format(tj.id, tj.pid))
log.error(f'Transfer job {tj.id} is marked as running but pid {tj.pid} appears to be dead.')
dead.append(tj)
if dead:
self.run(dead)
+2 -2
View File
@@ -114,8 +114,8 @@ def get_object(trans, id, class_name, check_ownership=False, check_accessible=Fa
item = trans.sa_session.query(item_class).get(decoded_id)
assert item is not None
except Exception:
log.exception("Invalid {} id ( {} ) specified.".format(class_name, id))
raise exceptions.MessageException("Invalid {} id ( {} ) specified".format(class_name, id), type="error")
log.exception(f"Invalid {class_name} id ( {id} ) specified.")
raise exceptions.MessageException(f"Invalid {class_name} id ( {id} ) specified", type="error")
if check_ownership or check_accessible:
security_check(trans, item, check_ownership, check_accessible)
+1 -1
View File
@@ -67,7 +67,7 @@ def parse_citation(elem, citation_manager):
try:
citation = citation_class(elem, citation_manager)
except Exception as e:
raise Exception("Invalid citation of type '{}' with content '{}': {}".format(citation_type, elem.text, e))
raise Exception(f"Invalid citation of type '{citation_type}' with content '{elem.text}': {e}")
return citation
+2 -2
View File
@@ -248,7 +248,7 @@ class CloudManager(sharable.SharableModelManager):
try:
bucket = connection.storage.buckets.get(bucket_name)
if bucket is None:
raise RequestParameterInvalidException("The bucket `{}` not found.".format(bucket_name))
raise RequestParameterInvalidException(f"The bucket `{bucket_name}` not found.")
except Exception as e:
raise ItemAccessibilityException("Could not get the bucket `{}`: {}".format(bucket_name, util.unicodify(e)))
@@ -354,7 +354,7 @@ class CloudManager(sharable.SharableModelManager):
incoming = (util.Params(args, sanitize=False)).__dict__
d2c = trans.app.toolbox.get_tool(SEND_TOOL, SEND_TOOL_VERSION)
if not d2c:
log.debug("Failed to get the `send` tool per user `{}` request.".format(trans.user.id))
log.debug(f"Failed to get the `send` tool per user `{trans.user.id}` request.")
failed.append(json.dumps(
{
"object": object_label,
+1 -1
View File
@@ -110,7 +110,7 @@ class CloudAuthzsDeserializer(base.ModelDeserializer):
decoded_authn_id = self.app.security.decode_id(val)
except Exception:
log.debug("cannot decode authz_id `" + str(val) + "`")
raise MalformedId("Invalid `authz_id` {}!".format(val))
raise MalformedId(f"Invalid `authz_id` {val}!")
trans = context.get("trans")
if trans is None:
+3 -3
View File
@@ -78,12 +78,12 @@ def get_collection(collection, name=""):
hdas = []
if collection.has_subcollections:
for element in collection.elements:
subnames, subhdas = get_collection_elements(element.child_collection, name="{}/{}".format(name, element.element_identifier))
subnames, subhdas = get_collection_elements(element.child_collection, name=f"{name}/{element.element_identifier}")
names.extend(subnames)
hdas.extend(subhdas)
else:
for element in collection.elements:
names.append("{}/{}".format(name, element.element_identifier))
names.append(f"{name}/{element.element_identifier}")
hdas.append(element.dataset_instance)
return names, hdas
@@ -92,7 +92,7 @@ def get_collection_elements(collection, name=""):
names = []
hdas = []
for element in collection.elements:
full_element_name = "{}/{}".format(name, element.element_identifier)
full_element_name = f"{name}/{element.element_identifier}"
if element.is_collection:
subnames, subhdas = get_collection(element.child_collection, name=full_element_name)
names.extend(subnames)
+1 -1
View File
@@ -38,7 +38,7 @@ def get_path_key(path_tuple):
# we remove the last 2 items of the path tuple (values and list index)
return path_key
if path_key:
path_key = "{}{}{}".format(path_key, sep, p)
path_key = f"{path_key}{sep}{p}"
else:
path_key = p
return path_key
+2 -2
View File
@@ -123,7 +123,7 @@ class LibraryDatasetsManager(datasets.DatasetAssociationManager):
continue
if key in ('name'):
if len(val) < MINIMUM_STRING_LENGTH:
raise RequestParameterInvalidException('{} must have at least length of {}'.format(key, MINIMUM_STRING_LENGTH))
raise RequestParameterInvalidException(f'{key} must have at least length of {MINIMUM_STRING_LENGTH}')
val = validation.validate_and_sanitize_basestring(key, val)
validated_payload[key] = val
if key in ('misc_info', 'message'):
@@ -136,7 +136,7 @@ class LibraryDatasetsManager(datasets.DatasetAssociationManager):
validated_payload[key] = val
if key in ('genome_build'):
if len(val) < MINIMUM_STRING_LENGTH:
raise RequestParameterInvalidException('{} must have at least length of {}'.format(key, MINIMUM_STRING_LENGTH))
raise RequestParameterInvalidException(f'{key} must have at least length of {MINIMUM_STRING_LENGTH}')
val = validation.validate_and_sanitize_basestring(key, val)
validated_payload[key] = val
if key in ('tags'):
+2 -2
View File
@@ -42,7 +42,7 @@ GALAXY_FLAVORED_MARKDOWN_CONTAINER_REGEX = r'(?P<container>%s)' % "|".join(GALAX
ARG_VAL_REGEX = r'''[\w_\-]+|\"[^\"]+\"|\'[^\']+\''''
FUNCTION_ARG = r'\s*\w+\s*=\s*(?:%s)\s*' % ARG_VAL_REGEX
# embed commas between arguments
FUNCTION_MULTIPLE_ARGS = r'(?P<firstargcall>{})(?P<restargcalls>(?:,{})*)'.format(FUNCTION_ARG, FUNCTION_ARG)
FUNCTION_MULTIPLE_ARGS = fr'(?P<firstargcall>{FUNCTION_ARG})(?P<restargcalls>(?:,{FUNCTION_ARG})*)'
FUNCTION_MULTIPLE_ARGS_PATTERN = re.compile(FUNCTION_MULTIPLE_ARGS)
FUNCTION_CALL_LINE_TEMPLATE = r'\s*%s\s*\((?:' + FUNCTION_MULTIPLE_ARGS + r')?\)\s*'
GALAXY_MARKDOWN_FUNCTION_CALL_LINE = re.compile(FUNCTION_CALL_LINE_TEMPLATE % GALAXY_FLAVORED_MARKDOWN_CONTAINER_REGEX)
@@ -119,7 +119,7 @@ def validate_galaxy_markdown(galaxy_markdown, internal=True):
if expecting_container_close_for:
template = "Invalid line %d: %s"
msg = template % (last_line_no, "close of block for [{expected_for}] expected".format(expected_for=expecting_container_close_for))
msg = template % (last_line_no, f"close of block for [{expecting_container_close_for}] expected")
raise ValueError(msg)

Some files were not shown because too many files have changed in this diff Show More