mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-10-28 02:58:18 +08:00
Drop support for retired Python 3.5
Upgrade syntax using `pyupgrade --py36-plus` . Manually drop several `six` imports. Also: - Remove broken pr_cache in scripts/bootstrap_history.py - Fix broken prefix removal in lib/galaxy/tool_util/deps/mulled/mulled_build.py
This commit is contained in:
+19
-19
@@ -27,7 +27,7 @@ variables:
|
||||
jobs:
|
||||
get_code:
|
||||
docker:
|
||||
- image: circleci/python:3.5
|
||||
- image: circleci/python:3.6
|
||||
<<: *set_workdir
|
||||
steps:
|
||||
# Replace standard code checkout with shallow clone to speed things up.
|
||||
@@ -83,34 +83,34 @@ jobs:
|
||||
key: v1-repo-{{ .Environment.CIRCLE_SHA1 }}
|
||||
paths:
|
||||
- ~/repo
|
||||
py35_docstring:
|
||||
py36_docstring:
|
||||
docker:
|
||||
- image: circleci/python:3.5
|
||||
- image: circleci/python:3.6
|
||||
<<: *set_workdir
|
||||
steps:
|
||||
- *restore_repo_cache
|
||||
- *install_tox
|
||||
- run: tox -e py35-lint_docstring_include_list
|
||||
py35_lint:
|
||||
- run: tox -e py36-lint_docstring_include_list
|
||||
py36_lint:
|
||||
docker:
|
||||
- image: circleci/python:3.5
|
||||
- image: circleci/python:3.6
|
||||
<<: *set_workdir
|
||||
steps:
|
||||
- *restore_repo_cache
|
||||
- *install_tox
|
||||
- run: tox -e py35-lint
|
||||
py35_unit:
|
||||
- run: tox -e py36-lint
|
||||
py36_unit:
|
||||
docker:
|
||||
- image: circleci/python:3.5
|
||||
- image: circleci/python:3.6
|
||||
<<: *set_workdir
|
||||
steps:
|
||||
- *restore_repo_cache
|
||||
- *install_tox
|
||||
- *install_ffprobe
|
||||
- run: tox -e py35-unit
|
||||
py35_first_startup:
|
||||
- run: tox -e py36-unit
|
||||
py36_first_startup:
|
||||
docker:
|
||||
- image: circleci/python:3.5
|
||||
- image: circleci/python:3.6
|
||||
<<: *set_workdir
|
||||
steps:
|
||||
- *restore_repo_cache
|
||||
@@ -118,10 +118,10 @@ jobs:
|
||||
- run: wget -q https://github.com/jmchilton/galaxy-downloads/raw/master/db_gx_rev_0141.sqlite
|
||||
- run: mv db_gx_rev_0141.sqlite database/universe.sqlite
|
||||
- *install_tox
|
||||
- run: tox -e py35-first_startup
|
||||
- run: tox -e py36-first_startup
|
||||
validate_test_tools:
|
||||
docker:
|
||||
- image: circleci/python:3.5
|
||||
- image: circleci/python:3.6
|
||||
<<: *set_workdir
|
||||
steps:
|
||||
- *restore_repo_cache
|
||||
@@ -131,7 +131,7 @@ jobs:
|
||||
- run: tox -e validate_test_tools
|
||||
test_galaxy_packages:
|
||||
docker:
|
||||
- image: circleci/python:3.5
|
||||
- image: circleci/python:3.6
|
||||
<<: *set_workdir
|
||||
steps:
|
||||
- *restore_repo_cache
|
||||
@@ -167,13 +167,13 @@ workflows:
|
||||
get_code_and_test:
|
||||
jobs:
|
||||
- get_code
|
||||
- py35_docstring:
|
||||
- py36_docstring:
|
||||
<<: *requires_get_code
|
||||
- py35_lint:
|
||||
- py36_lint:
|
||||
<<: *requires_get_code
|
||||
- py35_unit:
|
||||
- py36_unit:
|
||||
<<: *requires_get_code
|
||||
- py35_first_startup:
|
||||
- py36_first_startup:
|
||||
<<: *requires_get_code
|
||||
- test_galaxy_packages:
|
||||
<<: *requires_get_code
|
||||
|
||||
+1
-1
@@ -24,7 +24,7 @@ Community support is available at `Galaxy Help <https://help.galaxyproject.org/>
|
||||
Galaxy Quickstart
|
||||
=================
|
||||
|
||||
Galaxy requires Python 3.5 or 3.6 . To check your Python version, run:
|
||||
Galaxy requires Python 3.6 . To check your Python version, run:
|
||||
|
||||
.. code:: console
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
"""
|
||||
usage: %prog width height bg_color hatch_color [color alpha stop_pos] +
|
||||
"""
|
||||
from __future__ import division
|
||||
|
||||
import sys
|
||||
from math import pi
|
||||
|
||||
@@ -8,7 +8,7 @@ from galaxy.util import Params
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TourGenerator(object):
|
||||
class TourGenerator:
|
||||
def __init__(self, trans, tool_id, tool_version):
|
||||
self._trans = trans
|
||||
self._tool = self._trans.app.toolbox.get_tool(tool_id, tool_version)
|
||||
@@ -177,14 +177,14 @@ class TourGenerator(object):
|
||||
if name in test_inputs:
|
||||
hid = self._hids[name]
|
||||
dataset = self._test.inputs[name][0]
|
||||
step['content'] = 'Select dataset: <b>%s: %s</b>' % (
|
||||
step['content'] = 'Select dataset: <b>{}: {}</b>'.format(
|
||||
hid, dataset
|
||||
)
|
||||
else:
|
||||
step['content'] = 'Select a dataset'
|
||||
|
||||
elif input.type == 'conditional':
|
||||
param_id = '%s|%s' % (input.name, input.test_param.name)
|
||||
param_id = f'{input.name}|{input.test_param.name}'
|
||||
step['title'] = input.test_param.label
|
||||
step['element'] = '[tour_id="%s"]' % param_id
|
||||
params = []
|
||||
@@ -203,7 +203,7 @@ class TourGenerator(object):
|
||||
cases[key] = value.label
|
||||
|
||||
for case_id, case_title in cases.items():
|
||||
tour_id = '%s|%s' % (input.name, case_id)
|
||||
tour_id = f'{input.name}|{case_id}'
|
||||
if tour_id in self._test.inputs.keys():
|
||||
if case_id in self._data_inputs.keys():
|
||||
hid = self._hids[case_id]
|
||||
|
||||
@@ -23,14 +23,12 @@ THE ORIGINAL WORK IS WITH YOU.
|
||||
|
||||
Script for merging specific local Galaxy config galaxy.ini.cri with default Galaxy galaxy.ini.sample
|
||||
'''
|
||||
from __future__ import print_function
|
||||
|
||||
import configparser
|
||||
import logging
|
||||
import optparse
|
||||
import sys
|
||||
|
||||
from six.moves import configparser
|
||||
|
||||
|
||||
def main():
|
||||
# logging configuration
|
||||
@@ -51,12 +49,12 @@ def main():
|
||||
|
||||
config_sample = configparser.RawConfigParser()
|
||||
config_sample.read(options.sample)
|
||||
config_sample_content = open(options.sample, 'r').read()
|
||||
config_sample_content = open(options.sample).read()
|
||||
|
||||
config = configparser.RawConfigParser()
|
||||
config.read(options.config)
|
||||
|
||||
logging.info("Merging your own config file %s into the sample one %s." % (options.config, options.sample))
|
||||
logging.info(f"Merging your own config file {options.config} into the sample one {options.sample}.")
|
||||
logging.info("---------- DIFFERENCE ANALYSIS BEGIN ----------")
|
||||
for section in config.sections():
|
||||
if not config_sample.has_section(section):
|
||||
@@ -65,13 +63,13 @@ def main():
|
||||
for (name, value) in config.items(section):
|
||||
if not config_sample.has_option(section, name):
|
||||
if not "#%s" % name in config_sample_content:
|
||||
logging.warning("-MISSING- section [%s] option '%s' not found in sample file. It will be ignored." % (section, name))
|
||||
logging.warning(f"-MISSING- section [{section}] option '{name}' not found in sample file. It will be ignored.")
|
||||
else:
|
||||
logging.info("-notset- section [%s] option '%s' not set in sample file. It will be added." % (section, name))
|
||||
logging.info(f"-notset- section [{section}] option '{name}' not set in sample file. It will be added.")
|
||||
config_sample.set(section, name, value)
|
||||
else:
|
||||
if not config_sample.get(section, name) == value:
|
||||
logging.info("- diff - section [%s] option '%s' has different value ('%s':'%s'). It will be modified." % (section, name, config_sample.get(section, name), value))
|
||||
logging.info("- diff - section [{}] option '{}' has different value ('{}':'{}'). It will be modified.".format(section, name, config_sample.get(section, name), value))
|
||||
config_sample.set(section, name, value)
|
||||
logging.info("---------- DIFFERENCE ANALYSIS END ----------")
|
||||
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
check_galaxy can be run by hand, although it is meant to run from cron
|
||||
via the check_galaxy.sh script in Galaxy's cron/ directory.
|
||||
"""
|
||||
from __future__ import print_function
|
||||
|
||||
import formatter
|
||||
import getopt
|
||||
@@ -14,13 +13,12 @@ import socket
|
||||
import sys
|
||||
import time
|
||||
import warnings
|
||||
from user import home
|
||||
|
||||
from six.moves.urllib.request import (
|
||||
from urllib.request import (
|
||||
build_opener,
|
||||
HTTPCookieProcessor,
|
||||
Request
|
||||
Request,
|
||||
)
|
||||
from user import home
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore')
|
||||
@@ -52,7 +50,7 @@ handler = args[3]
|
||||
warntime = 240
|
||||
|
||||
new_history = False
|
||||
for o, a in opts:
|
||||
for o, _ in opts:
|
||||
if o == "-n":
|
||||
if debug:
|
||||
print("Specified -n, will create a new history")
|
||||
@@ -73,7 +71,7 @@ tc.agent("Mozilla/5.0 (compatible; check_galaxy/0.2)")
|
||||
tc.config('use_tidy', 0)
|
||||
|
||||
|
||||
class Browser(object):
|
||||
class Browser:
|
||||
def __init__(self):
|
||||
self.server = server
|
||||
self.handler = handler
|
||||
@@ -93,7 +91,7 @@ class Browser(object):
|
||||
self.opener = build_opener(HTTPCookieProcessor(tc.get_browser().cj))
|
||||
|
||||
def get(self, path):
|
||||
tc.go("%s%s" % (self.server, path))
|
||||
tc.go(f"{self.server}{path}")
|
||||
tc.code(200)
|
||||
|
||||
def req(self, path, data=None, method=None):
|
||||
@@ -105,7 +103,7 @@ class Browser(object):
|
||||
if method:
|
||||
req.get_method = lambda: method
|
||||
res = self.opener.open(req)
|
||||
print('==> at %s (%s)' % (url, method or 'GET'))
|
||||
print('==> at {} ({})'.format(url, method or 'GET'))
|
||||
assert res.getcode() == 200, url
|
||||
return res
|
||||
|
||||
@@ -226,7 +224,7 @@ class Browser(object):
|
||||
if self.hda_state != "ok":
|
||||
self.get("/datasets/%s/stderr" % self.hda_id)
|
||||
print(tc.browser.get_html())
|
||||
raise Exception("HDA %s NOT OK: %s" % (self.hda_id, self.hda_state))
|
||||
raise Exception(f"HDA {self.hda_id} NOT OK: {self.hda_state}")
|
||||
|
||||
def check_hda_content(self):
|
||||
self.get("/datasets/%s/display?to_ext=txt" % self.hda_id)
|
||||
@@ -240,7 +238,7 @@ class Browser(object):
|
||||
|
||||
def delete_datasets(self):
|
||||
for hda in self.undeleted_hdas:
|
||||
path = '/api/histories/%s/contents/%s' % (self.history_id, hda['id'])
|
||||
path = '/api/histories/{}/contents/{}'.format(self.history_id, hda['id'])
|
||||
self.req(path, method='DELETE')
|
||||
hdas = [hda['id'] for hda in self.undeleted_hdas]
|
||||
if hdas:
|
||||
|
||||
@@ -6,7 +6,6 @@ Adds Manually created builds and chrom info to Galaxy's info tables
|
||||
Usage:
|
||||
python add_manual_builds.py input_file builds.txt chrom_length_dir
|
||||
"""
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
@@ -11,14 +11,13 @@ All chromInfo is placed in a path with the convention
|
||||
Usage:
|
||||
python build_chrom_db.py dbpath/ [builds_file]
|
||||
"""
|
||||
from __future__ import print_function
|
||||
|
||||
import fileinput
|
||||
import os
|
||||
import sys
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import requests
|
||||
from six.moves.urllib.parse import urlencode
|
||||
|
||||
import parse_builds # noqa: I100,I202
|
||||
|
||||
@@ -78,5 +77,5 @@ if __name__ == "__main__":
|
||||
for chrominfo in getchrominfo("http://genome-test.gi.ucsc.edu/cgi-bin/hgTables?", build):
|
||||
print("\t".join(chrominfo), file=outfile)
|
||||
except Exception as e:
|
||||
print("Failed to retrieve %s: %s" % (build, e))
|
||||
print(f"Failed to retrieve {build}: {e}")
|
||||
os.remove(outfile_name)
|
||||
|
||||
@@ -4,7 +4,6 @@ Connects to the URL specified and outputs builds available at that
|
||||
DSN in tabular format. UCSC Main gateway is used as default.
|
||||
build description
|
||||
"""
|
||||
from __future__ import print_function
|
||||
|
||||
import sys
|
||||
import xml.etree.ElementTree as ElementTree
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
"""
|
||||
Connects to sites and determines which builds are available at each.
|
||||
"""
|
||||
from __future__ import print_function
|
||||
|
||||
import xml.etree.ElementTree as ElementTree
|
||||
|
||||
|
||||
+7
-9
@@ -1,17 +1,15 @@
|
||||
# coding: utf-8
|
||||
# TODO: Add examples, tables and best practice links to command
|
||||
# TODO: Examples of truevalue, falsevalue
|
||||
# TODO: Test param extra_file
|
||||
# Things dropped from schema_template.md (still documented inside schema).
|
||||
# - request_parameter_translation
|
||||
from __future__ import print_function
|
||||
|
||||
import sys
|
||||
from io import StringIO
|
||||
|
||||
from lxml import etree
|
||||
from six import StringIO
|
||||
|
||||
with open(sys.argv[2], "r") as f:
|
||||
with open(sys.argv[2]) as f:
|
||||
xmlschema_doc = etree.parse(f)
|
||||
|
||||
markdown_buffer = StringIO()
|
||||
@@ -22,7 +20,7 @@ def main():
|
||||
toc_list = []
|
||||
content_list = []
|
||||
found_tag = False
|
||||
with open(sys.argv[1], "r") as markdown_template:
|
||||
with open(sys.argv[1]) as markdown_template:
|
||||
for line in markdown_template:
|
||||
if line.startswith("$tag:"):
|
||||
found_tag = True
|
||||
@@ -41,7 +39,7 @@ def main():
|
||||
print(el, end='')
|
||||
|
||||
|
||||
class Tag(object):
|
||||
class Tag:
|
||||
|
||||
def __init__(self, line):
|
||||
assert line.startswith("$tag:")
|
||||
@@ -71,7 +69,7 @@ class Tag(object):
|
||||
return " > ".join("``%s``" % p for p in self.title.split("|"))
|
||||
|
||||
def build_toc_entry(self):
|
||||
return "* [%s](%s)" % (self._pretty_title, self._anchor)
|
||||
return f"* [{self._pretty_title}]({self._anchor})"
|
||||
|
||||
def build_help(self):
|
||||
tag = xmlschema_doc.find(self.xpath)
|
||||
@@ -113,7 +111,7 @@ def _build_tag(tag, hide_attributes):
|
||||
doc = _doc_or_none(_type_el(element))
|
||||
assert doc is not None, "Documentation for %s is empty" % element.attrib["name"]
|
||||
doc = doc.strip()
|
||||
assertions_buffer.write("``%s`` | %s\n" % (element.attrib["name"], doc))
|
||||
assertions_buffer.write("``{}`` | {}\n".format(element.attrib["name"], doc))
|
||||
text = text.replace(line, assertions_buffer.getvalue())
|
||||
tag_help.write(text)
|
||||
best_practices = _get_bp_link(annotation_el)
|
||||
@@ -165,7 +163,7 @@ def _build_attributes_table(tag, attributes, hide_attributes=False, attribute_na
|
||||
if best_practices:
|
||||
details += """ Find the Intergalactic Utilities Commision suggested best practices for this element [here](%s).""" % best_practices
|
||||
|
||||
attribute_table.write("``%s`` | %s | %s\n" % (name, details, use))
|
||||
attribute_table.write(f"``{name}`` | {details} | {use}\n")
|
||||
return attribute_table.getvalue()
|
||||
|
||||
|
||||
|
||||
+9
-10
@@ -1,4 +1,3 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Galaxy documentation build configuration file, created by
|
||||
# sphinx-quickstart on Tue Mar 6 10:44:44 2012.
|
||||
@@ -80,8 +79,8 @@ source_suffix = ['.rst', '.md']
|
||||
master_doc = 'index'
|
||||
|
||||
# General information about the project.
|
||||
project = u'Galaxy Project'
|
||||
copyright = str(datetime.datetime.now().year) + u', Galaxy Committers'
|
||||
project = 'Galaxy Project'
|
||||
copyright = str(datetime.datetime.now().year) + ', Galaxy Committers'
|
||||
|
||||
# The version info for the project you're documenting, acts as replacement for
|
||||
# |version| and |release|, also used in various other places throughout the
|
||||
@@ -233,8 +232,8 @@ latex_elements = {
|
||||
# Grouping the document tree into LaTeX files. List of tuples
|
||||
# (source start file, target name, title, author, documentclass [howto/manual]).
|
||||
latex_documents = [
|
||||
('index', 'Galaxy.tex', u'Galaxy Code Documentation',
|
||||
u'Galaxy Team', 'manual'),
|
||||
('index', 'Galaxy.tex', 'Galaxy Code Documentation',
|
||||
'Galaxy Team', 'manual'),
|
||||
]
|
||||
|
||||
# The name of an image file (relative to this directory) to place at the top of
|
||||
@@ -263,8 +262,8 @@ latex_documents = [
|
||||
# One entry per manual page. List of tuples
|
||||
# (source start file, name, description, authors, manual section).
|
||||
man_pages = [
|
||||
('index', 'galaxy', u'Galaxy Documentation',
|
||||
[u'Galaxy Team'], 1)
|
||||
('index', 'galaxy', 'Galaxy Documentation',
|
||||
['Galaxy Team'], 1)
|
||||
]
|
||||
|
||||
# If true, show URL addresses after external links.
|
||||
@@ -277,8 +276,8 @@ man_pages = [
|
||||
# (source start file, target name, title, author,
|
||||
# dir menu entry, description, category)
|
||||
texinfo_documents = [
|
||||
('index', 'Galaxy', u'Galaxy Documentation',
|
||||
u'Galaxy Team', 'Galaxy', 'Data intensive biology for everyone.',
|
||||
('index', 'Galaxy', 'Galaxy Documentation',
|
||||
'Galaxy Team', 'Galaxy', 'Data intensive biology for everyone.',
|
||||
'Miscellaneous'),
|
||||
]
|
||||
|
||||
@@ -293,7 +292,7 @@ texinfo_documents = [
|
||||
|
||||
|
||||
# -- ReadTheDocs.org Settings ------------------------------------------------
|
||||
class Mock(object):
|
||||
class Mock:
|
||||
def __init__(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Galaxy sphinxcontrib-simpleversioning documentation build configuration file
|
||||
#
|
||||
|
||||
@@ -76,7 +76,7 @@ class AdminActions:
|
||||
quota.description = params.description
|
||||
self.sa_session.add(quota)
|
||||
self.sa_session.flush()
|
||||
message = "Quota '{}' has been renamed to '{}'.".format(old_name, params.name)
|
||||
message = f"Quota '{old_name}' has been renamed to '{params.name}'."
|
||||
return message
|
||||
|
||||
def _manage_users_and_groups_for_quota(self, quota, params, decode_id=None):
|
||||
@@ -122,7 +122,7 @@ class AdminActions:
|
||||
else:
|
||||
if params.default != 'no':
|
||||
self.app.quota_agent.set_default_quota(params.default, quota)
|
||||
message = "Quota '{}' is now the default for {} users.".format(quota.name, params.default)
|
||||
message = f"Quota '{quota.name}' is now the default for {params.default} users."
|
||||
else:
|
||||
if quota.default:
|
||||
message = "Quota '{}' is no longer the default for {} users.".format(quota.name, quota.default[0].type)
|
||||
|
||||
@@ -109,23 +109,23 @@ class PAM(AuthProvider):
|
||||
|
||||
pam_service = options.get('pam-service', 'galaxy')
|
||||
use_helper = string_as_bool(options.get('use-external-helper', False))
|
||||
log.debug("PAM auth: will use external helper: {}".format(use_helper))
|
||||
log.debug(f"PAM auth: will use external helper: {use_helper}")
|
||||
authenticated = False
|
||||
if use_helper:
|
||||
authentication_helper = options.get('authentication-helper-script', '/bin/false').strip()
|
||||
log.debug("PAM auth: external helper script: {}".format(authentication_helper))
|
||||
log.debug(f"PAM auth: external helper script: {authentication_helper}")
|
||||
if not authentication_helper.startswith('/'):
|
||||
# don't accept relative path
|
||||
authenticated = False
|
||||
else:
|
||||
auth_cmd = shlex.split('/usr/bin/sudo -n {}'.format(authentication_helper))
|
||||
log.debug("PAM auth: external helper cmd: {}".format(auth_cmd))
|
||||
message = '{}\n{}\n{}\n'.format(pam_service, pam_username, password)
|
||||
auth_cmd = shlex.split(f'/usr/bin/sudo -n {authentication_helper}')
|
||||
log.debug(f"PAM auth: external helper cmd: {auth_cmd}")
|
||||
message = f'{pam_service}\n{pam_username}\n{password}\n'
|
||||
try:
|
||||
output = commands.execute(auth_cmd, input=message)
|
||||
except commands.CommandLineException as e:
|
||||
if e.stderr != '':
|
||||
log.debug("PAM auth: external authentication script had errors: status {} error {}".format(e.returncode, e.stderr))
|
||||
log.debug(f"PAM auth: external authentication script had errors: status {e.returncode} error {e.stderr}")
|
||||
output = e.stdout
|
||||
if output.strip() == 'True':
|
||||
authenticated = True
|
||||
|
||||
@@ -86,7 +86,7 @@ def parse_auth_results(trans, auth_results, options):
|
||||
i += 1
|
||||
else:
|
||||
raise Conflict("Cannot make unique username")
|
||||
log.debug("Email: {}, auto-register with username: {}".format(auto_email, auto_username))
|
||||
log.debug(f"Email: {auto_email}, auto-register with username: {auto_username}")
|
||||
auth_return["auto_reg"] = string_as_bool(options.get('auto-register', False))
|
||||
auth_return["email"] = auto_email
|
||||
auth_return["username"] = auto_username
|
||||
|
||||
@@ -3,12 +3,12 @@ import hashlib
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime, timedelta
|
||||
from urllib.parse import quote
|
||||
|
||||
import jwt
|
||||
import requests
|
||||
from oauthlib.common import generate_nonce
|
||||
from requests_oauthlib import OAuth2Session
|
||||
from six.moves.urllib.parse import quote
|
||||
|
||||
from galaxy import exceptions
|
||||
from galaxy import util
|
||||
@@ -248,16 +248,16 @@ class CustosAuthnz(IdentityProvider):
|
||||
base_url = self.config["url"]
|
||||
# Remove potential trailing slash to avoid "//realms"
|
||||
base_url = base_url if base_url[-1] != "/" else base_url[:-1]
|
||||
return "{}/.well-known/openid-configuration".format(base_url)
|
||||
return f"{base_url}/.well-known/openid-configuration"
|
||||
else:
|
||||
raise Exception("Unknown Custos provider name: {}".format(provider))
|
||||
raise Exception(f"Unknown Custos provider name: {provider}")
|
||||
|
||||
def _fetch_well_known_oidc_config(self, well_known_uri):
|
||||
try:
|
||||
return requests.get(well_known_uri,
|
||||
verify=self._get_verify_param()).json()
|
||||
except Exception:
|
||||
log.error("Failed to load well-known OIDC config URI: {}".format(well_known_uri))
|
||||
log.error(f"Failed to load well-known OIDC config URI: {well_known_uri}")
|
||||
raise
|
||||
|
||||
def _load_well_known_oidc_config(self, well_known_oidc_config):
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import builtins
|
||||
import copy
|
||||
import json
|
||||
import logging
|
||||
@@ -10,7 +11,6 @@ from cloudauthz import CloudAuthz
|
||||
from cloudauthz.exceptions import (
|
||||
CloudAuthzBaseException
|
||||
)
|
||||
from six.moves import builtins
|
||||
|
||||
from galaxy import exceptions
|
||||
from galaxy import model
|
||||
@@ -89,7 +89,7 @@ class AuthnzManager:
|
||||
except ImportError:
|
||||
raise
|
||||
except etree.ParseError as e:
|
||||
raise etree.ParseError("Invalid configuration at `{}`: {} -- unable to continue.".format(config_file, e))
|
||||
raise etree.ParseError(f"Invalid configuration at `{config_file}`: {e} -- unable to continue.")
|
||||
|
||||
def _get_idp_icon(self, idp):
|
||||
return self.oidc_backends_config[idp].get('icon') or DEFAULT_OIDC_IDP_ICONS.get(idp)
|
||||
@@ -109,7 +109,7 @@ class AuthnzManager:
|
||||
"skipping the node.".format(child.tag))
|
||||
continue
|
||||
if 'name' not in child.attrib:
|
||||
log.error("Could not find a node attribute 'name'; skipping the node '{}'.".format(child.tag))
|
||||
log.error(f"Could not find a node attribute 'name'; skipping the node '{child.tag}'.")
|
||||
continue
|
||||
idp = child.get('name').lower()
|
||||
if idp in BACKENDS_NAME:
|
||||
@@ -127,7 +127,7 @@ class AuthnzManager:
|
||||
except ImportError:
|
||||
raise
|
||||
except etree.ParseError as e:
|
||||
raise etree.ParseError("Invalid configuration at `{}`: {} -- unable to continue.".format(config_file, e))
|
||||
raise etree.ParseError(f"Invalid configuration at `{config_file}`: {e} -- unable to continue.")
|
||||
|
||||
def _parse_idp_config(self, config_xml):
|
||||
rtv = {
|
||||
@@ -188,10 +188,10 @@ class AuthnzManager:
|
||||
else:
|
||||
return True, "", identity_provider_class(unified_provider_name, self.oidc_config, self.oidc_backends_config[unified_provider_name])
|
||||
except Exception as e:
|
||||
log.exception('An error occurred when loading {}'.format(identity_provider_class.__name__))
|
||||
log.exception(f'An error occurred when loading {identity_provider_class.__name__}')
|
||||
return False, unicodify(e), None
|
||||
else:
|
||||
msg = 'The requested identity provider, `{}`, is not a recognized/expected provider.'.format(provider)
|
||||
msg = f'The requested identity provider, `{provider}`, is not a recognized/expected provider.'
|
||||
log.debug(msg)
|
||||
return False, msg, None
|
||||
|
||||
@@ -281,13 +281,13 @@ class AuthnzManager:
|
||||
return False, message, None
|
||||
elif provider in KEYCLOAK_BACKENDS:
|
||||
if (self.allowed_idps and (idphint not in self.allowed_idps)):
|
||||
msg = 'An error occurred when authenticating a user. Invalid EntityID: `{}`'.format(idphint)
|
||||
msg = f'An error occurred when authenticating a user. Invalid EntityID: `{idphint}`'
|
||||
log.exception(msg)
|
||||
return False, msg, None
|
||||
return True, "Redirecting to the `{}` identity provider for authentication".format(provider), backend.authenticate(trans, idphint)
|
||||
return True, "Redirecting to the `{}` identity provider for authentication".format(provider), backend.authenticate(trans)
|
||||
return True, f"Redirecting to the `{provider}` identity provider for authentication", backend.authenticate(trans, idphint)
|
||||
return True, f"Redirecting to the `{provider}` identity provider for authentication", backend.authenticate(trans)
|
||||
except Exception:
|
||||
msg = 'An error occurred when authenticating a user on `{}` identity provider'.format(provider)
|
||||
msg = f'An error occurred when authenticating a user on `{provider}` identity provider'
|
||||
log.exception(msg)
|
||||
return False, msg, None
|
||||
|
||||
@@ -322,7 +322,7 @@ class AuthnzManager:
|
||||
# check if logout is enabled for this idp and return false if not
|
||||
unified_provider_name = self._unify_provider_name(provider)
|
||||
if self.oidc_backends_config[unified_provider_name]['enable_idp_logout'] is False:
|
||||
return False, "IDP logout is not enabled for {}".format(provider), None
|
||||
return False, f"IDP logout is not enabled for {provider}", None
|
||||
|
||||
success, message, backend = self._get_authnz_backend(provider)
|
||||
if success is False:
|
||||
|
||||
@@ -316,11 +316,11 @@ def contains_required_data(response=None, is_new=False, **kwargs):
|
||||
# sent back from the identity provider. PSA internally handles such
|
||||
# scenarios; however, this case is implemented to prevent uncaught
|
||||
# server-side errors.
|
||||
raise MalformedContents(err_msg="`response` not found. {}".format(hint_msg))
|
||||
raise MalformedContents(err_msg=f"`response` not found. {hint_msg}")
|
||||
if not response.get("id_token"):
|
||||
# This can happen if a non-OIDC compliant backend is used;
|
||||
# e.g., an OAuth2.0-based backend that only generates access token.
|
||||
raise MalformedContents(err_msg="Missing identity token. {}".format(hint_msg))
|
||||
raise MalformedContents(err_msg=f"Missing identity token. {hint_msg}")
|
||||
if is_new and not response.get("refresh_token"):
|
||||
# An identity provider (e.g., Google) sends a refresh token the first
|
||||
# time user consents Galaxy's access (i.e., the first time user logs in
|
||||
@@ -331,7 +331,7 @@ def contains_required_data(response=None, is_new=False, **kwargs):
|
||||
# user has provided consent. This can also happen under dev efforts.
|
||||
# The solution is to revoke the consent by visiting the identity provider's
|
||||
# website, and then retry the login process.
|
||||
raise MalformedContents(err_msg="Missing refresh token. {}".format(hint_msg))
|
||||
raise MalformedContents(err_msg=f"Missing refresh token. {hint_msg}")
|
||||
|
||||
|
||||
def verify(strategy=None, response=None, details=None, **kwargs):
|
||||
@@ -344,7 +344,7 @@ def verify(strategy=None, response=None, details=None, **kwargs):
|
||||
|
||||
if provider.lower() == "gcp":
|
||||
result = requests.post(
|
||||
"https://iam.googleapis.com/v1/projects/-/serviceAccounts/{}:getIamPolicy".format(endpoint),
|
||||
f"https://iam.googleapis.com/v1/projects/-/serviceAccounts/{endpoint}:getIamPolicy",
|
||||
headers={
|
||||
'Authorization': 'Bearer {}'.format(response.get("access_token")),
|
||||
'Accept': 'application/json'})
|
||||
@@ -364,7 +364,7 @@ def verify(strategy=None, response=None, details=None, **kwargs):
|
||||
# sensitive information that should not be exposed to users.
|
||||
raise Exception(res["error"]["message"])
|
||||
else:
|
||||
raise Exception("`{}` is an unsupported secondary authorization provider, contact admin.".format(provider))
|
||||
raise Exception(f"`{provider}` is an unsupported secondary authorization provider, contact admin.")
|
||||
|
||||
|
||||
def allowed_to_disconnect(name=None, user=None, user_storage=None, strategy=None,
|
||||
|
||||
@@ -4,6 +4,7 @@ Universe configuration builder.
|
||||
# absolute_import needed for tool_shed package.
|
||||
|
||||
import collections
|
||||
import configparser
|
||||
import errno
|
||||
import ipaddress
|
||||
import logging
|
||||
@@ -22,7 +23,6 @@ from datetime import timedelta
|
||||
import yaml
|
||||
from beaker.cache import CacheManager
|
||||
from beaker.util import parse_cache_config_options
|
||||
from six.moves import configparser
|
||||
|
||||
from galaxy.config.schema import AppSchema
|
||||
from galaxy.containers import parse_containers_config
|
||||
|
||||
@@ -8,12 +8,12 @@ from collections import (
|
||||
namedtuple,
|
||||
OrderedDict
|
||||
)
|
||||
from io import StringIO
|
||||
from textwrap import TextWrapper
|
||||
|
||||
import requests
|
||||
import yaml
|
||||
from boltons.iterutils import remap
|
||||
from six import StringIO
|
||||
|
||||
try:
|
||||
from pykwalify.core import Core
|
||||
@@ -405,7 +405,7 @@ def _to_rst(args, app_desc, heading_level="~"):
|
||||
def _write_option_rst(args, rst, key, heading_level, option_value):
|
||||
title = "``%s``" % key
|
||||
heading = heading_level * len(title)
|
||||
rst.write("{}\n{}\n{}\n\n".format(heading, title, heading))
|
||||
rst.write(f"{heading}\n{title}\n{heading}\n\n")
|
||||
option, value = _parse_option_value(option_value)
|
||||
desc = option["desc"]
|
||||
rst.write(":Description:\n")
|
||||
@@ -649,7 +649,7 @@ def _is_ini(path):
|
||||
def _replace_file(args, f, app_desc, from_path, to_path):
|
||||
_write_to_file(args, f, to_path)
|
||||
backup_path = "%s.backup" % from_path
|
||||
print("Moving [{}] to [{}]".format(from_path, backup_path))
|
||||
print(f"Moving [{from_path}] to [{backup_path}]")
|
||||
if args.dry_run:
|
||||
print("... skipping because --dry-run is enabled.")
|
||||
else:
|
||||
@@ -661,7 +661,7 @@ def _build_sample_yaml(args, app_desc):
|
||||
UWSGI_OPTIONS.update(SHED_ONLY_UWSGI_OPTIONS)
|
||||
schema = app_desc.schema
|
||||
f = StringIO()
|
||||
for key, value in UWSGI_OPTIONS.items():
|
||||
for value in UWSGI_OPTIONS.values():
|
||||
for field in ["desc", "default"]:
|
||||
if field not in value:
|
||||
continue
|
||||
@@ -693,7 +693,7 @@ def _write_to_file(args, f, path):
|
||||
contents = f
|
||||
if args.dry_run:
|
||||
contents_indented = "\n".join(" |%s" % l for l in contents.splitlines())
|
||||
print("Overwriting {} with the following contents:\n{}".format(path, contents_indented))
|
||||
print(f"Overwriting {path} with the following contents:\n{contents_indented}")
|
||||
print("... skipping because --dry-run is enabled.")
|
||||
else:
|
||||
print("Overwriting %s" % path)
|
||||
@@ -743,10 +743,10 @@ def _write_option(args, f, key, option_value, as_comment=False, uwsgi_hack=False
|
||||
if uwsgi_hack:
|
||||
if option.get("type", "str") == "bool":
|
||||
value = str(value).lower()
|
||||
key_val_str = "{}: {}".format(key, value)
|
||||
key_val_str = f"{key}: {value}"
|
||||
else:
|
||||
key_val_str = yaml.dump({key: value}, width=float("inf")).lstrip("{").rstrip("\n}")
|
||||
lines = "{}{}{}".format(comment, as_comment_str, key_val_str)
|
||||
lines = f"{comment}{as_comment_str}{key_val_str}"
|
||||
lines_idented = "\n".join(" %s" % l for l in lines.split("\n"))
|
||||
f.write("%s\n\n" % lines_idented)
|
||||
|
||||
@@ -774,7 +774,7 @@ def _server_paste_to_uwsgi(app_desc, server_config, applied_filters):
|
||||
if server_config.get("use", "egg:Paste#http") != "egg:Paste#http":
|
||||
raise Exception("Unhandled paste server 'use' value [%s], file must be manually migrate.")
|
||||
|
||||
uwsgi_dict["http"] = "{}:{}".format(host, port)
|
||||
uwsgi_dict["http"] = f"{host}:{port}"
|
||||
# default changing from 10 to 8
|
||||
uwsgi_dict["threads"] = int(server_config.get("threadpool_workers", 8))
|
||||
# required for static...
|
||||
@@ -791,7 +791,7 @@ def _server_paste_to_uwsgi(app_desc, server_config, applied_filters):
|
||||
uwsgi_dict["http-auto-gzip"] = True
|
||||
|
||||
if prefix:
|
||||
uwsgi_dict["mount"] = "{}={}".format(prefix, app_desc.uwsgi_module)
|
||||
uwsgi_dict["mount"] = f"{prefix}={app_desc.uwsgi_module}"
|
||||
uwsgi_dict["manage-script-name"] = True
|
||||
else:
|
||||
uwsgi_dict["module"] = app_desc.uwsgi_module
|
||||
|
||||
@@ -84,7 +84,7 @@ class AppSchema(Schema):
|
||||
|
||||
def check_type_is_str_or_any(option, key):
|
||||
if option.get('type') not in ('str', 'any'):
|
||||
message = "Invalid schema: property '{}' should have type 'str'".format(key)
|
||||
message = f"Invalid schema: property '{key}' should have type 'str'"
|
||||
raise_error(message)
|
||||
|
||||
def check_is_dag():
|
||||
|
||||
@@ -17,7 +17,6 @@ from abc import (
|
||||
from collections import namedtuple
|
||||
|
||||
import yaml
|
||||
from six.moves import shlex_quote
|
||||
|
||||
from galaxy.exceptions import ContainerCLIError
|
||||
from galaxy.util.submodules import import_submodules
|
||||
@@ -234,14 +233,14 @@ class ContainerInterface(metaclass=ABCMeta):
|
||||
def _stringify_kwopt_string(self, flag, val):
|
||||
"""
|
||||
"""
|
||||
return '{flag} {value}'.format(flag=flag, value=shlex_quote(str(val)))
|
||||
return '{flag} {value}'.format(flag=flag, value=shlex.quote(str(val)))
|
||||
|
||||
def _stringify_kwopt_list(self, flag, val):
|
||||
"""
|
||||
"""
|
||||
if isinstance(val, str):
|
||||
return self._stringify_kwopt_string(flag, val)
|
||||
return ' '.join('{flag} {value}'.format(flag=flag, value=shlex_quote(str(v))) for v in val)
|
||||
return ' '.join('{flag} {value}'.format(flag=flag, value=shlex.quote(str(v))) for v in val)
|
||||
|
||||
def _stringify_kwopt_list_of_kvpairs(self, flag, val):
|
||||
"""
|
||||
@@ -253,7 +252,7 @@ class ContainerInterface(metaclass=ABCMeta):
|
||||
else:
|
||||
# {'foo': 'bar', 'baz': 'quux'}
|
||||
for k, v in dict(val).items():
|
||||
l.append('{k}={v}'.format(k=k, v=v))
|
||||
l.append(f'{k}={v}')
|
||||
return self._stringify_kwopt_list(flag, l)
|
||||
|
||||
def _stringify_kwopt_list_of_kovtrips(self, flag, val):
|
||||
@@ -263,7 +262,7 @@ class ContainerInterface(metaclass=ABCMeta):
|
||||
return self._stringify_kwopt_string(flag, val)
|
||||
l = []
|
||||
for k, o, v in val:
|
||||
l.append('{k}{o}{v}'.format(k=k, o=o, v=v))
|
||||
l.append(f'{k}{o}{v}')
|
||||
return self._stringify_kwopt_list(flag, l)
|
||||
|
||||
def _run_command(self, command, verbose=False):
|
||||
@@ -275,7 +274,7 @@ class ContainerInterface(metaclass=ABCMeta):
|
||||
if p.returncode == 0:
|
||||
return stdout.strip()
|
||||
else:
|
||||
msg = "Command '{}' returned non-zero exit status {}".format(command, p.returncode)
|
||||
msg = f"Command '{command}' returned non-zero exit status {p.returncode}"
|
||||
log.error(msg + ': ' + stderr.strip())
|
||||
raise ContainerCLIError(
|
||||
msg,
|
||||
@@ -324,7 +323,7 @@ class ContainerInterfaceConfig(dict):
|
||||
try:
|
||||
return self[name]
|
||||
except KeyError:
|
||||
raise AttributeError("'{}' object has no attribute '{}'".format(self.__class__.__name__, name))
|
||||
raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{name}'")
|
||||
|
||||
def get(self, name, default=None):
|
||||
try:
|
||||
|
||||
@@ -4,6 +4,7 @@ Interface to Docker
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shlex
|
||||
from functools import partial
|
||||
from itertools import cycle, repeat
|
||||
from time import sleep
|
||||
@@ -18,7 +19,6 @@ try:
|
||||
except ImportError:
|
||||
ConnectionError = None
|
||||
ReadTimeout = None
|
||||
from six.moves import shlex_quote
|
||||
|
||||
from galaxy.containers import ContainerInterface
|
||||
from galaxy.containers.docker_decorators import (
|
||||
@@ -130,7 +130,7 @@ class DockerCLIInterface(DockerInterface):
|
||||
global_kwopts = []
|
||||
if self._conf.host:
|
||||
global_kwopts.append('--host')
|
||||
global_kwopts.append(shlex_quote(self._conf.host))
|
||||
global_kwopts.append(shlex.quote(self._conf.host))
|
||||
if self._conf.force_tlsverify:
|
||||
global_kwopts.append('--tlsverify')
|
||||
self._docker_command = self._conf['command_template'].format(
|
||||
@@ -142,9 +142,9 @@ class DockerCLIInterface(DockerInterface):
|
||||
|
||||
def _filter_by_id_or_name(self, id, name):
|
||||
if id:
|
||||
return '--filter id={}'.format(id)
|
||||
return f'--filter id={id}'
|
||||
elif name:
|
||||
return '--filter name={}'.format(name)
|
||||
return f'--filter name={name}'
|
||||
return None
|
||||
|
||||
def _stringify_kwopt_docker_volumes(self, flag, val):
|
||||
@@ -159,7 +159,7 @@ class DockerCLIInterface(DockerInterface):
|
||||
for hostvol, guestopts in val.items():
|
||||
if isinstance(guestopts, str):
|
||||
# {'/host/vol': '/container/vol'}
|
||||
l.append('{}:{}'.format(hostvol, guestopts))
|
||||
l.append(f'{hostvol}:{guestopts}')
|
||||
else:
|
||||
# {'/host/vol': {'bind': '/container/vol'}}
|
||||
# {'/host/vol': {'bind': '/container/vol', 'mode': 'rw'}}
|
||||
@@ -198,7 +198,7 @@ class DockerCLIInterface(DockerInterface):
|
||||
return self._run_docker(subcommand='inspect', args=container_id)[0]
|
||||
except (IndexError, ContainerCLIError) as exc:
|
||||
msg = "Invalid container id: %s" % container_id
|
||||
if exc.stdout == '[]' and exc.stderr == 'Error: no such object: {container_id}'.format(container_id=container_id):
|
||||
if exc.stdout == '[]' and exc.stderr == f'Error: no such object: {container_id}':
|
||||
log.warning(msg)
|
||||
return []
|
||||
else:
|
||||
@@ -210,7 +210,7 @@ class DockerCLIInterface(DockerInterface):
|
||||
return self._run_docker(subcommand='image inspect', args=image)[0]
|
||||
except (IndexError, ContainerCLIError) as exc:
|
||||
msg = "%s not pulled, cannot get digest" % image
|
||||
if exc.stdout == '[]' and exc.stderr == 'Error: no such image: {image}'.format(image=image):
|
||||
if exc.stdout == '[]' and exc.stderr == f'Error: no such image: {image}':
|
||||
log.warning(msg, image)
|
||||
return []
|
||||
else:
|
||||
|
||||
@@ -3,13 +3,13 @@ Model objects for docker objects
|
||||
"""
|
||||
|
||||
import logging
|
||||
import shlex
|
||||
|
||||
try:
|
||||
import docker
|
||||
except ImportError:
|
||||
from galaxy.util.bunch import Bunch
|
||||
docker = Bunch(errors=Bunch(NotFound=None))
|
||||
from six.moves import shlex_quote
|
||||
|
||||
from galaxy.containers import (
|
||||
Container,
|
||||
@@ -106,7 +106,7 @@ class DockerVolume(ContainerVolume):
|
||||
def __str__(self):
|
||||
volume_str = ":".join(filter(lambda x: x is not None, (self.host_path, self.path, self.mode)))
|
||||
if "$" not in volume_str:
|
||||
volume_for_cmd_line = shlex_quote(volume_str)
|
||||
volume_for_cmd_line = shlex.quote(volume_str)
|
||||
else:
|
||||
# e.g. $_GALAXY_JOB_TMP_DIR:$_GALAXY_JOB_TMP_DIR:rw so don't single quote.
|
||||
volume_for_cmd_line = '"%s"' % volume_str
|
||||
@@ -393,10 +393,10 @@ class DockerServiceConstraint:
|
||||
return hash((self._name, self._op, self._value))
|
||||
|
||||
def __repr__(self):
|
||||
return '{}({}{}{})'.format(self.__class__.__name__, self._name, self._op, self._value)
|
||||
return f'{self.__class__.__name__}({self._name}{self._op}{self._value})'
|
||||
|
||||
def __str__(self):
|
||||
return '{}{}{}'.format(self._name, self._op, self._value)
|
||||
return f'{self._name}{self._op}{self._value}'
|
||||
|
||||
@staticmethod
|
||||
def split_constraint_string(constraint_str):
|
||||
@@ -508,7 +508,7 @@ class DockerNode:
|
||||
|
||||
@property
|
||||
def state(self):
|
||||
return ('{}-{}'.format(self._status, self._availability)).lower()
|
||||
return (f'{self._status}-{self._availability}').lower()
|
||||
|
||||
@property
|
||||
def cpus(self):
|
||||
@@ -597,10 +597,10 @@ class DockerNodeLabel:
|
||||
return hash((self._name, self._value))
|
||||
|
||||
def __repr__(self):
|
||||
return '{}({}: {})'.format(self.__class__.__name__, self._name, self._value)
|
||||
return f'{self.__class__.__name__}({self._name}: {self._value})'
|
||||
|
||||
def __str__(self):
|
||||
return '{}: {}'.format(self._name, self._value)
|
||||
return f'{self._name}: {self._value}'
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
@@ -612,12 +612,12 @@ class DockerNodeLabel:
|
||||
|
||||
@property
|
||||
def constraint_string(self):
|
||||
return 'node.labels.{name}=={value}'.format(name=self.name, value=self.value)
|
||||
return f'node.labels.{self.name}=={self.value}'
|
||||
|
||||
@property
|
||||
def constraint(self):
|
||||
return DockerServiceConstraint(
|
||||
name='node.labels.{name}'.format(name=self.name),
|
||||
name=f'node.labels.{self.name}',
|
||||
op='==',
|
||||
value=self.value
|
||||
)
|
||||
@@ -744,7 +744,7 @@ class DockerTask:
|
||||
|
||||
@property
|
||||
def state(self):
|
||||
return ('{}-{}'.format(self._desired_state, self._state)).lower()
|
||||
return (f'{self._desired_state}-{self._state}').lower()
|
||||
|
||||
@property
|
||||
def current_state(self):
|
||||
|
||||
@@ -331,7 +331,7 @@ class DockerSwarmCLIInterface(DockerSwarmInterface, DockerCLIInterface):
|
||||
|
||||
@docker_columns
|
||||
def service_ps(self, service_id):
|
||||
return self._run_docker(subcommand='service ps', args='--no-trunc {}'.format(service_id))
|
||||
return self._run_docker(subcommand='service ps', args=f'--no-trunc {service_id}')
|
||||
|
||||
def service_rm(self, service_ids):
|
||||
service_ids = ' '.join(service_ids)
|
||||
@@ -347,7 +347,7 @@ class DockerSwarmCLIInterface(DockerSwarmInterface, DockerCLIInterface):
|
||||
|
||||
@docker_columns
|
||||
def node_ps(self, node_id):
|
||||
return self._run_docker(subcommand='node ps', args='--no-trunc {}'.format(node_id))
|
||||
return self._run_docker(subcommand='node ps', args=f'--no-trunc {node_id}')
|
||||
|
||||
def node_update(self, node_id, **kwopts):
|
||||
return self._run_docker(subcommand='node update', args='{kwopts} {node_id}'.format(
|
||||
|
||||
@@ -37,7 +37,7 @@ class AnvioComposite(Html):
|
||||
missing_text = ''
|
||||
if not os.path.exists(os.path.join(dataset.extra_files_path, composite_name)):
|
||||
missing_text = ' (missing)'
|
||||
rval.append('<li><a href="{}">{}</a>{}{}</li>'.format(composite_name, composite_name, opt_text, missing_text))
|
||||
rval.append(f'<li><a href="{composite_name}">{composite_name}</a>{opt_text}{missing_text}</li>')
|
||||
rval.append("</ul>")
|
||||
defined_files = map(lambda x: x[0], defined_files)
|
||||
extra_files = []
|
||||
@@ -49,7 +49,7 @@ class AnvioComposite(Html):
|
||||
if extra_files:
|
||||
rval.append("<p/>This composite dataset contains these undefined files:<p/><ul>")
|
||||
for rel_path in extra_files:
|
||||
rval.append('<li><a href="{}">{}</a></li>'.format(rel_path, rel_path))
|
||||
rval.append(f'<li><a href="{rel_path}">{rel_path}</a></li>')
|
||||
rval.append('</ul>')
|
||||
if not (defined_files or extra_files):
|
||||
rval.append("<p/>This composite dataset does not contain any files!<p/><ul>")
|
||||
|
||||
@@ -157,7 +157,7 @@ class Velvet(Html):
|
||||
if composite_file.get('description'):
|
||||
rval.append('<li><a href="{}" type="text/plain">{} ({})</a>{}</li>'.format(fn, fn, composite_file.get('description'), opt_text))
|
||||
else:
|
||||
rval.append('<li><a href="{}" type="text/plain">{}</a>{}</li>'.format(fn, fn, opt_text))
|
||||
rval.append(f'<li><a href="{fn}" type="text/plain">{fn}</a>{opt_text}</li>')
|
||||
rval.append('</ul></div></html>')
|
||||
return "\n".join(rval)
|
||||
|
||||
@@ -204,7 +204,7 @@ class Velvet(Html):
|
||||
if composite_file.get('description'):
|
||||
rval.append('<li><a href="{}" type="text/plain">{} ({})</a>{}</li>'.format(fn, fn, composite_file.get('description'), opt_text))
|
||||
else:
|
||||
rval.append('<li><a href="{}" type="text/plain">{}</a>{}</li>'.format(fn, fn, opt_text))
|
||||
rval.append(f'<li><a href="{fn}" type="text/plain">{fn}</a>{opt_text}</li>')
|
||||
rval.append('</ul></div></html>')
|
||||
with open(dataset.file_name, 'w') as f:
|
||||
f.write("\n".join(rval))
|
||||
|
||||
@@ -403,7 +403,7 @@ class BamNative(CompressedArchive):
|
||||
# Galaxy display each tag as separate column because 'tostring()' funcition put tabs in between each tag of tags column.
|
||||
# Below code will remove spaces between each tag.
|
||||
bamline_modified = ('\t').join(bamline.split()[:11] + [(' ').join(bamline.split()[11:])])
|
||||
ck_data = "{}\n{}".format(ck_data, bamline_modified)
|
||||
ck_data = f"{ck_data}\n{bamline_modified}"
|
||||
else:
|
||||
# Nothing to enumerate; we've either offset to the end
|
||||
# of the bamfile, or there is no data. (possible with
|
||||
@@ -489,9 +489,9 @@ class Bam(BamNative):
|
||||
# we start another process and discard stderr.
|
||||
if index_flag == '-b':
|
||||
# IOError: No such file or directory: '-b' if index_flag is set to -b (pysam 0.15.4)
|
||||
cmd = ['python', '-c', "import pysam; pysam.set_verbosity(0); pysam.index('{}', '{}')".format(file_name, index_name)]
|
||||
cmd = ['python', '-c', f"import pysam; pysam.set_verbosity(0); pysam.index('{file_name}', '{index_name}')"]
|
||||
else:
|
||||
cmd = ['python', '-c', "import pysam; pysam.set_verbosity(0); pysam.index('{}', '{}', '{}')".format(index_flag, file_name, index_name)]
|
||||
cmd = ['python', '-c', f"import pysam; pysam.set_verbosity(0); pysam.index('{index_flag}', '{file_name}', '{index_name}')"]
|
||||
with open(os.devnull, 'w') as devnull:
|
||||
subprocess.check_call(cmd, stderr=devnull, shell=False)
|
||||
needs_sorting = False
|
||||
@@ -1711,7 +1711,7 @@ class CuffDiffSQlite(SQlite):
|
||||
for gene_id, gene_name in result:
|
||||
if gene_name is None:
|
||||
continue
|
||||
gene = '{}: {}'.format(gene_id, gene_name)
|
||||
gene = f'{gene_id}: {gene_name}'
|
||||
if gene not in genes:
|
||||
genes.append(gene)
|
||||
samples_query = 'SELECT DISTINCT(sample_name) as sample_name FROM samples ORDER BY sample_name'
|
||||
@@ -1859,7 +1859,7 @@ class BlibSQlite(SQlite):
|
||||
c = conn.cursor()
|
||||
tables_query = "SELECT majorVersion,minorVersion FROM LibInfo"
|
||||
(majorVersion, minorVersion) = c.execute(tables_query).fetchall()[0]
|
||||
dataset.metadata.blib_version = '{}.{}'.format(majorVersion, minorVersion)
|
||||
dataset.metadata.blib_version = f'{majorVersion}.{minorVersion}'
|
||||
except Exception as e:
|
||||
log.warning('%s, set_meta Exception: %s', self, e)
|
||||
|
||||
|
||||
@@ -147,7 +147,7 @@ class BlastXml(GenericXml):
|
||||
raise ValueError("The header in BLAST XML file %s is too long" % f)
|
||||
if "<BlastOutput>" not in header:
|
||||
h.close()
|
||||
raise ValueError("{} is not a BLAST XML file:\n{}\n...".format(f, header))
|
||||
raise ValueError(f"{f} is not a BLAST XML file:\n{header}\n...")
|
||||
if f == split_files[0]:
|
||||
out.write(header)
|
||||
old_header = header
|
||||
@@ -223,7 +223,7 @@ class _BlastDb(Data):
|
||||
if not msg:
|
||||
msg = title
|
||||
# Galaxy assumes HTML for the display of composite datatypes,
|
||||
return smart_str("<html><head><title>{}</title></head><body><pre>{}</pre></body></html>".format(title, msg))
|
||||
return smart_str(f"<html><head><title>{title}</title></head><body><pre>{msg}</pre></body></html>")
|
||||
|
||||
def merge(split_files, output_file):
|
||||
"""Merge BLAST databases (not implemented for now)."""
|
||||
|
||||
@@ -34,7 +34,7 @@ def main():
|
||||
current_line += 1
|
||||
if 0 == current_line % lines_per_chunk:
|
||||
chunk_end = in_file.tell()
|
||||
out_file.write('{{"start":"{}","end":"{}","sequences":"{}"}},'.format(chunk_begin, chunk_end, sequences))
|
||||
out_file.write(f'{{"start":"{chunk_begin}","end":"{chunk_end}","sequences":"{sequences}"}},')
|
||||
chunk_begin = chunk_end
|
||||
line = in_file.readline()
|
||||
|
||||
|
||||
@@ -41,7 +41,7 @@ def rgConv(inpedfilepath, outhtmlname, outfilepath):
|
||||
try:
|
||||
mf = open(inmap)
|
||||
except Exception:
|
||||
sys.exit('{} cannot open inmap file {} - do you have permission?\n'.format(prog, inmap))
|
||||
sys.exit(f'{prog} cannot open inmap file {inmap} - do you have permission?\n')
|
||||
try:
|
||||
rsl = [x.split()[1] for x in mf]
|
||||
except Exception:
|
||||
@@ -94,8 +94,8 @@ def main():
|
||||
flist = os.listdir(outfilepath)
|
||||
with open(outhtmlname, 'w') as f:
|
||||
f.write(galhtmlprefix % prog)
|
||||
print('## Rgenetics: http://rgenetics.org Galaxy Tools {} {}'.format(prog, timenow())) # becomes info
|
||||
f.write('<div>## Rgenetics: http://rgenetics.org Galaxy Tools {} {}\n<ol>'.format(prog, timenow()))
|
||||
print(f'## Rgenetics: http://rgenetics.org Galaxy Tools {prog} {timenow()}') # becomes info
|
||||
f.write(f'<div>## Rgenetics: http://rgenetics.org Galaxy Tools {prog} {timenow()}\n<ol>')
|
||||
for i, data in enumerate(flist):
|
||||
f.write('<li><a href="{}">{}</a></li>\n'.format(os.path.split(data)[-1], os.path.split(data)[-1]))
|
||||
f.write("</div></body></html>")
|
||||
|
||||
@@ -99,7 +99,7 @@ def main():
|
||||
flist = os.listdir(outfilepath)
|
||||
with open(outhtmlname, 'w') as f:
|
||||
f.write(galhtmlprefix % prog)
|
||||
s = '## Rgenetics: http://rgenetics.org Galaxy Tools {} {}'.format(prog, timenow()) # becomes info
|
||||
s = f'## Rgenetics: http://rgenetics.org Galaxy Tools {prog} {timenow()}' # becomes info
|
||||
print(s)
|
||||
f.write('<div>%s\n<ol>' % (s))
|
||||
for i, data in enumerate(flist):
|
||||
|
||||
@@ -60,8 +60,8 @@ def makeLDreduced(basename, infpath=None, outfpath=None, plinke='plink', forcere
|
||||
inbase = os.path.join(infpath)
|
||||
plinktasks = []
|
||||
vclbase = [plinke, '--noweb']
|
||||
plinktasks += [['--bfile', inbase, '--indep-pairwise {} {} {}'.format(winsize, winmove, r2thresh), '--out %s' % outbase],
|
||||
['--bfile', inbase, '--extract {}.prune.in --make-bed --out {}'.format(outbase, outbase)]]
|
||||
plinktasks += [['--bfile', inbase, f'--indep-pairwise {winsize} {winmove} {r2thresh}', '--out %s' % outbase],
|
||||
['--bfile', inbase, f'--extract {outbase}.prune.in --make-bed --out {outbase}']]
|
||||
vclbase = [plinke, '--noweb']
|
||||
pruneLD(plinktasks=plinktasks, cd=outfpath, vclbase=vclbase)
|
||||
|
||||
@@ -99,10 +99,10 @@ def main():
|
||||
flist = os.listdir(outfilepath)
|
||||
with open(outhtmlname, 'w') as f:
|
||||
f.write(galhtmlprefix % prog)
|
||||
s1 = '## Rgenetics: http://rgenetics.org Galaxy Tools {} {}'.format(prog, timenow()) # becomes info
|
||||
s2 = 'Input {}, winsize={}, winmove={}, r2thresh={}'.format(base_name, winsize, winmove, r2thresh)
|
||||
print('{} {}'.format(s1, s2))
|
||||
f.write('<div>{}\n{}\n<ol>'.format(s1, s2))
|
||||
s1 = f'## Rgenetics: http://rgenetics.org Galaxy Tools {prog} {timenow()}' # becomes info
|
||||
s2 = f'Input {base_name}, winsize={winsize}, winmove={winmove}, r2thresh={r2thresh}'
|
||||
print(f'{s1} {s2}')
|
||||
f.write(f'<div>{s1}\n{s2}\n<ol>')
|
||||
for i, data in enumerate(flist):
|
||||
f.write('<li><a href="{}">{}</a></li>\n'.format(os.path.split(data)[-1], os.path.split(data)[-1]))
|
||||
f.write("</div></body></html>")
|
||||
|
||||
@@ -65,7 +65,7 @@ def main():
|
||||
flist = os.listdir(outfilepath)
|
||||
with open(outhtmlname, 'w') as f:
|
||||
f.write(galhtmlprefix % prog)
|
||||
s = '## Rgenetics: http://bitbucket.org/rgalaxy Galaxy Tools {} {}'.format(prog, timenow()) # becomes info
|
||||
s = f'## Rgenetics: http://bitbucket.org/rgalaxy Galaxy Tools {prog} {timenow()}' # becomes info
|
||||
print(s)
|
||||
f.write('<div>%s\n<ol>' % (s))
|
||||
for i, data in enumerate(flist):
|
||||
|
||||
@@ -65,7 +65,7 @@ class DatatypeValidation:
|
||||
return DatatypeValidation("unknown", "Dataset validation unimplemented for this datatype.")
|
||||
|
||||
def __repr__(self):
|
||||
return "DatatypeValidation[state={},message={}]".format(self.state, self.message)
|
||||
return f"DatatypeValidation[state={self.state},message={self.message}]"
|
||||
|
||||
|
||||
def validate(dataset_instance):
|
||||
@@ -313,7 +313,7 @@ class Data(metaclass=DataMeta):
|
||||
|
||||
display_name = os.path.splitext(outfname)[0]
|
||||
if not display_name.endswith(ext):
|
||||
display_name = '{}_{}'.format(display_name, ext)
|
||||
display_name = f'{display_name}_{ext}'
|
||||
|
||||
error, msg = self._archive_main_file(archive, display_name, path)[:2]
|
||||
if not error:
|
||||
@@ -339,7 +339,7 @@ class Data(metaclass=DataMeta):
|
||||
outext = 'tgz'
|
||||
if do_action == 'tbz':
|
||||
outext = 'tbz'
|
||||
trans.response.headers["Content-Disposition"] = 'attachment; filename="{}.{}"'.format(outfname, outext)
|
||||
trans.response.headers["Content-Disposition"] = f'attachment; filename="{outfname}.{outext}"'
|
||||
archive.wsgi_status = trans.response.wsgi_status()
|
||||
archive.wsgi_headeritems = trans.response.wsgi_headeritems()
|
||||
return archive.stream
|
||||
@@ -438,7 +438,7 @@ class Data(metaclass=DataMeta):
|
||||
self._clean_and_set_mime_type(trans, mime)
|
||||
return self._yield_user_file_content(trans, data, file_path)
|
||||
else:
|
||||
return webob.exc.HTTPNotFound("Could not find '{}' on the extra files path {}.".format(filename, file_path))
|
||||
return webob.exc.HTTPNotFound(f"Could not find '{filename}' on the extra files path {file_path}.")
|
||||
self._clean_and_set_mime_type(trans, data.get_mime())
|
||||
|
||||
trans.log_event("Display dataset id: %s" % str(data.id))
|
||||
@@ -627,7 +627,7 @@ class Data(metaclass=DataMeta):
|
||||
return getattr(self, self.supported_display_apps[type]['file_function'])(dataset, **kwd)
|
||||
except Exception:
|
||||
log.exception('Function %s is referred to in datatype %s for displaying as type %s, but is not accessible', self.supported_display_apps[type]['file_function'], self.__class__.__name__, type)
|
||||
return "This display type ({}) is not implemented for this datatype ({}).".format(type, dataset.ext)
|
||||
return f"This display type ({type}) is not implemented for this datatype ({dataset.ext})."
|
||||
|
||||
def get_display_links(self, dataset, type, app, base_url, target_frame='_blank', **kwd):
|
||||
"""
|
||||
@@ -657,7 +657,7 @@ class Data(metaclass=DataMeta):
|
||||
converter = trans.app.datatypes_registry.get_converter_by_target_type(original_dataset.ext, target_type)
|
||||
|
||||
if converter is None:
|
||||
raise Exception("A converter does not exist for {} to {}.".format(original_dataset.ext, target_type))
|
||||
raise Exception(f"A converter does not exist for {original_dataset.ext} to {target_type}.")
|
||||
# Generate parameter dictionary
|
||||
params = {}
|
||||
# determine input parameter name and add to params
|
||||
@@ -683,7 +683,7 @@ class Data(metaclass=DataMeta):
|
||||
value.visible = False
|
||||
if return_output:
|
||||
return converted_dataset
|
||||
return "The file conversion of {} on data {} has been added to the Queue.".format(converter.name, original_dataset.hid)
|
||||
return f"The file conversion of {converter.name} on data {original_dataset.hid} has been added to the Queue."
|
||||
|
||||
# We need to clear associated files before we set metadata
|
||||
# so that as soon as metadata starts to be set, e.g. implicitly converted datasets are deleted and no longer available 'while' metadata is being set, not just after
|
||||
@@ -856,7 +856,7 @@ class Text(Data):
|
||||
sample_lines = dataset_read.count('\n')
|
||||
est_lines = int(sample_lines * (float(dataset.get_size()) / float(sample_size)))
|
||||
except UnicodeDecodeError:
|
||||
log.error('Unable to estimate lines in file {}'.format(dataset.file_name))
|
||||
log.error(f'Unable to estimate lines in file {dataset.file_name}')
|
||||
est_lines = None
|
||||
return est_lines
|
||||
|
||||
@@ -880,7 +880,7 @@ class Text(Data):
|
||||
if line and not line.startswith('#'):
|
||||
data_lines += 1
|
||||
except UnicodeDecodeError:
|
||||
log.error('Unable to count lines in file {}'.format(dataset.file_name))
|
||||
log.error(f'Unable to count lines in file {dataset.file_name}')
|
||||
data_lines = None
|
||||
return data_lines
|
||||
|
||||
|
||||
@@ -4,8 +4,7 @@ is further subdivided into multiple data (e.g. columns from a line).
|
||||
"""
|
||||
import logging
|
||||
import re
|
||||
|
||||
from six.moves.urllib.parse import unquote_plus
|
||||
from urllib.parse import unquote_plus
|
||||
|
||||
from . import line
|
||||
|
||||
|
||||
@@ -17,8 +17,7 @@ DataProvider related decorators.
|
||||
import copy
|
||||
import logging
|
||||
from functools import wraps
|
||||
|
||||
from six.moves.urllib.parse import unquote
|
||||
from urllib.parse import unquote
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -104,8 +103,8 @@ def dataprovider_factory(name, settings=None):
|
||||
def named_dataprovider_factory(func):
|
||||
setattr(func, _DATAPROVIDER_METHOD_NAME_KEY, name)
|
||||
|
||||
setattr(func, 'parse_query_string_settings', parse_query_string_settings)
|
||||
setattr(func, 'settings', settings)
|
||||
func.parse_query_string_settings = parse_query_string_settings
|
||||
func.settings = settings
|
||||
# TODO: I want a way to inherit settings from the previous provider( this_name ) instead of defining over and over
|
||||
|
||||
@wraps(func)
|
||||
|
||||
@@ -6,9 +6,11 @@ import gzip
|
||||
import logging
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
from six.moves.urllib.parse import urlencode, urlparse
|
||||
from six.moves.urllib.request import urlopen
|
||||
from urllib.parse import (
|
||||
urlencode,
|
||||
urlparse,
|
||||
)
|
||||
from urllib.request import urlopen
|
||||
|
||||
from . import (
|
||||
base,
|
||||
|
||||
@@ -2,8 +2,7 @@
|
||||
import logging
|
||||
from collections import OrderedDict
|
||||
from copy import deepcopy
|
||||
|
||||
from six.moves.urllib.parse import quote_plus
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
from galaxy.util import (
|
||||
parse_xml,
|
||||
@@ -192,7 +191,7 @@ class DynamicDisplayApplicationBuilder:
|
||||
# now populate
|
||||
links.append(DisplayApplicationLink.from_elem(new_elem, display_application, other_values=dynamic_values))
|
||||
else:
|
||||
log.warning('Invalid dynamic display application link specified in {}: "{}"'.format(filename, line))
|
||||
log.warning(f'Invalid dynamic display application link specified in {filename}: "{line}"')
|
||||
self.links = links
|
||||
|
||||
def __iter__(self):
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# Contains parameters that are used in Display Applications
|
||||
import mimetypes
|
||||
|
||||
from six.moves.urllib.parse import quote_plus
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
from galaxy.util import string_as_bool
|
||||
from galaxy.util.bunch import Bunch
|
||||
@@ -83,7 +82,7 @@ class DisplayApplicationDataParameter(DisplayApplicationParameter):
|
||||
data = data.value
|
||||
if self.metadata:
|
||||
rval = getattr(data.metadata, self.metadata, None)
|
||||
assert rval, 'Unknown metadata name ({}) provided for dataset type ({}).'.format(self.metadata, data.datatype.__class__.name)
|
||||
assert rval, f'Unknown metadata name ({self.metadata}) provided for dataset type ({data.datatype.__class__.name}).'
|
||||
return Bunch(file_name=rval.file_name, state=data.state, states=data.states, extension='data')
|
||||
elif self.extensions and (self.force_conversion or not isinstance(data.datatype, self.formats)):
|
||||
for ext in self.extensions:
|
||||
|
||||
@@ -15,9 +15,9 @@ import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
from markupsafe import escape
|
||||
from six.moves.urllib.parse import quote_plus
|
||||
|
||||
from galaxy.datatypes import metadata
|
||||
from galaxy.datatypes.data import (
|
||||
@@ -107,7 +107,7 @@ class GenomeGraphs(Tabular):
|
||||
display_url = quote_plus(display_url)
|
||||
# was display_url = quote_plus( "%s/display_as?id=%i&display_app=%s" % (base_url, dataset.id, type) )
|
||||
# redirect_url = quote_plus( "%sdb=%s&position=%s:%s-%s&hgt.customText=%%s" % (site_url, dataset.dbkey, chrom, start, stop) )
|
||||
sl = ["{}db={}".format(site_url, dataset.dbkey), ]
|
||||
sl = [f"{site_url}db={dataset.dbkey}", ]
|
||||
# sl.append("&hgt.customText=%s")
|
||||
sl.append("&hgGenome_dataSetName={}&hgGenome_dataSetDescription={}".format(dataset.name, 'GalaxyGG_data'))
|
||||
sl.append("&hgGenome_formatType=best guess&hgGenome_markerType=best guess")
|
||||
@@ -117,7 +117,7 @@ class GenomeGraphs(Tabular):
|
||||
s = ''.join(sl)
|
||||
s = quote_plus(s)
|
||||
redirect_url = s
|
||||
link = '{}?redirect_url={}&display_url={}'.format(internal_url, redirect_url, display_url)
|
||||
link = f'{internal_url}?redirect_url={redirect_url}&display_url={display_url}'
|
||||
ret_val.append((site_name, link))
|
||||
return ret_val
|
||||
|
||||
@@ -286,7 +286,7 @@ class Rgenetics(Html):
|
||||
if composite_file.get('description'):
|
||||
rval.append('<li><a href="{}" type="application/binary">{} ({})</a>{}</li>'.format(fn, fn, composite_file.get('description'), opt_text))
|
||||
else:
|
||||
rval.append('<li><a href="{}" type="application/binary">{}</a>{}</li>'.format(fn, fn, opt_text))
|
||||
rval.append(f'<li><a href="{fn}" type="application/binary">{fn}</a>{opt_text}</li>')
|
||||
rval.append('</ul></div></html>')
|
||||
return "\n".join(rval)
|
||||
|
||||
@@ -296,11 +296,11 @@ class Rgenetics(Html):
|
||||
"""
|
||||
efp = dataset.extra_files_path
|
||||
flist = os.listdir(efp)
|
||||
rval = ['<html><head><title>Files for Composite Dataset {}</title></head><body><p/>Composite {} contains:<p/><ul>'.format(dataset.name, dataset.name)]
|
||||
rval = [f'<html><head><title>Files for Composite Dataset {dataset.name}</title></head><body><p/>Composite {dataset.name} contains:<p/><ul>']
|
||||
for fname in flist:
|
||||
sfname = os.path.split(fname)[-1]
|
||||
f, e = os.path.splitext(fname)
|
||||
rval.append('<li><a href="{}">{}</a></li>'.format(sfname, sfname))
|
||||
rval.append(f'<li><a href="{sfname}">{sfname}</a></li>')
|
||||
rval.append('</ul></body></html>')
|
||||
with open(dataset.file_name, 'w') as f:
|
||||
f.write("\n".join(rval))
|
||||
@@ -334,7 +334,7 @@ class Rgenetics(Html):
|
||||
return False
|
||||
if len(flist) == 0:
|
||||
if verbose:
|
||||
gal_Log.debug('@@@rgenetics set_meta failed - {} efp {} is empty?'.format(dataset.name, efp))
|
||||
gal_Log.debug(f'@@@rgenetics set_meta failed - {dataset.name} efp {efp} is empty?')
|
||||
return False
|
||||
self.regenerate_primary_file(dataset)
|
||||
if not dataset.info:
|
||||
@@ -555,7 +555,7 @@ class IdeasPre(Html):
|
||||
rval.append('<ul>')
|
||||
for composite_name in self.get_composite_files(dataset=dataset).keys():
|
||||
fn = composite_name
|
||||
rval.append('<li><a href="{}>{}</a></li>'.format(fn, fn))
|
||||
rval.append(f'<li><a href="{fn}>{fn}</a></li>')
|
||||
rval.append('</ul></body></html>\n')
|
||||
return "\n".join(rval)
|
||||
|
||||
@@ -566,7 +566,7 @@ class IdeasPre(Html):
|
||||
rval.append('<ul>')
|
||||
for fname in os.listdir(dataset.extra_files_path):
|
||||
fn = os.path.split(fname)[-1]
|
||||
rval.append('<li><a href="{}">{}</a></li>'.format(fn, fn))
|
||||
rval.append(f'<li><a href="{fn}">{fn}</a></li>')
|
||||
rval.append('</ul></body></html>')
|
||||
with open(dataset.file_name, 'w') as f:
|
||||
f.write("\n".join(rval))
|
||||
@@ -752,7 +752,7 @@ class RexpBase(Html):
|
||||
rval = ['<html><head><title>Files for Composite Dataset %s</title></head><p/>Comprises the following files:<p/><ul>' % (bn)]
|
||||
for fname in flist:
|
||||
sfname = os.path.split(fname)[-1]
|
||||
rval.append('<li><a href="{}">{}</a>'.format(sfname, sfname))
|
||||
rval.append(f'<li><a href="{sfname}">{sfname}</a>')
|
||||
rval.append('</ul></html>')
|
||||
with open(dataset.file_name, 'w') as f:
|
||||
f.write("\n".join(rval))
|
||||
|
||||
@@ -42,7 +42,7 @@ class Shapefile(Binary):
|
||||
if composite_file.get('description'):
|
||||
rval.append('<li><a href="{}" type="application/binary">{} ({})</a>{}</li>'.format(fn, fn, composite_file.get('description'), opt_text))
|
||||
else:
|
||||
rval.append('<li><a href="{}" type="application/binary">{}</a>{}</li>'.format(fn, fn, opt_text))
|
||||
rval.append(f'<li><a href="{fn}" type="application/binary">{fn}</a>{opt_text}</li>')
|
||||
rval.append('</ul></div></html>\n')
|
||||
return "\n".join(rval)
|
||||
|
||||
|
||||
@@ -4,8 +4,7 @@ Image classes
|
||||
import base64
|
||||
import logging
|
||||
import zipfile
|
||||
|
||||
from six.moves.urllib.parse import quote_plus
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
from galaxy.datatypes.text import Html as HtmlFromText
|
||||
from galaxy.util import nice_size
|
||||
@@ -52,7 +51,7 @@ class Image(data.Data):
|
||||
name = hda.name or ''
|
||||
with open(dataset.file_name, "rb") as f:
|
||||
base64_image_data = base64.b64encode(f.read()).decode("utf-8")
|
||||
return "".format(name, self.file_ext, base64_image_data)
|
||||
return f""
|
||||
|
||||
|
||||
class Jpg(Image):
|
||||
@@ -177,14 +176,14 @@ def create_applet_tag_peek(class_name, archive, params):
|
||||
height="30" width="200" align="center" >
|
||||
<param name="archive" value="{}"/>""".format(class_name, archive)
|
||||
for name, value in params.items():
|
||||
text += """<param name="{}" value="{}"/>""".format(name, value)
|
||||
text += f"""<param name="{name}" value="{value}"/>"""
|
||||
text += """
|
||||
<object classid="clsid:8AD9C840-044E-11D1-B3E9-00805F499D93"
|
||||
height="30" width="200" >
|
||||
<param name="code" value="{}" />
|
||||
<param name="archive" value="{}"/>""".format(class_name, archive)
|
||||
for name, value in params.items():
|
||||
text += """<param name="{}" value="{}"/>""".format(name, value)
|
||||
text += f"""<param name="{name}" value="{value}"/>"""
|
||||
text += """<div class="errormessage">You must install and enable Java in your browser in order to access this applet.<div></object>
|
||||
</object>
|
||||
"""
|
||||
@@ -265,7 +264,7 @@ class Laj(data.Text):
|
||||
"alignfile1": "display?id=%s" % dataset.id,
|
||||
"buttonlabel": "Launch LAJ",
|
||||
"title": "LAJ in Galaxy",
|
||||
"posturl": quote_plus("history_add_to?%s" % "&".join("{}={}".format(key, value) for key, value in {'history_id': dataset.history_id, 'ext': 'lav', 'name': 'LAJ Output', 'info': 'Added by LAJ', 'dbkey': dataset.dbkey, 'copy_access_from': dataset.id}.items())),
|
||||
"posturl": quote_plus("history_add_to?%s" % "&".join(f"{key}={value}" for key, value in {'history_id': dataset.history_id, 'ext': 'lav', 'name': 'LAJ Output', 'info': 'Added by LAJ', 'dbkey': dataset.dbkey, 'copy_access_from': dataset.id}.items())),
|
||||
"noseq": "true"
|
||||
}
|
||||
class_name = "edu.psu.cse.bio.laj.LajApplet.class"
|
||||
|
||||
@@ -5,9 +5,9 @@ import logging
|
||||
import math
|
||||
import sys
|
||||
import tempfile
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
from bx.intervals.io import GenomicIntervalReader, ParseError
|
||||
from six.moves.urllib.parse import quote_plus
|
||||
|
||||
from galaxy import util
|
||||
from galaxy.datatypes import metadata
|
||||
@@ -264,7 +264,7 @@ class Interval(Tabular):
|
||||
(base_url, app.url_for(controller='root'), dataset.id, type))
|
||||
redirect_url = quote_plus("%sdb=%s&position=%s:%s-%s&hgt.customText=%%s" %
|
||||
(site_url, dataset.dbkey, chrom, start, stop))
|
||||
link = '{}?redirect_url={}&display_url={}'.format(internal_url, redirect_url, display_url)
|
||||
link = f'{internal_url}?redirect_url={redirect_url}&display_url={display_url}'
|
||||
ret_val.append((site_name, link))
|
||||
return ret_val
|
||||
|
||||
@@ -604,11 +604,11 @@ class _RemoteCallMixin:
|
||||
the data available, followed by redirecting to the remote site with a
|
||||
link back to the available information.
|
||||
"""
|
||||
internal_url = "%s" % app.url_for(controller='dataset', dataset_id=dataset.id, action='display_at', filename='{}_{}'.format(type, site_name))
|
||||
internal_url = "%s" % app.url_for(controller='dataset', dataset_id=dataset.id, action='display_at', filename=f'{type}_{site_name}')
|
||||
base_url = app.config.get("display_at_callback", base_url)
|
||||
display_url = quote_plus("%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" %
|
||||
(base_url, app.url_for(controller='root'), dataset.id, type))
|
||||
link = '{}?redirect_url={}&display_url={}'.format(internal_url, redirect_url, display_url)
|
||||
link = f'{internal_url}?redirect_url={redirect_url}&display_url={display_url}'
|
||||
return link
|
||||
|
||||
|
||||
@@ -794,7 +794,7 @@ class Gff(Tabular, _RemoteCallMixin):
|
||||
if site_name in app.datatypes_registry.get_display_sites('gbrowse'):
|
||||
if seqid.startswith('chr') and len(seqid) > 3:
|
||||
seqid = seqid[3:]
|
||||
redirect_url = quote_plus("{}/?q={}:{}..{}&eurl=%s".format(site_url, seqid, start, stop))
|
||||
redirect_url = quote_plus(f"{site_url}/?q={seqid}:{start}..{stop}&eurl=%s")
|
||||
link = self._get_remote_call_url(redirect_url, site_name, dataset, type, app, base_url)
|
||||
ret_val.append((site_name, link))
|
||||
return ret_val
|
||||
@@ -1146,7 +1146,7 @@ class Wiggle(Tabular, _RemoteCallMixin):
|
||||
if site_name in app.datatypes_registry.get_display_sites('gbrowse'):
|
||||
if chrom.startswith('chr') and len(chrom) > 3:
|
||||
chrom = chrom[3:]
|
||||
redirect_url = quote_plus("{}/?q={}:{}..{}&eurl=%s".format(site_url, chrom, start, stop))
|
||||
redirect_url = quote_plus(f"{site_url}/?q={chrom}:{start}..{stop}&eurl=%s")
|
||||
link = self._get_remote_call_url(redirect_url, site_name, dataset, type, app, base_url)
|
||||
ret_val.append((site_name, link))
|
||||
return ret_val
|
||||
@@ -1157,7 +1157,7 @@ class Wiggle(Tabular, _RemoteCallMixin):
|
||||
if chrom is not None:
|
||||
for site_name, site_url in app.datatypes_registry.get_legacy_sites_by_build('ucsc', dataset.dbkey):
|
||||
if site_name in app.datatypes_registry.get_display_sites('ucsc'):
|
||||
redirect_url = quote_plus("{}db={}&position={}:{}-{}&hgt.customText=%s".format(site_url, dataset.dbkey, chrom, start, stop))
|
||||
redirect_url = quote_plus(f"{site_url}db={dataset.dbkey}&position={chrom}:{start}-{stop}&hgt.customText=%s")
|
||||
link = self._get_remote_call_url(redirect_url, site_name, dataset, type, app, base_url)
|
||||
ret_val.append((site_name, link))
|
||||
return ret_val
|
||||
@@ -1324,8 +1324,8 @@ class CustomTrack(Tabular):
|
||||
if site_name in app.datatypes_registry.get_display_sites('ucsc'):
|
||||
internal_url = "%s" % app.url_for(controller='dataset', dataset_id=dataset.id, action='display_at', filename='ucsc_' + site_name)
|
||||
display_url = quote_plus("%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" % (base_url, app.url_for(controller='root'), dataset.id, type))
|
||||
redirect_url = quote_plus("{}db={}&position={}:{}-{}&hgt.customText=%s".format(site_url, dataset.dbkey, chrom, start, stop))
|
||||
link = '{}?redirect_url={}&display_url={}'.format(internal_url, redirect_url, display_url)
|
||||
redirect_url = quote_plus(f"{site_url}db={dataset.dbkey}&position={chrom}:{start}-{stop}&hgt.customText=%s")
|
||||
link = f'{internal_url}?redirect_url={redirect_url}&display_url={display_url}'
|
||||
ret_val.append((site_name, link))
|
||||
return ret_val
|
||||
|
||||
|
||||
@@ -274,7 +274,7 @@ class _Isa(data.Data):
|
||||
</body></html>"""
|
||||
else:
|
||||
html = '<html><body>'
|
||||
html += '<h1>{} {}</h1>'.format(investigation.title, investigation.identifier)
|
||||
html += f'<h1>{investigation.title} {investigation.identifier}</h1>'
|
||||
|
||||
# Loop on all studies
|
||||
for study in investigation.studies:
|
||||
|
||||
@@ -38,9 +38,9 @@ class GenericMicroarrayFile(data.Text):
|
||||
def set_peek(self, dataset, is_multi_byte=False):
|
||||
if not dataset.dataset.purged:
|
||||
if dataset.metadata.block_count == 1:
|
||||
dataset.blurb = "{} {}: Format {}, 1 block, {} headers and {} columns".format(dataset.metadata.file_type, dataset.metadata.version_number, dataset.metadata.file_format, dataset.metadata.number_of_optional_header_records, dataset.metadata.number_of_data_columns)
|
||||
dataset.blurb = f"{dataset.metadata.file_type} {dataset.metadata.version_number}: Format {dataset.metadata.file_format}, 1 block, {dataset.metadata.number_of_optional_header_records} headers and {dataset.metadata.number_of_data_columns} columns"
|
||||
else:
|
||||
dataset.blurb = "{} {}: Format {}, {} blocks, {} headers and {} columns".format(dataset.metadata.file_type, dataset.metadata.version_number, dataset.metadata.file_format, dataset.metadata.block_count, dataset.metadata.number_of_optional_header_records, dataset.metadata.number_of_data_columns)
|
||||
dataset.blurb = f"{dataset.metadata.file_type} {dataset.metadata.version_number}: Format {dataset.metadata.file_format}, {dataset.metadata.block_count} blocks, {dataset.metadata.number_of_optional_header_records} headers and {dataset.metadata.number_of_data_columns} columns"
|
||||
dataset.peek = get_file_peek(dataset.file_name)
|
||||
else:
|
||||
dataset.peek = 'file does not exist'
|
||||
|
||||
@@ -514,7 +514,7 @@ class PDB(GenericMolFile):
|
||||
hetatm_numbers = count_special_lines("^HETATM", dataset.file_name)
|
||||
chain_ids = ','.join(dataset.metadata.chain_ids) if len(dataset.metadata.chain_ids) > 0 else 'None'
|
||||
dataset.peek = get_file_peek(dataset.file_name)
|
||||
dataset.blurb = "{} atoms and {} HET-atoms\nchain_ids: {}".format(atom_numbers, hetatm_numbers, chain_ids)
|
||||
dataset.blurb = f"{atom_numbers} atoms and {hetatm_numbers} HET-atoms\nchain_ids: {chain_ids}"
|
||||
else:
|
||||
dataset.peek = 'file does not exist'
|
||||
dataset.blurb = 'file purged from disk'
|
||||
@@ -565,7 +565,7 @@ class PDBQT(GenericMolFile):
|
||||
root_numbers = count_special_lines("^ROOT", dataset.file_name)
|
||||
branch_numbers = count_special_lines("^BRANCH", dataset.file_name)
|
||||
dataset.peek = get_file_peek(dataset.file_name)
|
||||
dataset.blurb = "{} roots and {} branches".format(root_numbers, branch_numbers)
|
||||
dataset.blurb = f"{root_numbers} roots and {branch_numbers} branches"
|
||||
else:
|
||||
dataset.peek = 'file does not exist'
|
||||
dataset.blurb = 'file purged from disk'
|
||||
|
||||
@@ -36,7 +36,7 @@ class BowtieIndex(Html):
|
||||
rval = ['<html><head><title>Files for Composite Dataset %s</title></head><p/>Comprises the following files:<p/><ul>' % (bn)]
|
||||
for fname in flist:
|
||||
sfname = os.path.split(fname)[-1]
|
||||
rval.append('<li><a href="{}">{}</a>'.format(sfname, sfname))
|
||||
rval.append(f'<li><a href="{sfname}">{sfname}</a>')
|
||||
rval.append('</ul></html>')
|
||||
with open(dataset.file_name, 'w') as f:
|
||||
f.write("\n".join(rval))
|
||||
|
||||
@@ -48,7 +48,7 @@ class Wiff(Binary):
|
||||
if composite_file.get('description'):
|
||||
rval.append('<li><a href="{}" type="text/plain">{} ({})</a>{}</li>'.format(fn, fn, composite_file.get('description'), opt_text))
|
||||
else:
|
||||
rval.append('<li><a href="{}" type="text/plain">{}</a>{}</li>'.format(fn, fn, opt_text))
|
||||
rval.append(f'<li><a href="{fn}" type="text/plain">{fn}</a>{opt_text}</li>')
|
||||
rval.append('</ul></div></html>')
|
||||
return "\n".join(rval)
|
||||
|
||||
@@ -900,7 +900,7 @@ class SPLib(Msp):
|
||||
if composite_file.get('description'):
|
||||
rval.append('<li><a href="{}" type="text/plain">{} ({})</a>{}</li>'.format(fn, fn, composite_file.get('description'), opt_text))
|
||||
else:
|
||||
rval.append('<li><a href="{}" type="text/plain">{}</a>{}</li>'.format(fn, fn, opt_text))
|
||||
rval.append(f'<li><a href="{fn}" type="text/plain">{fn}</a>{opt_text}</li>')
|
||||
rval.append('</ul></div></html>')
|
||||
return "\n".join(rval)
|
||||
|
||||
@@ -995,7 +995,7 @@ class ImzML(Binary):
|
||||
if composite_file.get('description'):
|
||||
rval.append('<li><a href="{}" type="text/plain">{} ({})</a>{}</li>'.format(fn, fn, composite_file.get('description'), opt_text))
|
||||
else:
|
||||
rval.append('<li><a href="{}" type="text/plain">{}</a>{}</li>'.format(fn, fn, opt_text))
|
||||
rval.append(f'<li><a href="{fn}" type="text/plain">{fn}</a>{opt_text}</li>')
|
||||
rval.append('</ul></div></html>')
|
||||
return "\n".join(rval)
|
||||
|
||||
@@ -1041,6 +1041,6 @@ class Analyze75(Binary):
|
||||
if composite_file.get('description'):
|
||||
rval.append('<li><a href="{}" type="text/plain">{} ({})</a>{}</li>'.format(fn, fn, composite_file.get('description'), opt_text))
|
||||
else:
|
||||
rval.append('<li><a href="{}" type="text/plain">{}</a>{}</li>'.format(fn, fn, opt_text))
|
||||
rval.append(f'<li><a href="{fn}" type="text/plain">{fn}</a>{opt_text}</li>')
|
||||
rval.append('</ul></div></html>')
|
||||
return "\n".join(rval)
|
||||
|
||||
@@ -330,7 +330,7 @@ class Registry:
|
||||
self.datatype_info_dicts.append(datatype_info_dict)
|
||||
|
||||
for auto_compressed_type in auto_compressed_types:
|
||||
compressed_extension = "{}.{}".format(extension, auto_compressed_type)
|
||||
compressed_extension = f"{extension}.{auto_compressed_type}"
|
||||
upper_compressed_type = auto_compressed_type[0].upper() + auto_compressed_type[1:]
|
||||
auto_compressed_type_name = datatype_class_name + upper_compressed_type
|
||||
attributes = {}
|
||||
@@ -372,7 +372,7 @@ class Registry:
|
||||
if not override:
|
||||
# Do not load the datatype since it conflicts with an existing datatype which we are not supposed
|
||||
# to override.
|
||||
self.log.debug("Ignoring conflicting datatype with extension '{}' from {}.".format(extension, config))
|
||||
self.log.debug(f"Ignoring conflicting datatype with extension '{extension}' from {config}.")
|
||||
# Load datatype sniffers from the config - we'll do this even if one or more datatypes were not properly processed in the config
|
||||
# since sniffers are not tightly coupled with datatypes.
|
||||
self.load_datatype_sniffers(root,
|
||||
@@ -413,7 +413,7 @@ class Registry:
|
||||
if not os.path.exists(path):
|
||||
sample_path = "%s.sample" % path
|
||||
if os.path.exists(sample_path):
|
||||
self.log.debug("Build site file [{}] not found using sample [{}].".format(path, sample_path))
|
||||
self.log.debug(f"Build site file [{path}] not found using sample [{sample_path}].")
|
||||
path = sample_path
|
||||
|
||||
self.build_sites[site_type] = path
|
||||
@@ -703,13 +703,13 @@ class Registry:
|
||||
del self.datatypes_by_extension[extension].display_applications[display_app.id]
|
||||
if inherit and (self.datatypes_by_extension[extension], display_app) in self.inherit_display_application_by_class:
|
||||
self.inherit_display_application_by_class.remove((self.datatypes_by_extension[extension], display_app))
|
||||
self.log.debug("Deactivated display application '{}' for datatype '{}'.".format(display_app.id, extension))
|
||||
self.log.debug(f"Deactivated display application '{display_app.id}' for datatype '{extension}'.")
|
||||
else:
|
||||
self.display_applications[display_app.id] = display_app
|
||||
self.datatypes_by_extension[extension].add_display_application(display_app)
|
||||
if inherit and (self.datatypes_by_extension[extension], display_app) not in self.inherit_display_application_by_class:
|
||||
self.inherit_display_application_by_class.append((self.datatypes_by_extension[extension], display_app))
|
||||
self.log.debug("Loaded display application '{}' for datatype '{}', inherit={}.".format(display_app.id, extension, inherit))
|
||||
self.log.debug(f"Loaded display application '{display_app.id}' for datatype '{extension}', inherit={inherit}.")
|
||||
except Exception:
|
||||
if deactivate:
|
||||
self.log.exception("Error deactivating display application (%s)" % config_path)
|
||||
@@ -720,7 +720,7 @@ class Registry:
|
||||
for d_type2, display_app in self.inherit_display_application_by_class:
|
||||
current_app = d_type1.get_display_application(display_app.id, None)
|
||||
if current_app is None and isinstance(d_type1, type(d_type2)):
|
||||
self.log.debug("Adding inherited display application '{}' to datatype '{}'".format(display_app.id, extension))
|
||||
self.log.debug(f"Adding inherited display application '{display_app.id}' to datatype '{extension}'")
|
||||
d_type1.add_display_application(display_app)
|
||||
|
||||
def reload_display_applications(self, display_application_ids=None):
|
||||
@@ -894,7 +894,7 @@ class Registry:
|
||||
for convert_ext in self.get_converters_by_datatype(ext):
|
||||
convert_ext_datatype = self.get_datatype_by_extension(convert_ext)
|
||||
if convert_ext_datatype is None:
|
||||
self.log.warning("Datatype class not found for extension '{}', which is used as target for conversion from datatype '{}'".format(convert_ext, dataset.ext))
|
||||
self.log.warning(f"Datatype class not found for extension '{convert_ext}', which is used as target for conversion from datatype '{dataset.ext}'")
|
||||
elif convert_ext_datatype.matches_any(accepted_formats):
|
||||
converted_dataset = dataset and dataset.get_converted_files_by_type(convert_ext)
|
||||
if converted_dataset:
|
||||
@@ -919,7 +919,7 @@ class Registry:
|
||||
help_txt = meta_spec.desc
|
||||
if not help_txt or help_txt == meta_name:
|
||||
help_txt = ""
|
||||
inputs.append('<param type="text" name="{}" label="Set metadata value for "{}"" value="{}" help="{}"/>'.format(meta_name, meta_name, meta_spec.default, help_txt))
|
||||
inputs.append(f'<param type="text" name="{meta_name}" label="Set metadata value for "{meta_name}"" value="{meta_spec.default}" help="{help_txt}"/>')
|
||||
rval[ext] = "\n".join(inputs)
|
||||
if 'auto' not in rval and 'txt' in rval: # need to manually add 'auto' datatype
|
||||
rval['auto'] = rval['txt']
|
||||
|
||||
@@ -186,7 +186,7 @@ class Sequence(data.Text):
|
||||
ds = input_datasets[ds_no]
|
||||
base_name = os.path.basename(ds.file_name)
|
||||
part_path = os.path.join(dir, base_name)
|
||||
split_data = dict(class_name='{}.{}'.format(cls.__module__, cls.__name__),
|
||||
split_data = dict(class_name=f'{cls.__module__}.{cls.__name__}',
|
||||
output_name=part_path,
|
||||
input_name=ds.file_name,
|
||||
args=dict(start_sequence=start_sequence, num_sequences=sequences_per_file[part_no]))
|
||||
@@ -431,7 +431,7 @@ class Fasta(Sequence):
|
||||
part_dir = subdir_generator_function()
|
||||
part_path = os.path.join(part_dir, os.path.basename(input_file))
|
||||
part_file = open(part_path, 'w')
|
||||
log.debug("Writing {} part to {}".format(input_file, part_path))
|
||||
log.debug(f"Writing {input_file} part to {part_path}")
|
||||
start_offset = 0
|
||||
while True:
|
||||
offset = f.tell()
|
||||
@@ -444,7 +444,7 @@ class Fasta(Sequence):
|
||||
part_dir = subdir_generator_function()
|
||||
part_path = os.path.join(part_dir, os.path.basename(input_file))
|
||||
part_file = open(part_path, 'w')
|
||||
log.debug("Writing {} part to {}".format(input_file, part_path))
|
||||
log.debug(f"Writing {input_file} part to {part_path}")
|
||||
start_offset = f.tell()
|
||||
part_file.write(line)
|
||||
except Exception as e:
|
||||
@@ -467,7 +467,7 @@ class Fasta(Sequence):
|
||||
part_dir = subdir_generator_function()
|
||||
part_path = os.path.join(part_dir, os.path.basename(input_file))
|
||||
part_file = open(part_path, 'w')
|
||||
log.debug("Writing {} part to {}".format(input_file, part_path))
|
||||
log.debug(f"Writing {input_file} part to {part_path}")
|
||||
rec_count = 0
|
||||
while True:
|
||||
line = f.readline()
|
||||
@@ -481,7 +481,7 @@ class Fasta(Sequence):
|
||||
part_dir = subdir_generator_function()
|
||||
part_path = os.path.join(part_dir, os.path.basename(input_file))
|
||||
part_file = open(part_path, 'w')
|
||||
log.debug("Writing {} part to {}".format(input_file, part_path))
|
||||
log.debug(f"Writing {input_file} part to {part_path}")
|
||||
rec_count = 1
|
||||
part_file.write(line)
|
||||
except Exception as e:
|
||||
@@ -633,7 +633,7 @@ class Fastg(Sequence):
|
||||
dataset.blurb += '\nversion=%s' % dataset.metadata.version
|
||||
for k, v in dataset.metadata.properties.items():
|
||||
if k != 'version':
|
||||
dataset.blurb += '\n{}={}'.format(k, v)
|
||||
dataset.blurb += f'\n{k}={v}'
|
||||
else:
|
||||
dataset.peek = 'file does not exist'
|
||||
dataset.blurb = 'file purged from disk'
|
||||
|
||||
@@ -88,7 +88,7 @@ class _SpalnDb(Data):
|
||||
for i, fname in enumerate(flist):
|
||||
sfname = os.path.split(fname)[-1]
|
||||
f, e = os.path.splitext(fname)
|
||||
rval.append('<li><a href="{}">{}</a></li>'.format(sfname, sfname))
|
||||
rval.append(f'<li><a href="{sfname}">{sfname}</a></li>')
|
||||
rval.append("</ul></body></html>")
|
||||
with open(dataset.file_name, "w") as f:
|
||||
f.write("\n".join(rval))
|
||||
|
||||
@@ -25,7 +25,7 @@ class TextGrid(Text):
|
||||
|
||||
def sniff(self, filename):
|
||||
|
||||
with open(filename, 'r') as fd:
|
||||
with open(filename) as fd:
|
||||
text = fd.read(len(self.header))
|
||||
return text == self.header
|
||||
|
||||
@@ -56,7 +56,7 @@ class BPF(Text):
|
||||
def set_meta(self, dataset, overwrite=True, **kwd):
|
||||
"""Set the metadata for this dataset from the file contents"""
|
||||
types = set()
|
||||
with open(dataset.dataset.file_name, 'r') as fd:
|
||||
with open(dataset.dataset.file_name) as fd:
|
||||
for line in fd:
|
||||
# Split the line on a colon rather than regexing it
|
||||
parts = line.split(':')
|
||||
|
||||
@@ -305,7 +305,7 @@ class Tabular(TabularData):
|
||||
if column_type2 == column_type:
|
||||
return False
|
||||
# neither column type was found in our ordered list, this cannot happen
|
||||
raise ValueError("Tried to compare unknown column types: {} and {}".format(column_type1, column_type2))
|
||||
raise ValueError(f"Tried to compare unknown column types: {column_type1} and {column_type2}")
|
||||
|
||||
def is_int(column_text):
|
||||
# Don't allow underscores in numeric literals (PEP 515)
|
||||
|
||||
@@ -6,11 +6,10 @@ import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
from six.moves import shlex_quote
|
||||
|
||||
from galaxy.datatypes.data import get_file_peek, Text
|
||||
from galaxy.datatypes.metadata import MetadataElement, MetadataParameter
|
||||
from galaxy.datatypes.sniff import build_sniff_from_prefix, iter_headers
|
||||
@@ -181,7 +180,7 @@ class Ipynb(Json):
|
||||
ofilename = '%s.html' % ofilename
|
||||
except subprocess.CalledProcessError:
|
||||
ofilename = dataset.file_name
|
||||
log.exception('Command "%s" failed. Could not convert the Jupyter Notebook to HTML, defaulting to plain text.', ' '.join(map(shlex_quote, cmd)))
|
||||
log.exception('Command "%s" failed. Could not convert the Jupyter Notebook to HTML, defaulting to plain text.', ' '.join(map(shlex.quote, cmd)))
|
||||
return open(ofilename, mode='rb')
|
||||
|
||||
def set_meta(self, dataset, **kwd):
|
||||
@@ -465,7 +464,7 @@ class Arff(Text):
|
||||
if not dataset.dataset.purged:
|
||||
dataset.peek = get_file_peek(dataset.file_name)
|
||||
dataset.blurb = "Attribute-Relation File Format (ARFF)"
|
||||
dataset.blurb += ", {} comments, {} attributes".format(dataset.metadata.comment_lines, dataset.metadata.columns)
|
||||
dataset.blurb += f", {dataset.metadata.comment_lines} comments, {dataset.metadata.columns} attributes"
|
||||
else:
|
||||
dataset.peek = 'file does not exist'
|
||||
dataset.blurb = 'file purged from disc'
|
||||
|
||||
@@ -38,7 +38,7 @@ class UCSCTrackHub(Html):
|
||||
opt_text = ''
|
||||
if composite_file.optional:
|
||||
opt_text = ' (optional)'
|
||||
rval.append('<li><a href="{}">{}</a>{}'.format(composite_name, composite_name, opt_text))
|
||||
rval.append(f'<li><a href="{composite_name}">{composite_name}</a>{opt_text}')
|
||||
rval.append('</ul></html>')
|
||||
return "\n".join(rval)
|
||||
|
||||
|
||||
@@ -16,7 +16,6 @@ from errno import EMFILE
|
||||
import bx.align.maf
|
||||
import bx.interval_index_file
|
||||
import bx.intervals
|
||||
from six.moves import xrange
|
||||
|
||||
try:
|
||||
maketrans = str.maketrans
|
||||
@@ -132,7 +131,7 @@ class TempFileHandler:
|
||||
self.files[index].flush()
|
||||
|
||||
def __del__(self):
|
||||
for i in xrange(len(self.files)):
|
||||
for i in range(len(self.files)):
|
||||
self.close(i, delete=True)
|
||||
|
||||
|
||||
@@ -295,7 +294,7 @@ def maf_index_by_uid(maf_uid, index_location_file):
|
||||
maf_files = fields[4].replace("\n", "").replace("\r", "").split(",")
|
||||
return bx.align.maf.MultiIndexed(maf_files, keep_open=True, parse_e_rows=False)
|
||||
except Exception as e:
|
||||
raise Exception('MAF UID ({}) found, but configuration appears to be malformed: {}'.format(maf_uid, e))
|
||||
raise Exception(f'MAF UID ({maf_uid}) found, but configuration appears to be malformed: {e}')
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
@@ -347,7 +346,7 @@ def build_maf_index_species_chromosomes(filename, index_species=None):
|
||||
indexes.add(c.src, forward_strand_start, forward_strand_end, pos, max=c.src_size)
|
||||
except Exception as e:
|
||||
# most likely a bad MAF
|
||||
log.debug('Building MAF index on {} failed: {}'.format(filename, e))
|
||||
log.debug(f'Building MAF index on {filename} failed: {e}')
|
||||
return (None, [], {}, 0)
|
||||
return (indexes, species, species_chromosomes, blocks)
|
||||
|
||||
@@ -427,7 +426,7 @@ def orient_block_by_region(block, src, region, force_strand=None):
|
||||
|
||||
|
||||
def get_oriented_chopped_blocks_for_region(index, src, region, species=None, mincols=0, force_strand=None):
|
||||
for block, idx, offset in get_oriented_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols, force_strand):
|
||||
for block, _, _ in get_oriented_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols, force_strand):
|
||||
yield block
|
||||
|
||||
|
||||
@@ -488,7 +487,7 @@ def iter_blocks_split_by_species(block, species=None):
|
||||
|
||||
# generator yielding only chopped and valid blocks for a specified region
|
||||
def get_chopped_blocks_for_region(index, src, region, species=None, mincols=0):
|
||||
for block, idx, offset in get_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols):
|
||||
for block, _, _ in get_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols):
|
||||
yield block
|
||||
|
||||
|
||||
@@ -512,7 +511,7 @@ def get_region_alignment(index, primary_species, chrom, start, end, strand='+',
|
||||
def reduce_block_by_primary_genome(block, species, chromosome, region_start):
|
||||
# returns ( startIndex, {species:texts}
|
||||
# where texts' contents are reduced to only positions existing in the primary genome
|
||||
src = "{}.{}".format(species, chromosome)
|
||||
src = f"{species}.{chromosome}"
|
||||
ref = block.get_component_by_src(src)
|
||||
start_offset = ref.start - region_start
|
||||
species_texts = {}
|
||||
@@ -533,7 +532,7 @@ def fill_region_alignment(alignment, index, primary_species, chrom, start, end,
|
||||
region = bx.intervals.Interval(start, end)
|
||||
region.chrom = chrom
|
||||
region.strand = strand
|
||||
primary_src = "{}.{}".format(primary_species, chrom)
|
||||
primary_src = f"{primary_species}.{chrom}"
|
||||
|
||||
# Order blocks overlaping this position by score, lowest first
|
||||
blocks = []
|
||||
@@ -684,9 +683,9 @@ def remove_temp_index_file(index_filename):
|
||||
def get_fasta_header(component, attributes={}, suffix=None):
|
||||
header = ">%s(%s):%i-%i|" % (component.src, component.strand, component.get_forward_strand_start(), component.get_forward_strand_end())
|
||||
for key, value in attributes.items():
|
||||
header = "{}{}={}|".format(header, key, value)
|
||||
header = f"{header}{key}={value}|"
|
||||
if suffix:
|
||||
header = "{}{}".format(header, suffix)
|
||||
header = f"{header}{suffix}"
|
||||
else:
|
||||
header = "{}{}".format(header, src_split(component.src)[0])
|
||||
return header
|
||||
|
||||
@@ -11,7 +11,6 @@ name = "pypi"
|
||||
[dev-packages]
|
||||
gunicorn = "*"
|
||||
lxml = "!=4.2.2"
|
||||
mock = "*"
|
||||
NoseHTML = "*"
|
||||
pygithub = "*"
|
||||
pytest = "*"
|
||||
@@ -61,7 +60,6 @@ Beaker = "*"
|
||||
dictobj = "*"
|
||||
nose = "*"
|
||||
Parsley = "*"
|
||||
six = "*"
|
||||
sortedcontainers = "*"
|
||||
Whoosh = "*"
|
||||
galaxy_sequence_utils = "*"
|
||||
@@ -88,4 +86,4 @@ gxformat2 = "*"
|
||||
refgenconf = ">=0.7.0"
|
||||
|
||||
[requires]
|
||||
python_version = "3.5"
|
||||
python_version = "3.6"
|
||||
|
||||
@@ -12,7 +12,7 @@ log = logging.getLogger(__name__)
|
||||
FileSourcePath = namedtuple('FileSourcePath', ['file_source', 'path'])
|
||||
|
||||
|
||||
class ConfiguredFileSources(object):
|
||||
class ConfiguredFileSources:
|
||||
"""Load plugins and resolve Galaxy URIs to FileSource objects."""
|
||||
|
||||
def __init__(self, file_sources_config, conf_file=None, conf_dict=None, load_stock_plugins=False):
|
||||
@@ -161,7 +161,7 @@ class ConfiguredFileSources(object):
|
||||
return ConfiguredFileSources(file_sources_config, conf_dict=sources_as_dict)
|
||||
|
||||
|
||||
class ConfiguredFileSourcesConfig(object):
|
||||
class ConfiguredFileSourcesConfig:
|
||||
|
||||
def __init__(self, symlink_allowlist=[], library_import_dir=None, user_library_import_dir=None, ftp_upload_dir=None, ftp_upload_purge=True):
|
||||
self.symlink_allowlist = symlink_allowlist
|
||||
@@ -202,7 +202,7 @@ class ConfiguredFileSourcesConfig(object):
|
||||
)
|
||||
|
||||
|
||||
class ProvidesUserFileSourcesUserContext(object):
|
||||
class ProvidesUserFileSourcesUserContext:
|
||||
"""Implement a FileSourcesUserContext from a Galaxy ProvidesUserContext (e.g. trans)."""
|
||||
|
||||
def __init__(self, trans):
|
||||
@@ -228,7 +228,7 @@ class ProvidesUserFileSourcesUserContext(object):
|
||||
return user and user.extra_preferences
|
||||
|
||||
|
||||
class DictFileSourcesUserContext(object):
|
||||
class DictFileSourcesUserContext:
|
||||
|
||||
def __init__(self, **kwd):
|
||||
self._kwd = kwd
|
||||
|
||||
@@ -2,16 +2,13 @@ import abc
|
||||
import os
|
||||
import time
|
||||
|
||||
import six
|
||||
|
||||
from galaxy.util.template import fill_template
|
||||
|
||||
DEFAULT_SCHEME = "gxfiles"
|
||||
DEFAULT_WRITABLE = False
|
||||
|
||||
|
||||
@six.add_metaclass(abc.ABCMeta)
|
||||
class FilesSource(object):
|
||||
class FilesSource(metaclass=abc.ABCMeta):
|
||||
"""
|
||||
"""
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ class UserFtpFilesSource(PosixFilesSource):
|
||||
if "delete_on_realize" not in posix_kwds:
|
||||
file_sources_config = kwd.get("file_sources_config")
|
||||
posix_kwds["delete_on_realize"] = file_sources_config.ftp_upload_purge
|
||||
super(UserFtpFilesSource, self).__init__(**posix_kwds)
|
||||
super().__init__(**posix_kwds)
|
||||
|
||||
def get_prefix(self):
|
||||
return None
|
||||
@@ -38,7 +38,7 @@ class LibraryImportFilesSource(PosixFilesSource):
|
||||
doc=doc,
|
||||
)
|
||||
posix_kwds.update(kwd)
|
||||
super(LibraryImportFilesSource, self).__init__(**posix_kwds)
|
||||
super().__init__(**posix_kwds)
|
||||
|
||||
def get_prefix(self):
|
||||
return None
|
||||
@@ -58,7 +58,7 @@ class UserLibraryImportFilesSource(PosixFilesSource):
|
||||
doc=doc,
|
||||
)
|
||||
posix_kwds.update(kwd)
|
||||
super(UserLibraryImportFilesSource, self).__init__(**posix_kwds)
|
||||
super().__init__(**posix_kwds)
|
||||
|
||||
def get_prefix(self):
|
||||
return None
|
||||
|
||||
@@ -21,7 +21,7 @@ class FormDefinitionFactory:
|
||||
"""
|
||||
Return new FormDefinition.
|
||||
"""
|
||||
assert form_type in self.form_types, 'Invalid FormDefinition type ( {} not in {} )'.format(form_type, self.form_types.keys())
|
||||
assert form_type in self.form_types, f'Invalid FormDefinition type ( {form_type} not in {self.form_types.keys()} )'
|
||||
assert name, 'FormDefinition requires a name'
|
||||
if description is None:
|
||||
description = ''
|
||||
@@ -113,7 +113,7 @@ class FormDefinitionFieldFactory:
|
||||
visible = string_as_bool(elem.get('visible', 'true'))
|
||||
field_layout = elem.get('layout', None)
|
||||
if field_layout:
|
||||
assert layout and field_layout in layout, 'Invalid layout specified: {} not in {}'.format(field_layout, layout)
|
||||
assert layout and field_layout in layout, f'Invalid layout specified: {field_layout} not in {layout}'
|
||||
field_layout = str(layout.index(field_layout)) # existing behavior: integer indexes are stored as strings. why?
|
||||
return self.new(name=name, label=label, required=required, helptext=helptext, default=default, visible=visible, layout=field_layout)
|
||||
|
||||
|
||||
@@ -386,7 +386,7 @@ def collect_primary_datasets(job_context, output, input_ext):
|
||||
if dbkey == INPUT_DBKEY_TOKEN:
|
||||
dbkey = job_context.input_dbkey
|
||||
if filename_index == 0 and extra_file_collector.assign_primary_output and output_index == 0:
|
||||
new_outdata_name = fields_match.name or "{} ({})".format(outdata.name, designation)
|
||||
new_outdata_name = fields_match.name or f"{outdata.name} ({designation})"
|
||||
outdata.change_datatype(ext)
|
||||
outdata.dbkey = dbkey
|
||||
outdata.designation = designation
|
||||
@@ -399,7 +399,7 @@ def collect_primary_datasets(job_context, output, input_ext):
|
||||
primary_datasets[name] = OrderedDict()
|
||||
visible = fields_match.visible
|
||||
# Create new primary dataset
|
||||
new_primary_name = fields_match.name or "{} ({})".format(outdata.name, designation)
|
||||
new_primary_name = fields_match.name or f"{outdata.name} ({designation})"
|
||||
info = outdata.info
|
||||
|
||||
# TODO: should be able to disambiguate files in different directories...
|
||||
@@ -418,7 +418,7 @@ def collect_primary_datasets(job_context, output, input_ext):
|
||||
dataset_attributes=new_primary_datasets_attributes,
|
||||
)
|
||||
# Associate new dataset with job
|
||||
job_context.add_output_dataset_association('__new_primary_file_{}|{}__'.format(name, designation), primary_data)
|
||||
job_context.add_output_dataset_association(f'__new_primary_file_{name}|{designation}__', primary_data)
|
||||
|
||||
if new_primary_datasets_attributes:
|
||||
extra_files_path = new_primary_datasets_attributes.get('extra_files', None)
|
||||
@@ -577,7 +577,7 @@ def read_exit_code_from(exit_code_file, id_tag):
|
||||
exit_code = int(exit_code_str)
|
||||
except ValueError:
|
||||
galaxy_id_tag = id_tag
|
||||
log.warning("({}) Exit code '{}' invalid. Using 0.".format(galaxy_id_tag, exit_code_str))
|
||||
log.warning(f"({galaxy_id_tag}) Exit code '{exit_code_str}' invalid. Using 0.")
|
||||
exit_code = 0
|
||||
|
||||
return exit_code
|
||||
|
||||
@@ -110,13 +110,13 @@ class CollectlCli:
|
||||
if not interval2:
|
||||
return interval_arg
|
||||
self.__validate_interval_arg(interval2, multiple_of=int(interval))
|
||||
interval_arg = "{}:{}".format(interval_arg, interval2)
|
||||
interval_arg = f"{interval_arg}:{interval2}"
|
||||
|
||||
interval3 = kwargs.get("interval3", None)
|
||||
if not interval3:
|
||||
return interval_arg
|
||||
self.__validate_interval_arg(interval3, multiple_of=int(interval))
|
||||
interval_arg = "{}:{}".format(interval_arg, interval3)
|
||||
interval_arg = f"{interval_arg}:{interval3}"
|
||||
return interval_arg
|
||||
|
||||
def __validate_interval_arg(self, value, multiple_of=None):
|
||||
|
||||
@@ -49,7 +49,7 @@ class InstrumentPlugin(metaclass=ABCMeta):
|
||||
""" Provide a common pattern for naming files used by instrumentation
|
||||
plugins - to ease their staging out of remote job directories.
|
||||
"""
|
||||
return "{}_{}_{}".format(INSTRUMENT_FILE_PREFIX, self.plugin_type, name)
|
||||
return f"{INSTRUMENT_FILE_PREFIX}_{self.plugin_type}_{name}"
|
||||
|
||||
def _instrument_file_path(self, job_directory, name):
|
||||
return os.path.join(job_directory, self._instrument_file_name(name))
|
||||
|
||||
@@ -48,7 +48,7 @@ class CollectlFormatter(formatting.JobMetricFormatter):
|
||||
else:
|
||||
value_str = str(value)
|
||||
resource_title = FORMATTED_RESOURCE_TITLES.get(resource_type, resource_type)
|
||||
return ("{} ({})".format(resource_title, stat_type), value_str)
|
||||
return (f"{resource_title} ({stat_type})", value_str)
|
||||
|
||||
|
||||
class CollectlPlugin(InstrumentPlugin):
|
||||
@@ -102,7 +102,7 @@ class CollectlPlugin(InstrumentPlugin):
|
||||
rel_path = filter(self._is_instrumented_collectl_log, contents)[0]
|
||||
path = os.path.join(job_directory, rel_path)
|
||||
except IndexError:
|
||||
message = "Failed to find collectl log in directory {}, files were {}".format(job_directory, contents)
|
||||
message = f"Failed to find collectl log in directory {job_directory}, files were {contents}"
|
||||
raise Exception(message)
|
||||
|
||||
properties = dict(
|
||||
|
||||
@@ -51,7 +51,7 @@ class CpuInfoPlugin(InstrumentPlugin):
|
||||
# If verbose, dump information about each processor
|
||||
# into database...
|
||||
key, value = line.split(":", 1)
|
||||
key = "processor_{}_{}".format(current_processor, key.strip())
|
||||
key = f"processor_{current_processor}_{key.strip()}"
|
||||
value = value
|
||||
properties["processor_count"] = processor_count
|
||||
return properties
|
||||
|
||||
+15
-15
@@ -803,7 +803,7 @@ class JobConfiguration(ConfiguresHandlers):
|
||||
log.warning("A non-class name was found in __all__, ignoring: %s" % id)
|
||||
continue
|
||||
except AssertionError:
|
||||
log.warning("Job runner classes must be subclassed from BaseJobRunner, {} has bases: {}".format(id, runner_class.__bases__))
|
||||
log.warning(f"Job runner classes must be subclassed from BaseJobRunner, {id} has bases: {runner_class.__bases__}")
|
||||
continue
|
||||
try:
|
||||
rval[id] = runner_class(self.app, runner.get('workers', JobConfiguration.DEFAULT_NWORKERS), **runner.get('kwds', {}))
|
||||
@@ -811,7 +811,7 @@ class JobConfiguration(ConfiguresHandlers):
|
||||
log.exception("Job runner '%s:%s' has not been converted to a new-style runner or encountered TypeError on load",
|
||||
module_name, class_name)
|
||||
rval[id] = runner_class(self.app)
|
||||
log.debug("Loaded job runner '{}:{}' as '{}'".format(module_name, class_name, id))
|
||||
log.debug(f"Loaded job runner '{module_name}:{class_name}' as '{id}'")
|
||||
return rval
|
||||
|
||||
def is_id(self, collection):
|
||||
@@ -847,13 +847,13 @@ class JobConfiguration(ConfiguresHandlers):
|
||||
destination.params = job_runners[destination.runner].url_to_destination(destination.url).params
|
||||
destination.converted = True
|
||||
if destination.params:
|
||||
log.debug("Legacy destination with id '{}', url '{}' converted, got params:".format(id, destination.url))
|
||||
log.debug(f"Legacy destination with id '{id}', url '{destination.url}' converted, got params:")
|
||||
for k, v in destination.params.items():
|
||||
log.debug(" {}: {}".format(k, v))
|
||||
log.debug(f" {k}: {v}")
|
||||
else:
|
||||
log.debug("Legacy destination with id '{}', url '{}' converted, got params:".format(id, destination.url))
|
||||
log.debug(f"Legacy destination with id '{id}', url '{destination.url}' converted, got params:")
|
||||
else:
|
||||
log.warning("Legacy destination with id '{}' could not be converted: Unknown runner plugin: {}".format(id, destination.runner))
|
||||
log.warning(f"Legacy destination with id '{id}' could not be converted: Unknown runner plugin: {destination.runner}")
|
||||
|
||||
|
||||
class HasResourceParameters:
|
||||
@@ -1129,7 +1129,7 @@ class JobWrapper(HasResourceParameters):
|
||||
if version_string_cmd_raw:
|
||||
version_command_template = string.Template(version_string_cmd_raw)
|
||||
version_string_cmd = version_command_template.safe_substitute({"__tool_directory__": compute_environment.tool_directory()})
|
||||
self.write_version_cmd = "{} > {} 2>&1".format(version_string_cmd, compute_environment.version_path())
|
||||
self.write_version_cmd = f"{version_string_cmd} > {compute_environment.version_path()} 2>&1"
|
||||
else:
|
||||
self.write_version_cmd = None
|
||||
return self.extra_filenames
|
||||
@@ -1412,7 +1412,7 @@ class JobWrapper(HasResourceParameters):
|
||||
"""
|
||||
if job is None:
|
||||
job = self.get_job()
|
||||
log.debug('({}) Persisting job destination (destination id: {})'.format(job.id, job_destination.id))
|
||||
log.debug(f'({job.id}) Persisting job destination (destination id: {job_destination.id})')
|
||||
job.destination_id = job_destination.id
|
||||
job.destination_params = job_destination.params
|
||||
job.job_runner_name = job_destination.runner
|
||||
@@ -1594,7 +1594,7 @@ class JobWrapper(HasResourceParameters):
|
||||
try:
|
||||
self.reclaim_ownership()
|
||||
except Exception:
|
||||
log.exception('({}) Failed to change ownership of {}, failing'.format(job.id, self.working_directory))
|
||||
log.exception(f'({job.id}) Failed to change ownership of {self.working_directory}, failing')
|
||||
return fail()
|
||||
|
||||
# if the job was deleted, don't finish it
|
||||
@@ -1641,7 +1641,7 @@ class JobWrapper(HasResourceParameters):
|
||||
for dataset_path in self.get_output_fnames():
|
||||
try:
|
||||
shutil.move(dataset_path.false_path, dataset_path.real_path)
|
||||
log.debug("finish(): Moved {} to {}".format(dataset_path.false_path, dataset_path.real_path))
|
||||
log.debug(f"finish(): Moved {dataset_path.false_path} to {dataset_path.real_path}")
|
||||
except OSError:
|
||||
# this can happen if Galaxy is restarted during the job's
|
||||
# finish method - the false_path file has already moved,
|
||||
@@ -1661,7 +1661,7 @@ class JobWrapper(HasResourceParameters):
|
||||
import_model_store = store.get_import_model_store_for_directory(os.path.join(self.working_directory, 'metadata', 'outputs_populated'), app=self.app, import_options=import_options)
|
||||
import_model_store.perform_import(history=job.history)
|
||||
except Exception:
|
||||
log.exception("problem importing job outputs. stdout [{}] stderr [{}]".format(job.stdout, job.stderr))
|
||||
log.exception(f"problem importing job outputs. stdout [{job.stdout}] stderr [{job.stderr}]")
|
||||
raise
|
||||
output_dataset_associations = job.output_datasets + job.output_library_datasets
|
||||
for dataset_assoc in output_dataset_associations:
|
||||
@@ -1883,14 +1883,14 @@ class JobWrapper(HasResourceParameters):
|
||||
def get_env_setup_clause(self):
|
||||
if self.app.config.environment_setup_file is None:
|
||||
return ''
|
||||
return '[ -f "{}" ] && . {}'.format(self.app.config.environment_setup_file, self.app.config.environment_setup_file)
|
||||
return f'[ -f "{self.app.config.environment_setup_file}" ] && . {self.app.config.environment_setup_file}'
|
||||
|
||||
def get_input_dataset_fnames(self, ds):
|
||||
filenames = []
|
||||
filenames = [ds.file_name]
|
||||
# we will need to stage in metadata file names also
|
||||
# TODO: would be better to only stage in metadata files that are actually needed (found in command line, referenced in config files, etc.)
|
||||
for key, value in ds.metadata.items():
|
||||
for _, value in ds.metadata.items():
|
||||
if isinstance(value, model.MetadataFile):
|
||||
filenames.append(value.file_name)
|
||||
return filenames
|
||||
@@ -1938,7 +1938,7 @@ class JobWrapper(HasResourceParameters):
|
||||
for (hda, dataset_path) in self.output_hdas_and_paths.values():
|
||||
if hda == dataset:
|
||||
return dataset_path
|
||||
raise KeyError("Couldn't find job output for [{}] in [{}]".format(dataset, self.output_hdas_and_paths.values()))
|
||||
raise KeyError(f"Couldn't find job output for [{dataset}] in [{self.output_hdas_and_paths.values()}]")
|
||||
|
||||
def get_mutable_output_fnames(self):
|
||||
if self.output_paths is None:
|
||||
@@ -2100,7 +2100,7 @@ class JobWrapper(HasResourceParameters):
|
||||
dependency_shell_commands = metadata_tool.build_dependency_shell_commands(job_directory=self.working_directory, metadata=True)
|
||||
if dependency_shell_commands:
|
||||
dependency_shell_commands = "; ".join(dependency_shell_commands)
|
||||
command = "{}; {}".format(dependency_shell_commands, command)
|
||||
command = f"{dependency_shell_commands}; {command}"
|
||||
return command
|
||||
|
||||
def check_for_entry_points(self, check_already_configured=True):
|
||||
|
||||
@@ -364,14 +364,14 @@ class DeleteIntermediatesAction(DefaultJobAction):
|
||||
if wfi_step_job:
|
||||
jobs_to_check.append(wfi_step_job)
|
||||
else:
|
||||
log.debug("No job found yet for wfi_step {}, (step {})".format(wfi_step, wfi_step.workflow_step))
|
||||
log.debug(f"No job found yet for wfi_step {wfi_step}, (step {wfi_step.workflow_step})")
|
||||
for j2c in jobs_to_check:
|
||||
creating_jobs = []
|
||||
for input_dataset in j2c.input_datasets:
|
||||
if not input_dataset.dataset:
|
||||
log.debug("PJA Async Issue: No dataset attached to input_dataset {} during handling of workflow invocation {}".format(input_dataset.id, wfi))
|
||||
log.debug(f"PJA Async Issue: No dataset attached to input_dataset {input_dataset.id} during handling of workflow invocation {wfi}")
|
||||
elif not input_dataset.dataset.creating_job:
|
||||
log.debug("PJA Async Issue: No creating job attached to dataset {} during handling of workflow invocation {}".format(input_dataset.dataset.id, wfi))
|
||||
log.debug(f"PJA Async Issue: No creating job attached to dataset {input_dataset.dataset.id} during handling of workflow invocation {wfi}")
|
||||
else:
|
||||
creating_jobs.append((input_dataset, input_dataset.dataset.creating_job))
|
||||
for (input_dataset, creating_job) in creating_jobs:
|
||||
@@ -383,7 +383,7 @@ class DeleteIntermediatesAction(DefaultJobAction):
|
||||
safe_to_delete = True
|
||||
for job_to_check in [d_j.job for d_j in input_dataset.dependent_jobs]:
|
||||
if job_to_check != job and job_to_check.state not in [job.states.OK, job.states.DELETED]:
|
||||
log.trace("Workflow Intermediates cleanup attempted, but non-terminal state '{}' detected for job {}".format(job_to_check.state, job_to_check.id))
|
||||
log.trace(f"Workflow Intermediates cleanup attempted, but non-terminal state '{job_to_check.state}' detected for job {job_to_check.id}")
|
||||
safe_to_delete = False
|
||||
if safe_to_delete:
|
||||
# Support purging here too.
|
||||
|
||||
@@ -155,7 +155,7 @@ def __externalize_commands(job_wrapper, shell, commands_builder, remote_command_
|
||||
tool_commands,
|
||||
)
|
||||
write_script(local_container_script, script_contents, config)
|
||||
commands = "{} {}".format(shell, local_container_script)
|
||||
commands = f"{shell} {local_container_script}"
|
||||
# TODO: Cleanup for_pulsar hack.
|
||||
# - Integrate Pulsar sending tool_stdout/tool_stderr back
|
||||
# https://github.com/galaxyproject/pulsar/pull/202
|
||||
@@ -170,7 +170,7 @@ def __externalize_commands(job_wrapper, shell, commands_builder, remote_command_
|
||||
for_pulsar = True
|
||||
if not for_pulsar:
|
||||
commands += " > ../outputs/tool_stdout 2> ../outputs/tool_stderr"
|
||||
log.info("Built script [{}] for tool command [{}]".format(local_container_script, tool_commands))
|
||||
log.info(f"Built script [{local_container_script}] for tool command [{tool_commands}]")
|
||||
return commands
|
||||
|
||||
|
||||
@@ -236,14 +236,14 @@ def __handle_metadata(commands_builder, job_wrapper, runner, remote_command_para
|
||||
metadata_command = metadata_command.strip()
|
||||
if metadata_command:
|
||||
# Place Galaxy and its dependencies in environment for metadata regardless of tool.
|
||||
metadata_command = "{}{}".format(SETUP_GALAXY_FOR_METADATA, metadata_command)
|
||||
metadata_command = f"{SETUP_GALAXY_FOR_METADATA}{metadata_command}"
|
||||
commands_builder.capture_return_code()
|
||||
commands_builder.append_command(metadata_command)
|
||||
|
||||
|
||||
def __copy_if_exists_command(work_dir_output):
|
||||
source_file, destination = work_dir_output
|
||||
return "if [ -f {} ] ; then cp {} {} ; fi".format(source_file, source_file, destination)
|
||||
return f"if [ -f {source_file} ] ; then cp {source_file} {destination} ; fi"
|
||||
|
||||
|
||||
class CommandsBuilder:
|
||||
|
||||
+14
-14
@@ -5,11 +5,11 @@ import datetime
|
||||
import os
|
||||
import time
|
||||
from collections import defaultdict
|
||||
|
||||
from six.moves.queue import (
|
||||
from queue import (
|
||||
Empty,
|
||||
Queue
|
||||
Queue,
|
||||
)
|
||||
|
||||
from sqlalchemy.exc import OperationalError
|
||||
from sqlalchemy.sql.expression import (
|
||||
and_,
|
||||
@@ -217,7 +217,7 @@ class JobHandlerQueue(Monitors):
|
||||
|
||||
for job in jobs_at_startup:
|
||||
if not self.app.toolbox.has_tool(job.tool_id, job.tool_version, exact=True):
|
||||
log.warning("({}) Tool '{}' removed from tool config, unable to recover job".format(job.id, job.tool_id))
|
||||
log.warning(f"({job.id}) Tool '{job.tool_id}' removed from tool config, unable to recover job")
|
||||
self.job_wrapper(job).fail('This tool was disabled before the job completed. Please contact your Galaxy administrator.')
|
||||
elif job.job_runner_name is not None and job.job_runner_external_id is None:
|
||||
# This could happen during certain revisions of Galaxy where a runner URL was persisted before the job was dispatched to a runner.
|
||||
@@ -238,7 +238,7 @@ class JobHandlerQueue(Monitors):
|
||||
log.info('(%s) Converted job from a URL to a destination and recovered' % (job.id))
|
||||
elif job.job_runner_name is None:
|
||||
# Never (fully) dispatched
|
||||
log.debug("({}) No job runner assigned and job still in '{}' state, adding to the job handler queue".format(job.id, job.state))
|
||||
log.debug(f"({job.id}) No job runner assigned and job still in '{job.state}' state, adding to the job handler queue")
|
||||
if self.track_jobs_in_database:
|
||||
job.set_state(model.Job.states.NEW)
|
||||
else:
|
||||
@@ -486,7 +486,7 @@ class JobHandlerQueue(Monitors):
|
||||
elif dataset_state == model.Dataset.states.ERROR:
|
||||
jobs_to_pause[job_id].append("Input dataset '%s' is in error state" % hda_name)
|
||||
elif dataset_state != model.Dataset.states.OK:
|
||||
jobs_to_ignore[job_id].append("Input dataset '{}' is in {} state".format(hda_name, dataset_state))
|
||||
jobs_to_ignore[job_id].append(f"Input dataset '{hda_name}' is in {dataset_state} state")
|
||||
for job_id in sorted(jobs_to_pause):
|
||||
pause_message = ", ".join(jobs_to_pause[job_id])
|
||||
pause_message = "%s. To resume this job fix the input dataset(s)." % pause_message
|
||||
@@ -557,7 +557,7 @@ class JobHandlerQueue(Monitors):
|
||||
# Cause the job_destination to be set and cached by the mapper
|
||||
job_destination = job_wrapper.job_destination
|
||||
except AssertionError as e:
|
||||
log.warning("({}) Tool '{}' removed from tool config, unable to run job".format(job.id, job.tool_id))
|
||||
log.warning(f"({job.id}) Tool '{job.tool_id}' removed from tool config, unable to run job")
|
||||
job_wrapper.fail(e)
|
||||
return JOB_ERROR, job_destination
|
||||
except JobNotReadyException as e:
|
||||
@@ -631,7 +631,7 @@ class JobHandlerQueue(Monitors):
|
||||
continue
|
||||
# don't run jobs for which the input dataset was deleted
|
||||
if idata.deleted:
|
||||
self.job_wrappers.pop(job.id, self.job_wrapper(job)).fail("input data {} (file: {}) was deleted before the job started".format(idata.hid, idata.file_name))
|
||||
self.job_wrappers.pop(job.id, self.job_wrapper(job)).fail(f"input data {idata.hid} (file: {idata.file_name}) was deleted before the job started")
|
||||
return JOB_INPUT_DELETED
|
||||
# an error in the input data causes us to bail immediately
|
||||
elif idata.state == idata.states.ERROR:
|
||||
@@ -1000,12 +1000,12 @@ class DefaultJobDispatcher:
|
||||
try:
|
||||
if isinstance(job_wrapper, TaskWrapper):
|
||||
# DBTODO Refactor
|
||||
log.debug("({}) Dispatching task {} to {} runner".format(job_wrapper.job_id, job_wrapper.task_id, runner_name))
|
||||
log.debug(f"({job_wrapper.job_id}) Dispatching task {job_wrapper.task_id} to {runner_name} runner")
|
||||
else:
|
||||
log.debug("({}) Dispatching to {} runner".format(job_wrapper.job_id, runner_name))
|
||||
log.debug(f"({job_wrapper.job_id}) Dispatching to {runner_name} runner")
|
||||
self.job_runners[runner_name].put(job_wrapper)
|
||||
except KeyError:
|
||||
log.error('put(): ({}) Invalid job runner: {}'.format(job_wrapper.job_id, runner_name))
|
||||
log.error(f'put(): ({job_wrapper.job_id}) Invalid job runner: {runner_name}')
|
||||
job_wrapper.fail(DEFAULT_JOB_PUT_FAILURE_MESSAGE)
|
||||
|
||||
def stop(self, job, job_wrapper):
|
||||
@@ -1023,11 +1023,11 @@ class DefaultJobDispatcher:
|
||||
job_runner_name = job.get_job_runner_name()
|
||||
if job_runner_name is not None:
|
||||
runner_name = job_runner_name.split(":", 1)[0]
|
||||
log.debug("Stopping job {} in {} runner".format(job_wrapper.get_id_tag(), runner_name))
|
||||
log.debug(f"Stopping job {job_wrapper.get_id_tag()} in {runner_name} runner")
|
||||
try:
|
||||
self.job_runners[runner_name].stop_job(job_wrapper)
|
||||
except KeyError:
|
||||
log.error('stop(): ({}) Invalid job runner: {}'.format(job_wrapper.get_id_tag(), runner_name))
|
||||
log.error(f'stop(): ({job_wrapper.get_id_tag()}) Invalid job runner: {runner_name}')
|
||||
# Job and output dataset states have already been updated, so nothing is done here.
|
||||
|
||||
def recover(self, job, job_wrapper):
|
||||
@@ -1036,7 +1036,7 @@ class DefaultJobDispatcher:
|
||||
try:
|
||||
self.job_runners[runner_name].recover(job, job_wrapper)
|
||||
except KeyError:
|
||||
log.error('recover(): ({}) Invalid job runner: {}'.format(job_wrapper.job_id, runner_name))
|
||||
log.error(f'recover(): ({job_wrapper.job_id}) Invalid job runner: {runner_name}')
|
||||
job_wrapper.fail(DEFAULT_JOB_PUT_FAILURE_MESSAGE)
|
||||
|
||||
def shutdown(self):
|
||||
|
||||
@@ -187,7 +187,7 @@ class RuleHelper:
|
||||
invocation for jobs outside of workflows.
|
||||
"""
|
||||
if hash_by not in VALID_JOB_HASH_STRATEGIES:
|
||||
message = "Do not know how to hash jobs by {}, must be one of {}".format(hash_by, VALID_JOB_HASH_STRATEGIES)
|
||||
message = f"Do not know how to hash jobs by {hash_by}, must be one of {VALID_JOB_HASH_STRATEGIES}"
|
||||
raise Exception(message)
|
||||
|
||||
if hash_by == "workflow_invocation":
|
||||
|
||||
@@ -9,10 +9,9 @@ import sys
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
|
||||
from six.moves.queue import (
|
||||
from queue import (
|
||||
Empty,
|
||||
Queue
|
||||
Queue,
|
||||
)
|
||||
|
||||
import galaxy.jobs
|
||||
@@ -90,7 +89,7 @@ class BaseJobRunner:
|
||||
"""
|
||||
self.work_queue = Queue()
|
||||
self.work_threads = []
|
||||
log.debug('Starting {} {} workers'.format(self.nworkers, self.runner_name))
|
||||
log.debug(f'Starting {self.nworkers} {self.runner_name} workers')
|
||||
for i in range(self.nworkers):
|
||||
worker = threading.Thread(name="%s.work_thread-%d" % (self.runner_name, i), target=self.run_next)
|
||||
worker.daemon = True
|
||||
@@ -129,7 +128,7 @@ class BaseJobRunner:
|
||||
except Exception:
|
||||
name = 'unknown'
|
||||
try:
|
||||
action_str = 'galaxy.jobs.runners.{}.{}'.format(self.__class__.__name__.lower(), name)
|
||||
action_str = f'galaxy.jobs.runners.{self.__class__.__name__.lower()}.{name}'
|
||||
action_timer = self.app.execution_timer_factory.get_timer(
|
||||
'internals.%s' % action_str,
|
||||
'job runner action %s for job ${job_id} executed' % (action_str)
|
||||
@@ -137,7 +136,7 @@ class BaseJobRunner:
|
||||
method(arg)
|
||||
log.trace(action_timer.to_str(job_id=job_id))
|
||||
except Exception:
|
||||
log.exception("({}) Unhandled exception calling {}".format(job_id, name))
|
||||
log.exception(f"({job_id}) Unhandled exception calling {name}")
|
||||
if not isinstance(arg, JobState):
|
||||
job_state = JobState(job_wrapper=arg, job_destination={})
|
||||
else:
|
||||
@@ -151,7 +150,7 @@ class BaseJobRunner:
|
||||
put_timer = ExecutionTimer()
|
||||
job_wrapper.enqueue()
|
||||
self.mark_as_queued(job_wrapper)
|
||||
log.debug("Job [{}] queued {}".format(job_wrapper.job_id, put_timer))
|
||||
log.debug(f"Job [{job_wrapper.job_id}] queued {put_timer}")
|
||||
|
||||
def mark_as_queued(self, job_wrapper):
|
||||
self.work_queue.put((self.queue_job, job_wrapper))
|
||||
@@ -222,12 +221,12 @@ class BaseJobRunner:
|
||||
|
||||
# Make sure the job hasn't been deleted
|
||||
if job_state == model.Job.states.DELETED:
|
||||
log.debug("({}) Job deleted by user before it entered the {} queue".format(job_id, self.runner_name))
|
||||
log.debug(f"({job_id}) Job deleted by user before it entered the {self.runner_name} queue")
|
||||
if self.app.config.cleanup_job in ("always", "onsuccess"):
|
||||
job_wrapper.cleanup()
|
||||
return False
|
||||
elif job_state != model.Job.states.QUEUED:
|
||||
log.info("({}) Job is in state {}, skipping execution".format(job_id, job_state))
|
||||
log.info(f"({job_id}) Job is in state {job_state}, skipping execution")
|
||||
# cleanup may not be safe in all states
|
||||
return False
|
||||
|
||||
@@ -352,13 +351,13 @@ class BaseJobRunner:
|
||||
tmp_dir=job_wrapper.working_directory,
|
||||
# We don't want to overwrite metadata that was copied over in init_meta(), as per established behavior
|
||||
kwds={'overwrite' : False})
|
||||
external_metadata_script = "{} {} {}".format(lib_adjust, venv, external_metadata_script)
|
||||
external_metadata_script = f"{lib_adjust} {venv} {external_metadata_script}"
|
||||
if resolve_requirements:
|
||||
dependency_shell_commands = self.app.datatypes_registry.set_external_metadata_tool.build_dependency_shell_commands(job_directory=job_wrapper.working_directory)
|
||||
if dependency_shell_commands:
|
||||
if isinstance(dependency_shell_commands, list):
|
||||
dependency_shell_commands = "&&".join(dependency_shell_commands)
|
||||
external_metadata_script = "{}&&{}".format(dependency_shell_commands, external_metadata_script)
|
||||
external_metadata_script = f"{dependency_shell_commands}&&{external_metadata_script}"
|
||||
log.debug('executing external set_meta script for job %d: %s' % (job_wrapper.job_id, external_metadata_script))
|
||||
external_metadata_proc = subprocess.Popen(args=external_metadata_script,
|
||||
shell=True,
|
||||
@@ -396,7 +395,7 @@ class BaseJobRunner:
|
||||
# Additional logging to enable if debugging from_work_dir handling, metadata
|
||||
# commands, etc... (or just peak in the job script.)
|
||||
job_id = job_wrapper.job_id
|
||||
log.debug('({}) command is: {}'.format(job_id, command_line))
|
||||
log.debug(f'({job_id}) command is: {command_line}')
|
||||
options.update(**kwds)
|
||||
return job_script(**options)
|
||||
|
||||
@@ -600,7 +599,7 @@ class JobState:
|
||||
if not hasattr(self, "job_id"):
|
||||
prefix = "(%s)" % self.job_wrapper.get_id_tag()
|
||||
else:
|
||||
prefix = "({}/{})".format(self.job_wrapper.get_id_tag(), self.job_id)
|
||||
prefix = f"({self.job_wrapper.get_id_tag()}/{self.job_id})"
|
||||
log.debug("{} Unable to cleanup {}: {}".format(prefix, file, unicodify(e)))
|
||||
|
||||
|
||||
|
||||
@@ -196,7 +196,7 @@ class ChronosJobRunner(AsynchronousJobRunner):
|
||||
msg = 'Job {name!r} failed more than {retries!s} times'
|
||||
reason = msg.format(name=job_name, retries=str(max_retries))
|
||||
return self._mark_as_failed(job_state, reason)
|
||||
reason = 'Job {name!r} not found'.format(name=job_name)
|
||||
reason = f'Job {job_name!r} not found'
|
||||
return self._mark_as_failed(job_state, reason)
|
||||
|
||||
def _mark_as_successful(self, job_state):
|
||||
@@ -264,7 +264,7 @@ class ChronosJobRunner(AsynchronousJobRunner):
|
||||
jobs = self._chronos_client.list()
|
||||
job = [x for x in jobs if x['name'] == job_id]
|
||||
if len(job) > 1:
|
||||
msg = 'Multiple jobs found with name {name!r}'.format(name=job_id)
|
||||
msg = f'Multiple jobs found with name {job_id!r}'
|
||||
LOGGER.error(msg)
|
||||
raise ChronosRunnerException(msg)
|
||||
return job[0] if job else None
|
||||
|
||||
@@ -47,7 +47,7 @@ class ShellJobRunner(AsynchronousJobRunner):
|
||||
job_params = {'job_' + k: v for k, v in [kv.split('=', 1) for kv in job_params.split('&')]}
|
||||
params.update(shell_params)
|
||||
params.update(job_params)
|
||||
log.debug("Converted URL '{}' to destination runner=cli, params={}".format(url, params))
|
||||
log.debug(f"Converted URL '{url}' to destination runner=cli, params={params}")
|
||||
# Create a dynamic JobDestination
|
||||
return JobDestination(runner='cli', params=params)
|
||||
|
||||
@@ -94,7 +94,7 @@ class ShellJobRunner(AsynchronousJobRunner):
|
||||
job_wrapper.cleanup()
|
||||
return
|
||||
|
||||
log.debug("({}) submitting file: {}".format(galaxy_id_tag, ajs.job_file))
|
||||
log.debug(f"({galaxy_id_tag}) submitting file: {ajs.job_file}")
|
||||
|
||||
returncode, stdout = self.submit(shell, job_interface, ajs.job_file, galaxy_id_tag, retry=MAX_SUBMIT_RETRY)
|
||||
if returncode != 0:
|
||||
@@ -108,7 +108,7 @@ class ShellJobRunner(AsynchronousJobRunner):
|
||||
job_wrapper.fail("failure submitting job")
|
||||
return
|
||||
|
||||
log.info("({}) queued with identifier: {}".format(galaxy_id_tag, external_job_id))
|
||||
log.info(f"({galaxy_id_tag}) queued with identifier: {external_job_id}")
|
||||
|
||||
# store runner information for tracking if Galaxy restarts
|
||||
job_wrapper.set_external_id(external_job_id)
|
||||
@@ -131,8 +131,8 @@ class ShellJobRunner(AsynchronousJobRunner):
|
||||
cmd_out = shell.execute(job_interface.submit(job_file))
|
||||
if cmd_out.returncode == 0:
|
||||
return cmd_out.returncode, cmd_out.stdout
|
||||
stdout = '({}) submission failed (stdout): {}'.format(galaxy_id_tag, cmd_out.stdout)
|
||||
stderr = '({}) submission failed (stderr): {}'.format(galaxy_id_tag, cmd_out.stderr)
|
||||
stdout = f'({galaxy_id_tag}) submission failed (stdout): {cmd_out.stdout}'
|
||||
stderr = f'({galaxy_id_tag}) submission failed (stderr): {cmd_out.stderr}'
|
||||
if retry > 0:
|
||||
log.debug("%s, retrying in %s seconds", stdout, timeout)
|
||||
log.debug("%s, retrying in %s seconds", stderr, timeout)
|
||||
@@ -161,15 +161,15 @@ class ShellJobRunner(AsynchronousJobRunner):
|
||||
if ajs.job_wrapper.get_state() == model.Job.states.DELETED:
|
||||
continue
|
||||
|
||||
log.debug("({}/{}) job not found in batch state check".format(id_tag, external_job_id))
|
||||
log.debug(f"({id_tag}/{external_job_id}) job not found in batch state check")
|
||||
shell_params, job_params = self.parse_destination_params(ajs.job_destination.params)
|
||||
shell, job_interface = self.get_cli_plugins(shell_params, job_params)
|
||||
cmd_out = shell.execute(job_interface.get_single_status(external_job_id))
|
||||
state = job_interface.parse_single_status(cmd_out.stdout, external_job_id)
|
||||
if not state == model.Job.states.OK:
|
||||
log.warning('({}/{}) job not found in batch state check, but found in individual state check'.format(id_tag, external_job_id))
|
||||
log.warning(f'({id_tag}/{external_job_id}) job not found in batch state check, but found in individual state check')
|
||||
if state != old_state:
|
||||
log.debug("({}/{}) state change: from {} to {}".format(id_tag, external_job_id, old_state, state))
|
||||
log.debug(f"({id_tag}/{external_job_id}) state change: from {old_state} to {state}")
|
||||
if not state == model.Job.states.OK:
|
||||
# No need to change_state when the state is OK, this will be handled by `self.finish_job`
|
||||
ajs.job_wrapper.change_state(state)
|
||||
@@ -186,7 +186,7 @@ class ShellJobRunner(AsynchronousJobRunner):
|
||||
external_metadata = not asbool(ajs.job_wrapper.job_destination.params.get("embed_metadata_in_job", DEFAULT_EMBED_METADATA_IN_JOB))
|
||||
if external_metadata:
|
||||
self.work_queue.put((self.handle_metadata_externally, ajs))
|
||||
log.debug('({}/{}) job execution finished, running job wrapper finish method'.format(id_tag, external_job_id))
|
||||
log.debug(f'({id_tag}/{external_job_id}) job execution finished, running job wrapper finish method')
|
||||
self.work_queue.put((self.finish_job, ajs))
|
||||
else:
|
||||
new_watched.append(ajs)
|
||||
@@ -234,9 +234,9 @@ class ShellJobRunner(AsynchronousJobRunner):
|
||||
shell, job_interface = self.get_cli_plugins(shell_params, job_params)
|
||||
cmd_out = shell.execute(job_interface.delete(job.job_runner_external_id))
|
||||
assert cmd_out.returncode == 0, cmd_out.stderr
|
||||
log.debug("({}/{}) Terminated at user's request".format(job.id, job.job_runner_external_id))
|
||||
log.debug(f"({job.id}/{job.job_runner_external_id}) Terminated at user's request")
|
||||
except Exception as e:
|
||||
log.debug("({}/{}) User killed running job, but error encountered during termination: {}".format(job.id, job.job_runner_external_id, e))
|
||||
log.debug(f"({job.id}/{job.job_runner_external_id}) User killed running job, but error encountered during termination: {e}")
|
||||
|
||||
def recover(self, job, job_wrapper):
|
||||
"""Recovers jobs stuck in the queued/running state when Galaxy started"""
|
||||
@@ -250,12 +250,12 @@ class ShellJobRunner(AsynchronousJobRunner):
|
||||
ajs.job_wrapper = job_wrapper
|
||||
ajs.job_destination = job_wrapper.job_destination
|
||||
if job.state == model.Job.states.RUNNING:
|
||||
log.debug("({}/{}) is still in running state, adding to the runner monitor queue".format(job.id, job.job_runner_external_id))
|
||||
log.debug(f"({job.id}/{job.job_runner_external_id}) is still in running state, adding to the runner monitor queue")
|
||||
ajs.old_state = model.Job.states.RUNNING
|
||||
ajs.running = True
|
||||
self.monitor_queue.put(ajs)
|
||||
elif job.state == model.Job.states.QUEUED:
|
||||
log.debug("({}/{}) is still in queued state, adding to the runner monitor queue".format(job.id, job.job_runner_external_id))
|
||||
log.debug(f"({job.id}/{job.job_runner_external_id}) is still in queued state, adding to the runner monitor queue")
|
||||
ajs.old_state = model.Job.states.QUEUED
|
||||
ajs.running = False
|
||||
self.monitor_queue.put(ajs)
|
||||
|
||||
@@ -63,7 +63,7 @@ class CondorJobRunner(AsynchronousJobRunner):
|
||||
Remove this function in 21.01
|
||||
"""
|
||||
if cjs.job_wrapper is not None:
|
||||
job_file = "{}/galaxy_{}.sh".format(self.app.config.cluster_files_directory, cjs.job_wrapper.job_id)
|
||||
job_file = f"{self.app.config.cluster_files_directory}/galaxy_{cjs.job_wrapper.job_id}.sh"
|
||||
if not os.path.exists(cjs.job_file) and os.path.exists(job_file):
|
||||
cluster_files_dir_and_id = (self.app.config.cluster_files_directory, cjs.job_wrapper.get_id_tag())
|
||||
cjs.output_file = "%s/galaxy_%s.o" % cluster_files_dir_and_id
|
||||
@@ -155,11 +155,11 @@ class CondorJobRunner(AsynchronousJobRunner):
|
||||
job_wrapper.cleanup()
|
||||
return
|
||||
|
||||
log.debug("({}) submitting file {}".format(galaxy_id_tag, executable))
|
||||
log.debug(f"({galaxy_id_tag}) submitting file {executable}")
|
||||
|
||||
external_job_id, message = condor_submit(submit_file)
|
||||
if external_job_id is None:
|
||||
log.debug("condor_submit failed for job {}: {}".format(job_wrapper.get_id_tag(), message))
|
||||
log.debug(f"condor_submit failed for job {job_wrapper.get_id_tag()}: {message}")
|
||||
if self.app.config.cleanup_job == "always":
|
||||
os.unlink(submit_file)
|
||||
cjs.cleanup()
|
||||
@@ -168,7 +168,7 @@ class CondorJobRunner(AsynchronousJobRunner):
|
||||
|
||||
os.unlink(submit_file)
|
||||
|
||||
log.info("({}) queued as {}".format(galaxy_id_tag, external_job_id))
|
||||
log.info(f"({galaxy_id_tag}) queued as {external_job_id}")
|
||||
|
||||
# store runner information for tracking if Galaxy restarts
|
||||
job_wrapper.set_external_id(external_job_id)
|
||||
@@ -201,8 +201,8 @@ class CondorJobRunner(AsynchronousJobRunner):
|
||||
cjs.user_log_size = log_size
|
||||
except Exception:
|
||||
# so we don't kill the monitor thread
|
||||
log.exception("({}/{}) Unable to check job status".format(galaxy_id_tag, job_id))
|
||||
log.warning("({}/{}) job will now be errored".format(galaxy_id_tag, job_id))
|
||||
log.exception(f"({galaxy_id_tag}/{job_id}) Unable to check job status")
|
||||
log.warning(f"({galaxy_id_tag}/{job_id}) job will now be errored")
|
||||
cjs.fail_message = "Cluster could not complete job"
|
||||
self.work_queue.put((self.fail_job, cjs))
|
||||
continue
|
||||
@@ -212,10 +212,10 @@ class CondorJobRunner(AsynchronousJobRunner):
|
||||
cjs.job_wrapper.check_for_entry_points()
|
||||
|
||||
if job_running and not cjs.running:
|
||||
log.debug("({}/{}) job is now running".format(galaxy_id_tag, job_id))
|
||||
log.debug(f"({galaxy_id_tag}/{job_id}) job is now running")
|
||||
cjs.job_wrapper.change_state(model.Job.states.RUNNING)
|
||||
if not job_running and cjs.running:
|
||||
log.debug("({}/{}) job has stopped running".format(galaxy_id_tag, job_id))
|
||||
log.debug(f"({galaxy_id_tag}/{job_id}) job has stopped running")
|
||||
# Will switching from RUNNING to QUEUED confuse Galaxy?
|
||||
# cjs.job_wrapper.change_state( model.Job.states.QUEUED )
|
||||
if job_complete:
|
||||
@@ -223,11 +223,11 @@ class CondorJobRunner(AsynchronousJobRunner):
|
||||
external_metadata = not asbool(cjs.job_wrapper.job_destination.params.get("embed_metadata_in_job", True))
|
||||
if external_metadata:
|
||||
self._handle_metadata_externally(cjs.job_wrapper, resolve_requirements=True)
|
||||
log.debug("({}/{}) job has completed".format(galaxy_id_tag, job_id))
|
||||
log.debug(f"({galaxy_id_tag}/{job_id}) job has completed")
|
||||
self.work_queue.put((self.finish_job, cjs))
|
||||
continue
|
||||
if job_failed:
|
||||
log.debug("({}/{}) job failed".format(galaxy_id_tag, job_id))
|
||||
log.debug(f"({galaxy_id_tag}/{job_id}) job failed")
|
||||
cjs.failed = True
|
||||
self.work_queue.put((self.finish_job, cjs))
|
||||
continue
|
||||
@@ -243,7 +243,7 @@ class CondorJobRunner(AsynchronousJobRunner):
|
||||
galaxy_id_tag = job_wrapper.get_id_tag()
|
||||
if job.container:
|
||||
try:
|
||||
log.info("stop_job(): {}: trying to stop container .... ({})".format(job.id, external_id))
|
||||
log.info(f"stop_job(): {job.id}: trying to stop container .... ({external_id})")
|
||||
# self.watched = [cjs for cjs in self.watched if cjs.job_id != external_id]
|
||||
new_watch_list = list()
|
||||
cjs = None
|
||||
@@ -260,21 +260,21 @@ class CondorJobRunner(AsynchronousJobRunner):
|
||||
external_metadata = not asbool(cjs.job_wrapper.job_destination.params.get("embed_metadata_in_job", True))
|
||||
if external_metadata:
|
||||
self._handle_metadata_externally(cjs.job_wrapper, resolve_requirements=True)
|
||||
log.debug("({}/{}) job has completed".format(galaxy_id_tag, external_id))
|
||||
log.debug(f"({galaxy_id_tag}/{external_id}) job has completed")
|
||||
self.work_queue.put((self.finish_job, cjs))
|
||||
except Exception as e:
|
||||
log.warning("stop_job(): {}: trying to stop container failed. ({})".format(job.id, e))
|
||||
log.warning(f"stop_job(): {job.id}: trying to stop container failed. ({e})")
|
||||
try:
|
||||
self._kill_container(job_wrapper)
|
||||
except Exception as e:
|
||||
log.warning("stop_job(): {}: trying to kill container failed. ({})".format(job.id, e))
|
||||
log.warning(f"stop_job(): {job.id}: trying to kill container failed. ({e})")
|
||||
failure_message = condor_stop(external_id)
|
||||
if failure_message:
|
||||
log.debug("({}). Failed to stop condor {}".format(external_id, failure_message))
|
||||
log.debug(f"({external_id}). Failed to stop condor {failure_message}")
|
||||
else:
|
||||
failure_message = condor_stop(external_id)
|
||||
if failure_message:
|
||||
log.debug("({}). Failed to stop condor {}".format(external_id, failure_message))
|
||||
log.debug(f"({external_id}). Failed to stop condor {failure_message}")
|
||||
|
||||
def recover(self, job, job_wrapper):
|
||||
"""Recovers jobs stuck in the queued/running state when Galaxy started"""
|
||||
@@ -293,11 +293,11 @@ class CondorJobRunner(AsynchronousJobRunner):
|
||||
cjs.register_cleanup_file_attribute('user_log')
|
||||
self.__old_state_paths(cjs) # remove in 21.01
|
||||
if job.state == model.Job.states.RUNNING:
|
||||
log.debug("({}/{}) is still in running state, adding to the DRM queue".format(job.id, job.job_runner_external_id))
|
||||
log.debug(f"({job.id}/{job.job_runner_external_id}) is still in running state, adding to the DRM queue")
|
||||
cjs.running = True
|
||||
self.monitor_queue.put(cjs)
|
||||
elif job.state == model.Job.states.QUEUED:
|
||||
log.debug("({}/{}) is still in DRM queued state, adding to the DRM queue".format(job.id, job.job_runner_external_id))
|
||||
log.debug(f"({job.id}/{job.job_runner_external_id}) is still in DRM queued state, adding to the DRM queue")
|
||||
cjs.running = False
|
||||
self.monitor_queue.put(cjs)
|
||||
|
||||
@@ -317,7 +317,7 @@ class CondorJobRunner(AsynchronousJobRunner):
|
||||
return self._run_command(cont.container_info['commands'][command], external_id)[0]
|
||||
|
||||
def _run_command(self, command, external_job_id):
|
||||
command = 'condor_ssh_to_job {} {}'.format(external_job_id, command)
|
||||
command = f'condor_ssh_to_job {external_job_id} {command}'
|
||||
|
||||
p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, close_fds=True, preexec_fn=os.setpgrp)
|
||||
stdout, stderr = p.communicate()
|
||||
|
||||
@@ -106,7 +106,7 @@ class DRMAAJobRunner(AsynchronousJobRunner):
|
||||
native_spec = url.split('/')[2]
|
||||
if native_spec:
|
||||
params = dict(nativeSpecification=native_spec)
|
||||
log.debug("Converted URL '{}' to destination runner=drmaa, params={}".format(url, params))
|
||||
log.debug(f"Converted URL '{url}' to destination runner=drmaa, params={params}")
|
||||
return JobDestination(runner='drmaa', params=params)
|
||||
else:
|
||||
log.debug("Converted URL '%s' to destination runner=drmaa" % url)
|
||||
@@ -218,7 +218,7 @@ class DRMAAJobRunner(AsynchronousJobRunner):
|
||||
if external_job_id is None:
|
||||
job_wrapper.fail("(%s) could not queue job" % galaxy_id_tag)
|
||||
return
|
||||
log.info("({}) queued as {}".format(galaxy_id_tag, external_job_id))
|
||||
log.info(f"({galaxy_id_tag}) queued as {external_job_id}")
|
||||
|
||||
# store runner information for tracking if Galaxy restarts
|
||||
job_wrapper.set_external_id(external_job_id)
|
||||
@@ -274,7 +274,7 @@ class DRMAAJobRunner(AsynchronousJobRunner):
|
||||
galaxy_id_tag = ajs.job_wrapper.get_id_tag()
|
||||
state = None
|
||||
try:
|
||||
assert external_job_id not in (None, 'None'), '({}/{}) Invalid job id'.format(galaxy_id_tag, external_job_id)
|
||||
assert external_job_id not in (None, 'None'), f'({galaxy_id_tag}/{external_job_id}) Invalid job id'
|
||||
state = self.ds.job_status(external_job_id)
|
||||
# Reset exception retries
|
||||
for retry_exception in RETRY_EXCEPTIONS_LOWER:
|
||||
@@ -306,8 +306,8 @@ class DRMAAJobRunner(AsynchronousJobRunner):
|
||||
return None
|
||||
except Exception:
|
||||
# so we don't kill the monitor thread
|
||||
log.exception("({}/{}) unable to check job status".format(galaxy_id_tag, external_job_id))
|
||||
log.warning("({}/{}) job will now be errored".format(galaxy_id_tag, external_job_id))
|
||||
log.exception(f"({galaxy_id_tag}/{external_job_id}) unable to check job status")
|
||||
log.warning(f"({galaxy_id_tag}/{external_job_id}) job will now be errored")
|
||||
ajs.fail_message = "Cluster could not complete job"
|
||||
self.work_queue.put((self.fail_job, ajs))
|
||||
return None
|
||||
@@ -361,13 +361,13 @@ class DRMAAJobRunner(AsynchronousJobRunner):
|
||||
cmd = shlex.split(kill_script)
|
||||
cmd.extend([str(ext_id), str(self.userid)])
|
||||
commands.execute(cmd)
|
||||
log.info("({}/{}) Removed from DRM queue at user's request".format(job.id, ext_id))
|
||||
log.info(f"({job.id}/{ext_id}) Removed from DRM queue at user's request")
|
||||
except drmaa.InvalidJobException:
|
||||
log.exception("({}/{}) User killed running job, but it was already dead".format(job.id, ext_id))
|
||||
log.exception(f"({job.id}/{ext_id}) User killed running job, but it was already dead")
|
||||
except commands.CommandLineException as e:
|
||||
log.error("({}/{}) User killed running job, but command execution failed: {}".format(job.id, ext_id, unicodify(e)))
|
||||
except Exception:
|
||||
log.exception("({}/{}) User killed running job, but error encountered removing from DRM queue".format(job.id, ext_id))
|
||||
log.exception(f"({job.id}/{ext_id}) User killed running job, but error encountered removing from DRM queue")
|
||||
|
||||
def recover(self, job, job_wrapper):
|
||||
"""Recovers jobs stuck in the queued/running state when Galaxy started"""
|
||||
@@ -381,12 +381,12 @@ class DRMAAJobRunner(AsynchronousJobRunner):
|
||||
ajs.job_wrapper = job_wrapper
|
||||
ajs.job_destination = job_wrapper.job_destination
|
||||
if job.state == model.Job.states.RUNNING:
|
||||
log.debug("({}/{}) is still in running state, adding to the DRM queue".format(job.id, job.get_job_runner_external_id()))
|
||||
log.debug(f"({job.id}/{job.get_job_runner_external_id()}) is still in running state, adding to the DRM queue")
|
||||
ajs.old_state = drmaa.JobState.RUNNING
|
||||
ajs.running = True
|
||||
self.monitor_queue.put(ajs)
|
||||
elif job.get_state() == model.Job.states.QUEUED:
|
||||
log.debug("({}/{}) is still in DRM queued state, adding to the DRM queue".format(job.id, job.get_job_runner_external_id()))
|
||||
log.debug(f"({job.id}/{job.get_job_runner_external_id()}) is still in DRM queued state, adding to the DRM queue")
|
||||
ajs.old_state = drmaa.JobState.QUEUED_ACTIVE
|
||||
ajs.running = False
|
||||
self.monitor_queue.put(ajs)
|
||||
@@ -395,10 +395,10 @@ class DRMAAJobRunner(AsynchronousJobRunner):
|
||||
""" Stores the content of a DRMAA JobTemplate object in a file as a JSON string.
|
||||
Path is hard-coded, but it's no worse than other path in this module.
|
||||
Uses Galaxy's JobID, so file is expected to be unique."""
|
||||
filename = "{}/{}.jt_json".format(self.app.config.cluster_files_directory, job_wrapper.get_id_tag())
|
||||
filename = f"{self.app.config.cluster_files_directory}/{job_wrapper.get_id_tag()}.jt_json"
|
||||
with open(filename, 'w+') as fp:
|
||||
json.dump(jt, fp)
|
||||
log.debug('({}) Job script for external submission is: {}'.format(job_wrapper.job_id, filename))
|
||||
log.debug(f'({job_wrapper.job_id}) Job script for external submission is: {filename}')
|
||||
return filename
|
||||
|
||||
def external_runjob(self, external_runjob_script, jobtemplate_filename, username):
|
||||
|
||||
@@ -253,13 +253,13 @@ class GodockerJobRunner(AsynchronousJobRunner):
|
||||
job_wrapper.command_line = job.command_line
|
||||
ajs.job_wrapper = job_wrapper
|
||||
if job.state == model.Job.states.RUNNING:
|
||||
log.debug("({}/{}) is still in running state, adding to the god queue".format(job.id, job.get_job_runner_external_id()))
|
||||
log.debug(f"({job.id}/{job.get_job_runner_external_id()}) is still in running state, adding to the god queue")
|
||||
ajs.old_state = 'R'
|
||||
ajs.running = True
|
||||
self.monitor_queue.put(ajs)
|
||||
|
||||
elif job.state == model.Job.states.QUEUED:
|
||||
log.debug("({}/{}) is still in god queued state, adding to the god queue".format(job.id, job.get_job_runner_external_id()))
|
||||
log.debug(f"({job.id}/{job.get_job_runner_external_id()}) is still in god queued state, adding to the god queue")
|
||||
ajs.old_state = 'Q'
|
||||
ajs.running = False
|
||||
self.monitor_queue.put(ajs)
|
||||
|
||||
@@ -527,7 +527,7 @@ class KubernetesJobRunner(AsynchronousJobRunner):
|
||||
self.__cleanup_k8s_job(job_to_delete)
|
||||
# TODO assert whether job parallelism == 0
|
||||
# assert not job_to_delete.exists(), "Could not delete job,"+job.job_runner_external_id+" it still exists"
|
||||
log.debug("({}/{}) Terminated at user's request".format(job.id, job.job_runner_external_id))
|
||||
log.debug(f"({job.id}/{job.job_runner_external_id}) Terminated at user's request")
|
||||
except Exception as e:
|
||||
log.exception("({}/{}) User killed running job, but error encountered during termination: {}".format(
|
||||
job.id, job.job_runner_external_id, e))
|
||||
|
||||
@@ -94,7 +94,7 @@ class LocalJobRunner(BaseJobRunner):
|
||||
try:
|
||||
stdout_file = tempfile.NamedTemporaryFile(mode='wb+', suffix='_stdout', dir=job_wrapper.working_directory)
|
||||
stderr_file = tempfile.NamedTemporaryFile(mode='wb+', suffix='_stderr', dir=job_wrapper.working_directory)
|
||||
log.debug('({}) executing job script: {}'.format(job_id, job_file))
|
||||
log.debug(f'({job_id}) executing job script: {job_file}')
|
||||
# The preexec_fn argument of Popen() is used to call os.setpgrp() in
|
||||
# the child process just before the child is executed. This will set
|
||||
# the PGID of the child process to its PID (i.e. ensures that it is
|
||||
|
||||
@@ -130,7 +130,7 @@ class PBSJobRunner(AsynchronousJobRunner):
|
||||
pbs_destination = '@%s' % server
|
||||
pbs_queue = url_split[3] or None
|
||||
if pbs_queue is not None:
|
||||
pbs_destination = '{}{}'.format(pbs_queue, pbs_destination)
|
||||
pbs_destination = f'{pbs_queue}{pbs_destination}'
|
||||
|
||||
params = dict(destination=pbs_destination)
|
||||
|
||||
@@ -147,7 +147,7 @@ class PBSJobRunner(AsynchronousJobRunner):
|
||||
param, value = opt.split(None, 1)
|
||||
params[param] = value
|
||||
|
||||
log.debug("Converted URL '{}' to destination runner=pbs, params={}".format(url, params))
|
||||
log.debug(f"Converted URL '{url}' to destination runner=pbs, params={params}")
|
||||
|
||||
# Create a dynamic JobDestination
|
||||
return JobDestination(runner='pbs', params=params)
|
||||
@@ -186,7 +186,7 @@ class PBSJobRunner(AsynchronousJobRunner):
|
||||
try:
|
||||
rval.append(dict(name=getattr(pbs, 'ATTR_' + arg), value=value))
|
||||
except AttributeError as e:
|
||||
raise Exception("Invalid parameter '{}': {}".format(arg, e))
|
||||
raise Exception(f"Invalid parameter '{arg}': {e}")
|
||||
return rval
|
||||
|
||||
def __get_pbs_server(self, job_destination_params):
|
||||
@@ -230,13 +230,13 @@ class PBSJobRunner(AsynchronousJobRunner):
|
||||
if c <= 0:
|
||||
errno, text = pbs.error()
|
||||
job_wrapper.fail("Unable to queue job for execution. Resubmitting the job may succeed.")
|
||||
log.error("Connection to PBS server for submit failed: {}: {}".format(errno, text))
|
||||
log.error(f"Connection to PBS server for submit failed: {errno}: {text}")
|
||||
return
|
||||
|
||||
# define job attributes
|
||||
ofile = "{}/{}.o".format(self.app.config.cluster_files_directory, job_wrapper.job_id)
|
||||
efile = "{}/{}.e".format(self.app.config.cluster_files_directory, job_wrapper.job_id)
|
||||
ecfile = "{}/{}.ec".format(self.app.config.cluster_files_directory, job_wrapper.job_id)
|
||||
ofile = f"{self.app.config.cluster_files_directory}/{job_wrapper.job_id}.o"
|
||||
efile = f"{self.app.config.cluster_files_directory}/{job_wrapper.job_id}.e"
|
||||
ecfile = f"{self.app.config.cluster_files_directory}/{job_wrapper.job_id}.ec"
|
||||
|
||||
output_fnames = job_wrapper.get_output_fnames()
|
||||
|
||||
@@ -262,7 +262,7 @@ class PBSJobRunner(AsynchronousJobRunner):
|
||||
]
|
||||
|
||||
# define PBS job options
|
||||
attrs.append(dict(name=pbs.ATTR_N, value=str("{}_{}_{}".format(job_wrapper.job_id, job_wrapper.tool.id, job_wrapper.user))))
|
||||
attrs.append(dict(name=pbs.ATTR_N, value=str(f"{job_wrapper.job_id}_{job_wrapper.tool.id}_{job_wrapper.user}")))
|
||||
job_attrs = pbs.new_attropl(len(attrs) + len(pbs_options))
|
||||
for i, attr in enumerate(attrs + pbs_options):
|
||||
job_attrs[i].name = attr['name']
|
||||
@@ -287,7 +287,7 @@ class PBSJobRunner(AsynchronousJobRunner):
|
||||
|
||||
env_setup_commands = [stage_commands]
|
||||
script = self.get_job_file(job_wrapper, exit_code_path=ecfile, env_setup_commands=env_setup_commands, shell=job_wrapper.shell)
|
||||
job_file = "{}/{}.sh".format(self.app.config.cluster_files_directory, job_wrapper.job_id)
|
||||
job_file = f"{self.app.config.cluster_files_directory}/{job_wrapper.job_id}.sh"
|
||||
self.write_executable_script(job_file, script)
|
||||
# job was deleted while we were preparing it
|
||||
if job_wrapper.get_state() == model.Job.states.DELETED:
|
||||
@@ -302,7 +302,7 @@ class PBSJobRunner(AsynchronousJobRunner):
|
||||
# The job tag includes the job and the task identifier
|
||||
# (if a TaskWrapper was passed in):
|
||||
galaxy_job_id = job_wrapper.get_id_tag()
|
||||
log.debug("({}) submitting file {}".format(galaxy_job_id, job_file))
|
||||
log.debug(f"({galaxy_job_id}) submitting file {job_file}")
|
||||
|
||||
tries = 0
|
||||
while tries < 5:
|
||||
@@ -320,9 +320,9 @@ class PBSJobRunner(AsynchronousJobRunner):
|
||||
return
|
||||
|
||||
if pbs_queue_name is None:
|
||||
log.debug("({}) queued in default queue as {}".format(galaxy_job_id, job_id))
|
||||
log.debug(f"({galaxy_job_id}) queued in default queue as {job_id}")
|
||||
else:
|
||||
log.debug("({}) queued in {} queue as {}".format(galaxy_job_id, pbs_queue_name, job_id))
|
||||
log.debug(f"({galaxy_job_id}) queued in {pbs_queue_name} queue as {job_id}")
|
||||
|
||||
# persist destination
|
||||
job_wrapper.set_job_destination(job_destination, job_id)
|
||||
@@ -356,7 +356,7 @@ class PBSJobRunner(AsynchronousJobRunner):
|
||||
old_state = pbs_job_state.old_state
|
||||
pbs_server_name = self.__get_pbs_server(pbs_job_state.job_destination.params)
|
||||
if pbs_server_name in failures:
|
||||
log.debug("({}/{}) Skipping state check because PBS server connection failed".format(galaxy_job_id, job_id))
|
||||
log.debug(f"({galaxy_job_id}/{job_id}) Skipping state check because PBS server connection failed")
|
||||
new_watched.append(pbs_job_state)
|
||||
continue
|
||||
try:
|
||||
@@ -367,13 +367,13 @@ class PBSJobRunner(AsynchronousJobRunner):
|
||||
try:
|
||||
# Recheck to make sure it wasn't a communication problem
|
||||
self.check_single_job(pbs_server_name, job_id)
|
||||
log.warning("({}/{}) PBS job was not in state check list, but was found with individual state check".format(galaxy_job_id, job_id))
|
||||
log.warning(f"({galaxy_job_id}/{job_id}) PBS job was not in state check list, but was found with individual state check")
|
||||
new_watched.append(pbs_job_state)
|
||||
except Exception:
|
||||
errno, text = pbs.error()
|
||||
if errno == 15001:
|
||||
# 15001 == job not in queue
|
||||
log.debug("({}/{}) PBS job has left queue".format(galaxy_job_id, job_id))
|
||||
log.debug(f"({galaxy_job_id}/{job_id}) PBS job has left queue")
|
||||
self.work_queue.put((self.finish_job, pbs_job_state))
|
||||
else:
|
||||
# Unhandled error, continue to monitor
|
||||
@@ -381,7 +381,7 @@ class PBSJobRunner(AsynchronousJobRunner):
|
||||
new_watched.append(pbs_job_state)
|
||||
continue
|
||||
if status.job_state != old_state:
|
||||
log.debug("({}/{}) PBS job state changed from {} to {}".format(galaxy_job_id, job_id, old_state, status.job_state))
|
||||
log.debug(f"({galaxy_job_id}/{job_id}) PBS job state changed from {old_state} to {status.job_state}")
|
||||
if status.job_state == "R" and not pbs_job_state.running:
|
||||
pbs_job_state.running = True
|
||||
pbs_job_state.job_wrapper.change_state(model.Job.states.RUNNING)
|
||||
@@ -396,18 +396,18 @@ class PBSJobRunner(AsynchronousJobRunner):
|
||||
# "keep_completed" is enabled in PBS, so try to check exit status
|
||||
try:
|
||||
assert int(status.exit_status) == 0
|
||||
log.debug("({}/{}) PBS job has completed successfully".format(galaxy_job_id, job_id))
|
||||
log.debug(f"({galaxy_job_id}/{job_id}) PBS job has completed successfully")
|
||||
except AssertionError:
|
||||
exit_status = int(status.exit_status)
|
||||
error_message = JOB_EXIT_STATUS.get(exit_status, 'Unknown error: %s' % status.exit_status)
|
||||
pbs_job_state.fail_message = CLUSTER_ERROR_MESSAGE % error_message
|
||||
log.error('({}/{}) PBS job failed: {}'.format(galaxy_job_id, job_id, error_message))
|
||||
log.error(f'({galaxy_job_id}/{job_id}) PBS job failed: {error_message}')
|
||||
pbs_job_state.stop_job = False
|
||||
self.work_queue.put((self.fail_job, pbs_job_state))
|
||||
continue
|
||||
except AttributeError:
|
||||
# No exit_status, can't verify proper completion so we just have to assume success.
|
||||
log.debug("({}/{}) PBS job has completed".format(galaxy_job_id, job_id))
|
||||
log.debug(f"({galaxy_job_id}/{job_id}) PBS job has completed")
|
||||
self.work_queue.put((self.finish_job, pbs_job_state))
|
||||
continue
|
||||
pbs_job_state.old_state = status.job_state
|
||||
@@ -497,14 +497,14 @@ class PBSJobRunner(AsynchronousJobRunner):
|
||||
stage_name = os.path.join(self.app.config.pbs_stage_path, os.path.split(fname)[1])
|
||||
else:
|
||||
stage_name = fname
|
||||
stage += "{}@{}:{}".format(stage_name, self.app.config.pbs_dataset_server, fname)
|
||||
stage += f"{stage_name}@{self.app.config.pbs_dataset_server}:{fname}"
|
||||
return stage
|
||||
|
||||
def stop_job(self, job_wrapper):
|
||||
"""Attempts to delete a job from the PBS queue"""
|
||||
job = job_wrapper.get_job()
|
||||
job_id = job.get_job_runner_external_id().encode('utf-8')
|
||||
job_tag = "({}/{})".format(job.get_id_tag(), job_id)
|
||||
job_tag = f"({job.get_id_tag()}/{job_id})"
|
||||
log.debug("%s Stopping PBS job" % job_tag)
|
||||
|
||||
# Declare the connection handle c so that it can be cleaned up:
|
||||
@@ -526,7 +526,7 @@ class PBSJobRunner(AsynchronousJobRunner):
|
||||
% job_tag)
|
||||
except Exception:
|
||||
e = traceback.format_exc()
|
||||
log.debug("{} Unable to stop job: {}".format(job_tag, e))
|
||||
log.debug(f"{job_tag} Unable to stop job: {e}")
|
||||
finally:
|
||||
# Cleanup: disconnect from the server.
|
||||
if (None is not c):
|
||||
@@ -536,22 +536,22 @@ class PBSJobRunner(AsynchronousJobRunner):
|
||||
"""Recovers jobs stuck in the queued/running state when Galaxy started"""
|
||||
job_id = job.get_job_runner_external_id()
|
||||
pbs_job_state = AsynchronousJobState()
|
||||
pbs_job_state.output_file = "{}/{}.o".format(self.app.config.cluster_files_directory, job.id)
|
||||
pbs_job_state.error_file = "{}/{}.e".format(self.app.config.cluster_files_directory, job.id)
|
||||
pbs_job_state.exit_code_file = "{}/{}.ec".format(self.app.config.cluster_files_directory, job.id)
|
||||
pbs_job_state.job_file = "{}/{}.sh".format(self.app.config.cluster_files_directory, job.id)
|
||||
pbs_job_state.output_file = f"{self.app.config.cluster_files_directory}/{job.id}.o"
|
||||
pbs_job_state.error_file = f"{self.app.config.cluster_files_directory}/{job.id}.e"
|
||||
pbs_job_state.exit_code_file = f"{self.app.config.cluster_files_directory}/{job.id}.ec"
|
||||
pbs_job_state.job_file = f"{self.app.config.cluster_files_directory}/{job.id}.sh"
|
||||
pbs_job_state.job_id = str(job_id)
|
||||
pbs_job_state.runner_url = job_wrapper.get_job_runner_url()
|
||||
pbs_job_state.job_destination = job_wrapper.job_destination
|
||||
job_wrapper.command_line = job.command_line
|
||||
pbs_job_state.job_wrapper = job_wrapper
|
||||
if job.state == model.Job.states.RUNNING:
|
||||
log.debug("({}/{}) is still in running state, adding to the PBS queue".format(job.id, job.get_job_runner_external_id()))
|
||||
log.debug(f"({job.id}/{job.get_job_runner_external_id()}) is still in running state, adding to the PBS queue")
|
||||
pbs_job_state.old_state = 'R'
|
||||
pbs_job_state.running = True
|
||||
self.monitor_queue.put(pbs_job_state)
|
||||
elif job.state == model.Job.states.QUEUED:
|
||||
log.debug("({}/{}) is still in PBS queued state, adding to the PBS queue".format(job.id, job.get_job_runner_external_id()))
|
||||
log.debug(f"({job.id}/{job.get_job_runner_external_id()}) is still in PBS queued state, adding to the PBS queue")
|
||||
pbs_job_state.old_state = 'Q'
|
||||
pbs_job_state.running = False
|
||||
self.monitor_queue.put(pbs_job_state)
|
||||
|
||||
@@ -539,7 +539,7 @@ class PulsarJobRunner(AsynchronousJobRunner):
|
||||
def get_client_from_wrapper(self, job_wrapper):
|
||||
job_id = job_wrapper.job_id
|
||||
if hasattr(job_wrapper, 'task_id'):
|
||||
job_id = "{}_{}".format(job_id, job_wrapper.task_id)
|
||||
job_id = f"{job_id}_{job_wrapper.task_id}"
|
||||
params = job_wrapper.job_destination.params.copy()
|
||||
user = job_wrapper.get_job().user
|
||||
if user:
|
||||
@@ -685,7 +685,7 @@ class PulsarJobRunner(AsynchronousJobRunner):
|
||||
# Remote kill
|
||||
pulsar_url = job.job_runner_name
|
||||
job_id = job.job_runner_external_id
|
||||
log.debug("Attempt remote Pulsar kill of job with url {} and id {}".format(pulsar_url, job_id))
|
||||
log.debug(f"Attempt remote Pulsar kill of job with url {pulsar_url} and id {job_id}")
|
||||
client = self.get_client(job.destination_params, job_id)
|
||||
client.kill()
|
||||
|
||||
@@ -1010,7 +1010,7 @@ class PulsarComputeEnvironment(ComputeEnvironment):
|
||||
# first but that adds untested logic that wouln't ever be used.
|
||||
remote_input_path = self.path_mapper.remote_input_path_rewrite(metadata_val, client_input_path_type=CLIENT_INPUT_PATH_TYPES.INPUT_METADATA_PATH)
|
||||
if remote_input_path:
|
||||
log.info("input_metadata_rewrite is {} from {}".format(remote_input_path, metadata_val))
|
||||
log.info(f"input_metadata_rewrite is {remote_input_path} from {metadata_val}")
|
||||
self.path_rewrites_input_metadata[metadata_val] = remote_input_path
|
||||
return remote_input_path
|
||||
|
||||
|
||||
@@ -93,7 +93,7 @@ def _handle_resubmit_definitions(resubmit_definitions, app, job_runner, job_stat
|
||||
|
||||
external_id = getattr(job_state, "job_id", None)
|
||||
if external_id:
|
||||
job_log_prefix = "({}/{})".format(job_state.job_wrapper.job_id, job_state.job_id)
|
||||
job_log_prefix = f"({job_state.job_wrapper.job_id}/{job_state.job_id})"
|
||||
else:
|
||||
job_log_prefix = "(%s)" % (job_state.job_wrapper.job_id)
|
||||
|
||||
|
||||
@@ -91,13 +91,13 @@ class UnivaJobRunner(DRMAAJobRunner):
|
||||
ajs.fail_message = "This job failed because it was cancelled."
|
||||
drmaa_state = self.drmaa.JobState.FAILED
|
||||
elif ("signal" in extinfo and extinfo["signal"] == "SIGKILL") and time_wasted > time_granted:
|
||||
log.error('({tag}/{jobid}) Job hit walltime'.format(tag=ajs.job_wrapper.get_id_tag(), jobid=ajs.job_id))
|
||||
log.error(f'({ajs.job_wrapper.get_id_tag()}/{ajs.job_id}) Job hit walltime')
|
||||
ajs.fail_message = "This job was terminated because it ran longer than the maximum allowed job run time."
|
||||
ajs.runner_state = ajs.runner_states.WALLTIME_REACHED
|
||||
drmaa_state = self.drmaa.JobState.FAILED
|
||||
# test wasted>granted memory only if failed != 0 and exit_status != 0, ie if marked as failed
|
||||
elif state == self.drmaa.JobState.FAILED and mem_wasted > mem_granted * slots:
|
||||
log.error('({idtag}/{jobid}) Job hit memory limit ({used}>{limit})'.format(idtag=ajs.job_wrapper.get_id_tag(), jobid=ajs.job_id, used=mem_wasted, limit=mem_granted))
|
||||
log.error(f'({ajs.job_wrapper.get_id_tag()}/{ajs.job_id}) Job hit memory limit ({mem_wasted}>{mem_granted})')
|
||||
ajs.fail_message = "This job was terminated because it used more than the maximum allowed memory."
|
||||
ajs.runner_state = ajs.runner_states.MEMORY_LIMIT_REACHED
|
||||
drmaa_state = self.drmaa_job_states.FAILED
|
||||
@@ -106,10 +106,10 @@ class UnivaJobRunner(DRMAAJobRunner):
|
||||
# TODO return True?
|
||||
return True # job was not actually terminal
|
||||
elif state == self.drmaa.JobState.UNDETERMINED:
|
||||
log.warning('({tag}/{jobid}) Job state could not be determined'.format(tag=ajs.job_wrapper.get_id_tag(), jobid=ajs.job_id))
|
||||
log.warning(f'({ajs.job_wrapper.get_id_tag()}/{ajs.job_id}) Job state could not be determined')
|
||||
drmaa_state = self.drmaa_job_states.FAILED
|
||||
else:
|
||||
log.error("DRMAAUniva: job {job_id} determined unknown state {state}".format(job_id=ajs.job_id, state=state))
|
||||
log.error(f"DRMAAUniva: job {ajs.job_id} determined unknown state {state}")
|
||||
drmaa_state = self.drmaa_job_states.FAILED
|
||||
# by default, finish the job with the state from drmaa
|
||||
return super()._complete_terminal_job(ajs, drmaa_state=drmaa_state)
|
||||
@@ -203,7 +203,7 @@ class UnivaJobRunner(DRMAAJobRunner):
|
||||
try:
|
||||
stdout = commands.execute(cmd).strip()
|
||||
except commands.CommandLineException as e:
|
||||
if slp <= 32 and "job id {jobid} not found".format(jobid=job_id) in e.stderr:
|
||||
if slp <= 32 and f"job id {job_id} not found" in e.stderr:
|
||||
time.sleep(slp)
|
||||
slp *= 2
|
||||
continue
|
||||
@@ -397,7 +397,7 @@ class UnivaJobRunner(DRMAAJobRunner):
|
||||
extinfo["slots"] = float(rv.resourceUsage['slots'])
|
||||
# log.debug("wait -> \texitStatus {0}\thasCoreDump {1}\thasExited {2}\thasSignal {3}\tjobId {4}\t\tterminatedSignal {5}\twasAborted {6}\tresourceUsage {7}".format(rv.exitStatus, rv.hasCoreDump, rv.hasExited, rv.hasSignal, rv.jobId, rv.terminatedSignal, rv.wasAborted, rv.resourceUsage))
|
||||
if rv.wasAborted:
|
||||
log.error("DRMAAUniva: job {job_id} was aborted according to wait()".format(job_id=job_id))
|
||||
log.error(f"DRMAAUniva: job {job_id} was aborted according to wait()")
|
||||
extinfo["deleted"] = True
|
||||
return self.drmaa.JobState.FAILED
|
||||
|
||||
@@ -405,19 +405,19 @@ class UnivaJobRunner(DRMAAJobRunner):
|
||||
# but also violation of scheduler constraints
|
||||
state = self.drmaa.JobState.DONE
|
||||
if rv.exitStatus != 0:
|
||||
log.error("DRMAAUniva: job {job_id} has exit status {status}".format(job_id=job_id, status=rv.exitStatus))
|
||||
log.error(f"DRMAAUniva: job {job_id} has exit status {rv.exitStatus}")
|
||||
extinfo["state"] = self.drmaa.JobState.FAILED
|
||||
|
||||
if not rv.hasExited or rv.hasSignal:
|
||||
if rv.hasCoreDump != 0:
|
||||
log.error("DRMAAUniva: job {job_id} has core dump".format(job_id=job_id))
|
||||
log.error(f"DRMAAUniva: job {job_id} has core dump")
|
||||
extinfo["state"] = self.drmaa.JobState.FAILED
|
||||
elif len(rv.terminatedSignal) > 0:
|
||||
log.error("DRMAAUniva: job {job_id} was kill by signal {signal}".format(job_id=job_id, signal=rv.terminatedSignal))
|
||||
log.error(f"DRMAAUniva: job {job_id} was kill by signal {rv.terminatedSignal}")
|
||||
state = self.drmaa.JobState.FAILED
|
||||
extinfo["signal"] = rv.terminatedSignal
|
||||
elif rv.wasAborted == 0:
|
||||
log.error("DRMAAUniva: job {job_id} has finished in unclear condition".format(job_id=job_id))
|
||||
log.error(f"DRMAAUniva: job {job_id} has finished in unclear condition")
|
||||
state = self.drmaa.JobState.FAILED
|
||||
# log.debug("UnivaJobRunner._get_drmaa_state_wait ({jobid}) -> {state}".format(jobid=job_id, state=self.drmaa_job_state_strings[state]))
|
||||
return state
|
||||
@@ -535,7 +535,7 @@ class UnivaJobRunner(DRMAAJobRunner):
|
||||
elif "w" in state:
|
||||
return self.drmaa.JobState.QUEUED_ACTIVE
|
||||
else:
|
||||
log.error("DRMAAUniva: job {job_id} unknown state from qstat: {state}".format(job_id=job_id, state=state))
|
||||
log.error(f"DRMAAUniva: job {job_id} unknown state from qstat: {state}")
|
||||
return self.drmaa.JobState.UNDETERMINED
|
||||
|
||||
|
||||
@@ -569,12 +569,12 @@ def _parse_native_specs(job_id, native_spec):
|
||||
if m is not None:
|
||||
tme = _parse_time(m.group(1))
|
||||
if tme is None:
|
||||
log.error("DRMAAUniva: job {job_id} has unparsable time native spec {spec}".format(job_id=job_id, spec=native_spec))
|
||||
log.error(f"DRMAAUniva: job {job_id} has unparsable time native spec {native_spec}")
|
||||
# parse memory
|
||||
m = re.search(r"mem=([\d.]+[KGMT]?)[\s,]*", native_spec)
|
||||
if m is not None:
|
||||
mem = size_to_bytes(m.group(1))
|
||||
# mem = _parse_mem(m.group(1))
|
||||
if mem is None:
|
||||
log.error("DRMAAUniva: job {job_id} has unparsable memory native spec {spec}".format(job_id=job_id, spec=native_spec))
|
||||
log.error(f"DRMAAUniva: job {job_id} has unparsable memory native spec {native_spec}")
|
||||
return tme, mem
|
||||
|
||||
@@ -39,7 +39,7 @@ class CliInterface:
|
||||
def __load_from_path(module_path):
|
||||
base_module = importlib.import_module(module_path)
|
||||
for module_info in pkgutil.iter_modules(base_module.__path__):
|
||||
module = importlib.import_module('{}.{}'.format(module_path, module_info.name))
|
||||
module = importlib.import_module(f'{module_path}.{module_info.name}')
|
||||
yield module
|
||||
|
||||
def __load(module_path, d):
|
||||
|
||||
@@ -52,7 +52,7 @@ class LSF(BaseJobExec):
|
||||
# Generated template.
|
||||
template_scriptargs = ''
|
||||
for k, v in scriptargs.items():
|
||||
template_scriptargs += '#BSUB {} {}\n'.format(k, v)
|
||||
template_scriptargs += f'#BSUB {k} {v}\n'
|
||||
return dict(headers=template_scriptargs)
|
||||
|
||||
def submit(self, script_file):
|
||||
|
||||
@@ -47,7 +47,7 @@ class Slurm(BaseJobExec):
|
||||
# Generated template.
|
||||
template_scriptargs = ''
|
||||
for k, v in scriptargs.items():
|
||||
template_scriptargs += '#SBATCH {} {}\n'.format(k, v)
|
||||
template_scriptargs += f'#SBATCH {k} {v}\n'
|
||||
return dict(headers=template_scriptargs)
|
||||
|
||||
def submit(self, script_file):
|
||||
|
||||
@@ -59,7 +59,7 @@ class Torque(BaseJobExec):
|
||||
log.warning(ERROR_MESSAGE_UNRECOGNIZED_ARG % k)
|
||||
template_pbsargs = ''
|
||||
for k, v in pbsargs.items():
|
||||
template_pbsargs += '#PBS {} {}\n'.format(k, v)
|
||||
template_pbsargs += f'#PBS {k} {v}\n'
|
||||
return dict(headers=template_pbsargs)
|
||||
|
||||
def submit(self, script_file):
|
||||
|
||||
@@ -58,7 +58,7 @@ def build_submit_description(executable, output, error, user_log, query_params):
|
||||
|
||||
submit_description = []
|
||||
for key, value in all_query_params.items():
|
||||
submit_description.append('{} = {}'.format(key, value))
|
||||
submit_description.append(f'{key} = {value}')
|
||||
submit_description.append('executable = ' + executable)
|
||||
submit_description.append('output = ' + output)
|
||||
submit_description.append('error = ' + error)
|
||||
|
||||
@@ -29,7 +29,7 @@ def env_to_statement(env):
|
||||
return execute
|
||||
name = env['name']
|
||||
value = __escape(env['value'], env)
|
||||
return '{}={}; export {}'.format(name, value, name)
|
||||
return f'{name}={value}; export {name}'
|
||||
|
||||
|
||||
def __escape(value, env):
|
||||
|
||||
@@ -5,12 +5,11 @@ IPC with multiple process configurations.
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shlex
|
||||
import socket
|
||||
import subprocess
|
||||
import threading
|
||||
|
||||
from six.moves import shlex_quote
|
||||
|
||||
from galaxy.util import (
|
||||
listify,
|
||||
sleeper,
|
||||
@@ -75,12 +74,12 @@ class TransferManager:
|
||||
# not the case, this process will need to be moved to a
|
||||
# non-blocking method.
|
||||
cmd = self.command + [tj.id]
|
||||
log.debug('Transfer command is: %s', ' '.join(map(shlex_quote, cmd)))
|
||||
log.debug('Transfer command is: %s', ' '.join(map(shlex.quote, cmd)))
|
||||
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
p.wait()
|
||||
output = p.stdout.read(32768)
|
||||
if p.returncode != 0:
|
||||
log.error('Spawning transfer job failed: {}: {}'.format(tj.id, output))
|
||||
log.error(f'Spawning transfer job failed: {tj.id}: {output}')
|
||||
tj.state = tj.states.ERROR
|
||||
tj.info = 'Spawning transfer job failed: %s' % output.splitlines()[-1]
|
||||
self.sa_session.add(tj)
|
||||
@@ -154,7 +153,7 @@ class TransferManager:
|
||||
except Exception:
|
||||
self.sa_session.refresh(tj)
|
||||
if tj.state == tj.states.RUNNING:
|
||||
log.error('Transfer job {} is marked as running but pid {} appears to be dead.'.format(tj.id, tj.pid))
|
||||
log.error(f'Transfer job {tj.id} is marked as running but pid {tj.pid} appears to be dead.')
|
||||
dead.append(tj)
|
||||
if dead:
|
||||
self.run(dead)
|
||||
|
||||
@@ -114,8 +114,8 @@ def get_object(trans, id, class_name, check_ownership=False, check_accessible=Fa
|
||||
item = trans.sa_session.query(item_class).get(decoded_id)
|
||||
assert item is not None
|
||||
except Exception:
|
||||
log.exception("Invalid {} id ( {} ) specified.".format(class_name, id))
|
||||
raise exceptions.MessageException("Invalid {} id ( {} ) specified".format(class_name, id), type="error")
|
||||
log.exception(f"Invalid {class_name} id ( {id} ) specified.")
|
||||
raise exceptions.MessageException(f"Invalid {class_name} id ( {id} ) specified", type="error")
|
||||
|
||||
if check_ownership or check_accessible:
|
||||
security_check(trans, item, check_ownership, check_accessible)
|
||||
|
||||
@@ -67,7 +67,7 @@ def parse_citation(elem, citation_manager):
|
||||
try:
|
||||
citation = citation_class(elem, citation_manager)
|
||||
except Exception as e:
|
||||
raise Exception("Invalid citation of type '{}' with content '{}': {}".format(citation_type, elem.text, e))
|
||||
raise Exception(f"Invalid citation of type '{citation_type}' with content '{elem.text}': {e}")
|
||||
return citation
|
||||
|
||||
|
||||
|
||||
@@ -248,7 +248,7 @@ class CloudManager(sharable.SharableModelManager):
|
||||
try:
|
||||
bucket = connection.storage.buckets.get(bucket_name)
|
||||
if bucket is None:
|
||||
raise RequestParameterInvalidException("The bucket `{}` not found.".format(bucket_name))
|
||||
raise RequestParameterInvalidException(f"The bucket `{bucket_name}` not found.")
|
||||
except Exception as e:
|
||||
raise ItemAccessibilityException("Could not get the bucket `{}`: {}".format(bucket_name, util.unicodify(e)))
|
||||
|
||||
@@ -354,7 +354,7 @@ class CloudManager(sharable.SharableModelManager):
|
||||
incoming = (util.Params(args, sanitize=False)).__dict__
|
||||
d2c = trans.app.toolbox.get_tool(SEND_TOOL, SEND_TOOL_VERSION)
|
||||
if not d2c:
|
||||
log.debug("Failed to get the `send` tool per user `{}` request.".format(trans.user.id))
|
||||
log.debug(f"Failed to get the `send` tool per user `{trans.user.id}` request.")
|
||||
failed.append(json.dumps(
|
||||
{
|
||||
"object": object_label,
|
||||
|
||||
@@ -110,7 +110,7 @@ class CloudAuthzsDeserializer(base.ModelDeserializer):
|
||||
decoded_authn_id = self.app.security.decode_id(val)
|
||||
except Exception:
|
||||
log.debug("cannot decode authz_id `" + str(val) + "`")
|
||||
raise MalformedId("Invalid `authz_id` {}!".format(val))
|
||||
raise MalformedId(f"Invalid `authz_id` {val}!")
|
||||
|
||||
trans = context.get("trans")
|
||||
if trans is None:
|
||||
|
||||
@@ -78,12 +78,12 @@ def get_collection(collection, name=""):
|
||||
hdas = []
|
||||
if collection.has_subcollections:
|
||||
for element in collection.elements:
|
||||
subnames, subhdas = get_collection_elements(element.child_collection, name="{}/{}".format(name, element.element_identifier))
|
||||
subnames, subhdas = get_collection_elements(element.child_collection, name=f"{name}/{element.element_identifier}")
|
||||
names.extend(subnames)
|
||||
hdas.extend(subhdas)
|
||||
else:
|
||||
for element in collection.elements:
|
||||
names.append("{}/{}".format(name, element.element_identifier))
|
||||
names.append(f"{name}/{element.element_identifier}")
|
||||
hdas.append(element.dataset_instance)
|
||||
return names, hdas
|
||||
|
||||
@@ -92,7 +92,7 @@ def get_collection_elements(collection, name=""):
|
||||
names = []
|
||||
hdas = []
|
||||
for element in collection.elements:
|
||||
full_element_name = "{}/{}".format(name, element.element_identifier)
|
||||
full_element_name = f"{name}/{element.element_identifier}"
|
||||
if element.is_collection:
|
||||
subnames, subhdas = get_collection(element.child_collection, name=full_element_name)
|
||||
names.extend(subnames)
|
||||
|
||||
@@ -38,7 +38,7 @@ def get_path_key(path_tuple):
|
||||
# we remove the last 2 items of the path tuple (values and list index)
|
||||
return path_key
|
||||
if path_key:
|
||||
path_key = "{}{}{}".format(path_key, sep, p)
|
||||
path_key = f"{path_key}{sep}{p}"
|
||||
else:
|
||||
path_key = p
|
||||
return path_key
|
||||
|
||||
@@ -123,7 +123,7 @@ class LibraryDatasetsManager(datasets.DatasetAssociationManager):
|
||||
continue
|
||||
if key in ('name'):
|
||||
if len(val) < MINIMUM_STRING_LENGTH:
|
||||
raise RequestParameterInvalidException('{} must have at least length of {}'.format(key, MINIMUM_STRING_LENGTH))
|
||||
raise RequestParameterInvalidException(f'{key} must have at least length of {MINIMUM_STRING_LENGTH}')
|
||||
val = validation.validate_and_sanitize_basestring(key, val)
|
||||
validated_payload[key] = val
|
||||
if key in ('misc_info', 'message'):
|
||||
@@ -136,7 +136,7 @@ class LibraryDatasetsManager(datasets.DatasetAssociationManager):
|
||||
validated_payload[key] = val
|
||||
if key in ('genome_build'):
|
||||
if len(val) < MINIMUM_STRING_LENGTH:
|
||||
raise RequestParameterInvalidException('{} must have at least length of {}'.format(key, MINIMUM_STRING_LENGTH))
|
||||
raise RequestParameterInvalidException(f'{key} must have at least length of {MINIMUM_STRING_LENGTH}')
|
||||
val = validation.validate_and_sanitize_basestring(key, val)
|
||||
validated_payload[key] = val
|
||||
if key in ('tags'):
|
||||
|
||||
@@ -42,7 +42,7 @@ GALAXY_FLAVORED_MARKDOWN_CONTAINER_REGEX = r'(?P<container>%s)' % "|".join(GALAX
|
||||
ARG_VAL_REGEX = r'''[\w_\-]+|\"[^\"]+\"|\'[^\']+\''''
|
||||
FUNCTION_ARG = r'\s*\w+\s*=\s*(?:%s)\s*' % ARG_VAL_REGEX
|
||||
# embed commas between arguments
|
||||
FUNCTION_MULTIPLE_ARGS = r'(?P<firstargcall>{})(?P<restargcalls>(?:,{})*)'.format(FUNCTION_ARG, FUNCTION_ARG)
|
||||
FUNCTION_MULTIPLE_ARGS = fr'(?P<firstargcall>{FUNCTION_ARG})(?P<restargcalls>(?:,{FUNCTION_ARG})*)'
|
||||
FUNCTION_MULTIPLE_ARGS_PATTERN = re.compile(FUNCTION_MULTIPLE_ARGS)
|
||||
FUNCTION_CALL_LINE_TEMPLATE = r'\s*%s\s*\((?:' + FUNCTION_MULTIPLE_ARGS + r')?\)\s*'
|
||||
GALAXY_MARKDOWN_FUNCTION_CALL_LINE = re.compile(FUNCTION_CALL_LINE_TEMPLATE % GALAXY_FLAVORED_MARKDOWN_CONTAINER_REGEX)
|
||||
@@ -119,7 +119,7 @@ def validate_galaxy_markdown(galaxy_markdown, internal=True):
|
||||
|
||||
if expecting_container_close_for:
|
||||
template = "Invalid line %d: %s"
|
||||
msg = template % (last_line_no, "close of block for [{expected_for}] expected".format(expected_for=expecting_container_close_for))
|
||||
msg = template % (last_line_no, f"close of block for [{expecting_container_close_for}] expected")
|
||||
raise ValueError(msg)
|
||||
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user